@gigzen/populace 1.3.1 → 1.3.3

package/package.json

@@ -1,60 +1,60 @@
-{
-  "name": "@gigzen/populace",
-  "version": "1.3.1",
-  "description": "A simulated population that uses your app through its real API, so you can test what needs more than one person.",
-  "type": "module",
-  "bin": {
-    "populace": "./src/cli.mjs"
-  },
-  "main": "./src/index.mjs",
-  "exports": {
-    ".": "./src/index.mjs",
-    "./engine": "./src/engine/index.mjs"
-  },
-  "files": [
-    "src",
-    "adapters",
-    "examples",
-    "!**/populace-report.json",
-    "!**/populace-report.html",
-    "!examples/buzzbuzz/run-test.ps1",
-    "populace.config.example.mjs",
-    "README.md",
-    "LICENSE",
-    "action.yml"
-  ],
-  "scripts": {
-    "demo": "node src/cli.mjs demo",
-    "doctor": "node src/cli.mjs doctor",
-    "test": "node src/selftest.mjs",
-    "prepublishOnly": "node src/selftest.mjs"
-  },
-  "engines": {
-    "node": ">=18"
-  },
-  "keywords": [
-    "simulation",
-    "testing",
-    "load-testing",
-    "multi-user",
-    "synthetic-users",
-    "qa"
-  ],
-  "license": "AGPL-3.0-only",
-  "dependencies": {},
-  "devDependencies": {
-    "@supabase/supabase-js": "^2.45.0"
-  },
-  "author": "Sankur Kundu <sankur.kundu.tw@gmail.com>",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/Shakhtar-Sankur/populace.git"
-  },
-  "homepage": "https://github.com/Shakhtar-Sankur/populace#readme",
-  "bugs": {
-    "url": "https://github.com/Shakhtar-Sankur/populace/issues"
-  },
-  "publishConfig": {
-    "access": "public"
-  }
-}
+{
+  "name": "@gigzen/populace",
+  "version": "1.3.3",
+  "description": "A simulated population that uses your app through its real API, so you can test what needs more than one person.",
+  "type": "module",
+  "bin": {
+    "populace": "./src/cli.mjs"
+  },
+  "main": "./src/index.mjs",
+  "exports": {
+    ".": "./src/index.mjs",
+    "./engine": "./src/engine/index.mjs"
+  },
+  "files": [
+    "src",
+    "adapters",
+    "examples",
+    "!**/populace-report.json",
+    "!**/populace-report.html",
+    "!examples/buzzbuzz/run-test.ps1",
+    "populace.config.example.mjs",
+    "README.md",
+    "LICENSE",
+    "action.yml"
+  ],
+  "scripts": {
+    "demo": "node src/cli.mjs demo",
+    "doctor": "node src/cli.mjs doctor",
+    "test": "node src/selftest.mjs",
+    "prepublishOnly": "node src/selftest.mjs"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "simulation",
+    "testing",
+    "load-testing",
+    "multi-user",
+    "synthetic-users",
+    "qa"
+  ],
+  "license": "AGPL-3.0-only",
+  "dependencies": {},
+  "devDependencies": {
+    "@supabase/supabase-js": "^2.45.0"
+  },
+  "author": "Sankur Kundu <sankur.kundu.tw@gmail.com>",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Shakhtar-Sankur/populace.git"
+  },
+  "homepage": "https://github.com/Shakhtar-Sankur/populace#readme",
+  "bugs": {
+    "url": "https://github.com/Shakhtar-Sankur/populace/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/config.mjs

@@ -16,6 +16,40 @@ export class ConfigError extends Error {}
 
 const strip = (u) => String(u || "").trim().replace(/\/+$/, "").toLowerCase();
 
+/**
+ * The hostname inside a string, however it was written.
+ *
+ * Everything here is a variation a person actually types: a trailing slash, a
+ * path on the end, no scheme at all, a port, capitals, stray whitespace. All of
+ * them must come out as the same host, because all of them are the same
+ * database.
+ *
+ * Returns null for a string that is not a host. That matters more than it
+ * looks: the denylist is compared against every string in the target block,
+ * keys and schema names included, and those must not be mistaken for hosts.
+ */
+function hostOf(value) {
+  const s = strip(value).replace(/^[a-z][a-z0-9+.-]*:\/\//, "").replace(/^[^/@]*@/, "");
+  const host = s.split(/[/?#]/)[0].split(":")[0];
+  return /^[a-z0-9.-]+\.[a-z]{2,}$/.test(host) ? host.replace(/^\.+|\.+$/g, "") : null;
+}
+
+/**
+ * Is this target one of the hosts the customer forbade?
+ *
+ * Host equality, or a subdomain of a forbidden host — so naming
+ * `example.com` also stops `api.example.com`, which is what someone listing
+ * their production domain means.
+ *
+ * It is deliberately NOT a substring test. A substring test refuses
+ * `key: "k"` because some forbidden URL happens to contain the letter k, and a
+ * guard that cries wolf over a single character is a guard people switch off.
+ */
+function sameHost(target, denied) {
+  if (!target || !denied) return false;
+  return target === denied || target.endsWith(`.${denied}`) || denied.endsWith(`.${target}`);
+}
+
 /** Pull every string out of a nested object, so we can scan a whole target block. */
 function stringsIn(value, found = []) {
   if (typeof value === "string") found.push(value);
@@ -134,14 +168,24 @@ export function guardProduction(config) {
       .split(",")
       .map((s) => s.trim())
       .filter(Boolean),
-  ].map(strip);
+  ];
+
+  // A denied entry that is not a host is kept as a literal, so a customer who
+  // writes something unusual in there still gets an exact match rather than
+  // silently nothing.
+  const deniedHosts = denied.map(hostOf).filter(Boolean);
+  const deniedLiterals = denied.filter((d) => !hostOf(d)).map(strip).filter(Boolean);
 
   if (denied.length) {
-    const targets = stringsIn(config.target).map(strip).filter(Boolean);
-    const hit = targets.find((t) => denied.some((d) => d && (t === d || t.includes(d) || d.includes(t))));
+    const targets = stringsIn(config.target);
+    const hit = targets.find((raw) => {
+      const host = hostOf(raw);
+      if (host && deniedHosts.some((d) => sameHost(host, d))) return true;
+      return deniedLiterals.includes(strip(raw));
+    });
     if (hit) {
       refuse(
-        `The target matches a host listed in neverRunAgainst:\n    ${hit}`,
+        `The target matches a host listed in neverRunAgainst:\n    ${hostOf(hit) || strip(hit)}`,
         `Simulated people must never be visible to real users.\n` +
           `  Point \`target\` at a separate test environment.`,
       );

package/src/html-report.mjs

@@ -50,6 +50,15 @@ export function renderHtmlReport(report) {
     )
     .join("");
 
+  // Implemented but never called. Kept apart from notTested because the fix is
+  // different: nothing to write, the run just has to reach it.
+  const notExercised = (r.coverage.notExercised || [])
+    .map(
+      (c) =>
+        `<li><span class="mono">${esc(c.method)}</span><span class="would">still untested: ${esc(c.wouldHaveTested)}</span></li>`,
+    )
+    .join("");
+
   const problems = r.verdict.problems.map((p) => `<li>${esc(p)}</li>`).join("");
 
   const engineBlock = engineBroke
@@ -170,7 +179,7 @@ ${engineBlock}
   <div class="stat"><span class="v" style="color:${r.api.failures ? "var(--bad)" : "var(--ok)"}">${r.api.failures}</span><span class="l">failed</span></div>
   <div class="stat"><span class="v">${pct(r.api.failureRate)}</span><span class="l">failure rate</span></div>
   <div class="stat"><span class="v">${r.population.signedIn}</span><span class="l">concurrent users</span></div>
-  <div class="stat"><span class="v">${esc(r.coverage.label)}</span><span class="l">coverage</span></div>
+  <div class="stat"><span class="v">${esc(r.coverage.label)}</span><span class="l">methods exercised</span></div>
 </div>
 
 <section>
@@ -194,10 +203,19 @@ ${engineBlock}
   </div>
 </section>
 
+${
+  notExercised
+    ? `<section>
+  <h2>Not exercised — implemented, but this run never called it</h2>
+  <div class="panel"><ul class="cov">${notExercised}</ul></div>
+</section>`
+    : ""
+}
+
 ${
   notTested
     ? `<section>
-  <h2>Not tested — adapter implements ${esc(r.coverage.label)}</h2>
+  <h2>Not tested — adapter implements ${esc(r.coverage.implementedLabel ?? r.coverage.label)}</h2>
   <div class="panel"><ul class="cov">${notTested}</ul></div>
 </section>`
     : ""

package/src/report.mjs

@@ -66,15 +66,39 @@ export function buildReport({ config, adapter, world, metrics, teardown, started
       groupJoins: totals.groups,
     },
     api,
-    coverage: {
-      label: coverage.label,
-      implemented: coverage.implemented.map((c) => c.method),
-      notTested: coverage.missing.map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
-    },
+    coverage: coverageFor(coverage, api),
     cleanup: teardown ?? { skipped: true, note: "Agents were left in place. Run `populace clean`." },
   };
 }
 
+/**
+ * Coverage counts what RAN, not what exists.
+ *
+ * It used to be the adapter's implemented count, so a three-minute run printed
+ * "Coverage was 13/13" having never once called refreshSession, which fires
+ * every thirty minutes. That is the same overclaim `isStub` exists to stop — a
+ * method credited as tested because it is there — just one step later.
+ *
+ * `label` is exercised/total. `implementedLabel` keeps what the adapter
+ * provides, because "not tested because you never wrote it" and "not tested
+ * because this run never reached it" have different fixes.
+ */
+function coverageFor(coverage, api) {
+  const total = coverage.implemented.length + coverage.missing.length;
+  const called = new Set((api.methods || []).filter((m) => m.calls > 0).map((m) => m.method));
+  const exercised = coverage.implemented.filter((c) => called.has(c.method));
+  return {
+    label: `${exercised.length}/${total}`,
+    implementedLabel: coverage.label,
+    implemented: coverage.implemented.map((c) => c.method),
+    exercised: exercised.map((c) => c.method),
+    notExercised: coverage.implemented
+      .filter((c) => !called.has(c.method))
+      .map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
+    notTested: coverage.missing.map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
+  };
+}
+
 /**
  * Deliberately conservative. Anything unproven is called unproven, never
  * "passed" — including a run where nothing failed because nothing ran.
@@ -256,8 +280,16 @@ export function renderReport(report) {
     }
   }
 
+  if (report.coverage.notExercised?.length) {
+    L.push(`  NOT EXERCISED — implemented, but this run never called it (coverage ${report.coverage.label})`);
+    for (const c of report.coverage.notExercised) {
+      L.push(`    · ${c.method.padEnd(21)} still untested: ${c.wouldHaveTested}`);
+    }
+    L.push("");
+  }
+
   if (report.coverage.notTested.length) {
-    L.push(`  NOT TESTED — adapter implements ${report.coverage.label}`);
+    L.push(`  NOT TESTED — adapter implements ${report.coverage.implementedLabel ?? report.coverage.label}`);
     for (const c of report.coverage.notTested) {
       L.push(`    · ${c.method.padEnd(21)} would have tested ${c.wouldHaveTested}`);
     }

package/src/selftest.mjs

@@ -25,7 +25,9 @@ import {
   normaliseError,
   summarise,
 } from "./instrument.mjs";
+import { guardProduction } from "./config.mjs";
 import { buildReport, renderReport } from "./report.mjs";
+import { renderHtmlReport } from "./html-report.mjs";
 import { canSignInOnly, CONTRACT_METHODS, coverageOf, isStub } from "./contract.mjs";
 import { diagnose } from "./diagnose.mjs";
 import { fill, match } from "./openapi.mjs";
@@ -347,6 +349,58 @@ check("skipped methods are listed with what they would have tested", () => {
   assert.ok(skipped?.wouldHaveTested.length > 10, "a gap should say what it costs you");
 });
 
+// Coverage is what RAN. A hosted run on 12 September printed "Coverage was
+// 13/13" having never called refreshSession, which fires every 30 minutes and
+// the run lasted under five. Implemented is not exercised.
+check("coverage counts methods that ran, not methods that exist", () => {
+  const ran = new Set(cleanReport.api.methods.filter((m) => m.calls > 0).map((m) => m.method));
+  const { coverage } = cleanReport;
+  for (const m of coverage.exercised) assert.ok(ran.has(m), `${m} counted as exercised with no calls`);
+  for (const c of coverage.notExercised) assert.ok(!ran.has(c.method), `${c.method} ran but is listed as not exercised`);
+  assert.equal(
+    coverage.exercised.length + coverage.notExercised.length,
+    coverage.implemented.length,
+    "every implemented method is either exercised or named as not",
+  );
+  assert.equal(coverage.label, `${coverage.exercised.length}/13`, "the headline label is the exercised count");
+});
+
+// A first draft of the next check guarded itself with "if the adapter does not
+// implement refreshSession, return" — and the in-memory adapter does not, so it
+// passed without asserting anything. It now builds the exact case it is about:
+// refreshSession implemented for real, and a run far too short to reach it.
+const idleAdapter = {
+  ...inMemoryAdapter(),
+  async refreshSession(user) {
+    await Promise.resolve();
+    return user;
+  },
+};
+const idleMetrics = createMetrics();
+const idleWorld = new World({
+  adapter: instrument(idleAdapter, idleMetrics),
+  personas: buildPersonas(3, ["manila"]),
+});
+await idleWorld.populate({ staggerMs: 0 });
+await idleWorld.run({ minutes: 1, tickSeconds: 5, realtime: false });
+const idleTeardown = await idleWorld.teardown();
+idleMetrics.endedAt = Date.now();
+const idleReport = buildReport({
+  config, adapter: idleAdapter, world: idleWorld,
+  metrics: idleMetrics, teardown: idleTeardown, startedAt: Date.now() - 5000,
+});
+
+check("an implemented method that never ran is named, and not counted", () => {
+  const { coverage } = idleReport;
+  assert.ok(coverage.implemented.includes("refreshSession"), "setup: refreshSession must be implemented");
+  assert.ok(!coverage.exercised.includes("refreshSession"), "a one-minute run cannot reach a 30-minute refresh");
+  const idle = coverage.notExercised.find((c) => c.method === "refreshSession");
+  assert.ok(idle?.wouldHaveTested.length > 10, "a never-called method should say what stays untested");
+  assert.notEqual(coverage.label, coverage.implementedLabel, "exercised must not quietly equal implemented");
+  assert.ok(/NOT EXERCISED/.test(renderReport(idleReport)), "the terminal report must say so");
+  assert.ok(/Not exercised/.test(renderHtmlReport(idleReport)), "and so must the page people share");
+});
+
 // --- 4b. cleanup that does not write to the customer's database -----------
 // clean reaches an account through createUser, which SIGNS UP when the identity
 // is absent. On an already-clean environment that creates every simulated
@@ -506,6 +560,77 @@ check("doctor names the cleanup mode", () => {
   );
 });
 
+// --- 4c-bis. the denylist, which is the one thing that must never be wrong --
+//
+// Everything else in this file protects a report. This protects a database
+// that real people's accounts are in. It had no coverage until a substring
+// match was found refusing `key: "k"` — and the same test showed the far worse
+// half: because the comparison included the scheme, writing the forbidden host
+// as http:// instead of https:// walked straight past it.
+//
+// So the rule is hosts, not strings, and both directions are asserted: every
+// way of spelling a forbidden host is refused, and nothing else is.
+
+const FORBIDDEN = "prod.example.com";
+const guarded = (target) => {
+  try {
+    guardProduction({
+      _file: "x", environment: "test",
+      neverRunAgainst: [`https://${FORBIDDEN}`], target,
+    });
+    return "ran";
+  } catch { return "refused"; }
+};
+
+for (const [how, target] of Object.entries({
+  "written plainly": `https://${FORBIDDEN}`,
+  "with no scheme": FORBIDDEN,
+  "over http instead of https": `http://${FORBIDDEN}`,
+  "as a websocket url": `wss://${FORBIDDEN}/realtime/v1`,
+  "as a postgres connection string": `postgresql://postgres:pw@${FORBIDDEN}:5432/postgres`,
+  "with credentials in front of it": `https://user:secret@${FORBIDDEN}`,
+  "with a port": `https://${FORBIDDEN}:443`,
+  "with a path, query and hash": `https://${FORBIDDEN}/rest/v1?a=1#b`,
+  "in capitals with a trailing slash": `HTTPS://${FORBIDDEN.toUpperCase()}/`,
+  "padded with whitespace": `   https://${FORBIDDEN}   `,
+  "as a subdomain of it": `https://api.${FORBIDDEN}`,
+  "buried in a nested field": { url: "http://127.0.0.1:54321", replica: `https://${FORBIDDEN}` },
+  "hidden in an array": ["http://127.0.0.1:54321", `https://${FORBIDDEN}`],
+})) {
+  check(`a forbidden host is refused ${how}`, () => {
+    assert.equal(guarded(target), "refused",
+      "this is the guard that keeps simulated people out of a real database");
+  });
+}
+
+check("a short field value is not mistaken for a forbidden host", () => {
+  // "e", "com" and "pro" are all substrings of prod.example.com. None is a host.
+  assert.equal(guarded({ url: "http://127.0.0.1:54321", key: "e", schema: "com", pool: "pro" }), "ran");
+});
+
+check("a different host on the same domain still runs", () => {
+  assert.equal(guarded({ url: "https://staging.example.com", key: "k" }), "ran");
+});
+
+check("naming a domain also covers its subdomains", () => {
+  // Someone who forbids example.com means all of it, not the apex alone.
+  assert.equal(guarded(`https://api.${FORBIDDEN}`), "refused");
+});
+
+check("an empty denylist warns rather than silently allowing", () => {
+  const c = guardProduction({ _file: "x", environment: "test", target: { url: "https://anything.example" } });
+  assert.ok((c._warnings || []).some((w) => /neverRunAgainst is empty/.test(w)));
+});
+
+check("a non-production environment is still required", () => {
+  for (const environment of ["production", "prod", undefined, "", "live"]) {
+    assert.throws(() => guardProduction({
+      _file: "x", environment, target: { url: "http://127.0.0.1:54321" },
+      neverRunAgainst: [`https://${FORBIDDEN}`],
+    }), `environment "${environment}" must not be accepted`);
+  }
+});
+
 // --- 4d. teardown must not claim removals it did not make ------------------
 // selfDestruct() does nothing when there is no account and no deleteUser, and
 // teardown counted both as successes — so a run could print "Cleanup complete