@gigzen/populace 1.3.2 → 1.3.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gigzen/populace",
-  "version": "1.3.2",
+  "version": "1.3.3",
   "description": "A simulated population that uses your app through its real API, so you can test what needs more than one person.",
   "type": "module",
   "bin": {

package/src/html-report.mjs

@@ -50,6 +50,15 @@ export function renderHtmlReport(report) {
     )
     .join("");
 
+  // Implemented but never called. Kept apart from notTested because the fix is
+  // different: nothing to write, the run just has to reach it.
+  const notExercised = (r.coverage.notExercised || [])
+    .map(
+      (c) =>
+        `<li><span class="mono">${esc(c.method)}</span><span class="would">still untested: ${esc(c.wouldHaveTested)}</span></li>`,
+    )
+    .join("");
+
   const problems = r.verdict.problems.map((p) => `<li>${esc(p)}</li>`).join("");
 
   const engineBlock = engineBroke
@@ -170,7 +179,7 @@ ${engineBlock}
   <div class="stat"><span class="v" style="color:${r.api.failures ? "var(--bad)" : "var(--ok)"}">${r.api.failures}</span><span class="l">failed</span></div>
   <div class="stat"><span class="v">${pct(r.api.failureRate)}</span><span class="l">failure rate</span></div>
   <div class="stat"><span class="v">${r.population.signedIn}</span><span class="l">concurrent users</span></div>
-  <div class="stat"><span class="v">${esc(r.coverage.label)}</span><span class="l">coverage</span></div>
+  <div class="stat"><span class="v">${esc(r.coverage.label)}</span><span class="l">methods exercised</span></div>
 </div>
 
 <section>
@@ -194,10 +203,19 @@ ${engineBlock}
   </div>
 </section>
 
+${
+  notExercised
+    ? `<section>
+  <h2>Not exercised — implemented, but this run never called it</h2>
+  <div class="panel"><ul class="cov">${notExercised}</ul></div>
+</section>`
+    : ""
+}
+
 ${
   notTested
     ? `<section>
-  <h2>Not tested — adapter implements ${esc(r.coverage.label)}</h2>
+  <h2>Not tested — adapter implements ${esc(r.coverage.implementedLabel ?? r.coverage.label)}</h2>
   <div class="panel"><ul class="cov">${notTested}</ul></div>
 </section>`
     : ""

package/src/report.mjs

@@ -66,15 +66,39 @@ export function buildReport({ config, adapter, world, metrics, teardown, started
       groupJoins: totals.groups,
     },
     api,
-    coverage: {
-      label: coverage.label,
-      implemented: coverage.implemented.map((c) => c.method),
-      notTested: coverage.missing.map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
-    },
+    coverage: coverageFor(coverage, api),
     cleanup: teardown ?? { skipped: true, note: "Agents were left in place. Run `populace clean`." },
   };
 }
 
+/**
+ * Coverage counts what RAN, not what exists.
+ *
+ * It used to be the adapter's implemented count, so a three-minute run printed
+ * "Coverage was 13/13" having never once called refreshSession, which fires
+ * every thirty minutes. That is the same overclaim `isStub` exists to stop — a
+ * method credited as tested because it is there — just one step later.
+ *
+ * `label` is exercised/total. `implementedLabel` keeps what the adapter
+ * provides, because "not tested because you never wrote it" and "not tested
+ * because this run never reached it" have different fixes.
+ */
+function coverageFor(coverage, api) {
+  const total = coverage.implemented.length + coverage.missing.length;
+  const called = new Set((api.methods || []).filter((m) => m.calls > 0).map((m) => m.method));
+  const exercised = coverage.implemented.filter((c) => called.has(c.method));
+  return {
+    label: `${exercised.length}/${total}`,
+    implementedLabel: coverage.label,
+    implemented: coverage.implemented.map((c) => c.method),
+    exercised: exercised.map((c) => c.method),
+    notExercised: coverage.implemented
+      .filter((c) => !called.has(c.method))
+      .map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
+    notTested: coverage.missing.map((c) => ({ method: c.method, wouldHaveTested: c.exercises })),
+  };
+}
+
 /**
  * Deliberately conservative. Anything unproven is called unproven, never
  * "passed" — including a run where nothing failed because nothing ran.
@@ -256,8 +280,16 @@ export function renderReport(report) {
     }
   }
 
+  if (report.coverage.notExercised?.length) {
+    L.push(`  NOT EXERCISED — implemented, but this run never called it (coverage ${report.coverage.label})`);
+    for (const c of report.coverage.notExercised) {
+      L.push(`    · ${c.method.padEnd(21)} still untested: ${c.wouldHaveTested}`);
+    }
+    L.push("");
+  }
+
   if (report.coverage.notTested.length) {
-    L.push(`  NOT TESTED — adapter implements ${report.coverage.label}`);
+    L.push(`  NOT TESTED — adapter implements ${report.coverage.implementedLabel ?? report.coverage.label}`);
     for (const c of report.coverage.notTested) {
       L.push(`    · ${c.method.padEnd(21)} would have tested ${c.wouldHaveTested}`);
     }

package/src/selftest.mjs

@@ -27,6 +27,7 @@ import {
 } from "./instrument.mjs";
 import { guardProduction } from "./config.mjs";
 import { buildReport, renderReport } from "./report.mjs";
+import { renderHtmlReport } from "./html-report.mjs";
 import { canSignInOnly, CONTRACT_METHODS, coverageOf, isStub } from "./contract.mjs";
 import { diagnose } from "./diagnose.mjs";
 import { fill, match } from "./openapi.mjs";
@@ -348,6 +349,58 @@ check("skipped methods are listed with what they would have tested", () => {
   assert.ok(skipped?.wouldHaveTested.length > 10, "a gap should say what it costs you");
 });
 
+// Coverage is what RAN. A hosted run on 12 September printed "Coverage was
+// 13/13" having never called refreshSession, which fires every 30 minutes and
+// the run lasted under five. Implemented is not exercised.
+check("coverage counts methods that ran, not methods that exist", () => {
+  const ran = new Set(cleanReport.api.methods.filter((m) => m.calls > 0).map((m) => m.method));
+  const { coverage } = cleanReport;
+  for (const m of coverage.exercised) assert.ok(ran.has(m), `${m} counted as exercised with no calls`);
+  for (const c of coverage.notExercised) assert.ok(!ran.has(c.method), `${c.method} ran but is listed as not exercised`);
+  assert.equal(
+    coverage.exercised.length + coverage.notExercised.length,
+    coverage.implemented.length,
+    "every implemented method is either exercised or named as not",
+  );
+  assert.equal(coverage.label, `${coverage.exercised.length}/13`, "the headline label is the exercised count");
+});
+
+// A first draft of the next check guarded itself with "if the adapter does not
+// implement refreshSession, return" — and the in-memory adapter does not, so it
+// passed without asserting anything. It now builds the exact case it is about:
+// refreshSession implemented for real, and a run far too short to reach it.
+const idleAdapter = {
+  ...inMemoryAdapter(),
+  async refreshSession(user) {
+    await Promise.resolve();
+    return user;
+  },
+};
+const idleMetrics = createMetrics();
+const idleWorld = new World({
+  adapter: instrument(idleAdapter, idleMetrics),
+  personas: buildPersonas(3, ["manila"]),
+});
+await idleWorld.populate({ staggerMs: 0 });
+await idleWorld.run({ minutes: 1, tickSeconds: 5, realtime: false });
+const idleTeardown = await idleWorld.teardown();
+idleMetrics.endedAt = Date.now();
+const idleReport = buildReport({
+  config, adapter: idleAdapter, world: idleWorld,
+  metrics: idleMetrics, teardown: idleTeardown, startedAt: Date.now() - 5000,
+});
+
+check("an implemented method that never ran is named, and not counted", () => {
+  const { coverage } = idleReport;
+  assert.ok(coverage.implemented.includes("refreshSession"), "setup: refreshSession must be implemented");
+  assert.ok(!coverage.exercised.includes("refreshSession"), "a one-minute run cannot reach a 30-minute refresh");
+  const idle = coverage.notExercised.find((c) => c.method === "refreshSession");
+  assert.ok(idle?.wouldHaveTested.length > 10, "a never-called method should say what stays untested");
+  assert.notEqual(coverage.label, coverage.implementedLabel, "exercised must not quietly equal implemented");
+  assert.ok(/NOT EXERCISED/.test(renderReport(idleReport)), "the terminal report must say so");
+  assert.ok(/Not exercised/.test(renderHtmlReport(idleReport)), "and so must the page people share");
+});
+
 // --- 4b. cleanup that does not write to the customer's database -----------
 // clean reaches an account through createUser, which SIGNS UP when the identity
 // is absent. On an already-clean environment that creates every simulated