forgedock 1.16.1 → 1.17.0

package/bin/stats/index.mjs

@@ -0,0 +1,154 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * forgedock stats (forge#3835): a deterministic version-over-version report, no model in the loop.
+ * GitHub is the primary source (trusted FORGE:TELEMETRY comments + labels); ~/.forge/stats-cache is an optional,
+ * deletable cache; ~/.forge/runs is an optional extra source for runs that predate telemetry.
+ */
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { aggregate, DEFAULT_MIN_N } from "./aggregate.mjs";
+import { fetchGithubRecords, isValidRepo, toModel } from "./github-source.mjs";
+import { readLocalRuns, runsDir } from "./local-source.mjs";
+import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
+
+const here = dirname(fileURLToPath(import.meta.url));
+
+export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
+                      [--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
+
+Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
+delta against the previous version and a low-n flag for samples under 10. Missing data prints n/a.
+
+  --since <v|date>      only versions >= <v> (e.g. 1.16.0) or activity on/after <date> (YYYY-MM-DD)
+  --repo <owner/repo>   target repo (default: forge.yaml project, else the git origin remote)
+  --json                machine-readable output
+  --refresh             ignore the local cache and re-read everything from GitHub
+  --compact             one-row-per-version table (embedded in the staging review summary)
+  --no-local            ignore ~/.forge/runs (FORGE_RUNS_DIR)
+  --no-cache            do not read or write ~/.forge/stats-cache
+  --min-n N             low-n threshold (default ${DEFAULT_MIN_N})
+  --release-times FILE  "<version> <epoch-seconds>" lines used to estimate versions of unversioned records
+                        (default: git tags v* of the ForgeDock checkout)
+`;
+
+export function parseArgs(argv) {
+  const o = { since: null, repo: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
+  const val = (i, name) => {
+    const v = argv[i + 1];
+    if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
+    return v;
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const eq = a.startsWith("--") && a.includes("=") ? [a.slice(0, a.indexOf("=")), a.slice(a.indexOf("=") + 1)] : null;
+    const name = eq ? eq[0] : a;
+    const take = () => { if (eq) return eq[1]; const v = val(i, name); i++; return v; };
+    if (name === "--since") o.since = take();
+    else if (name === "--repo") o.repo = take();
+    else if (name === "--release-times") o.releaseTimes = take();
+    else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
+    else if (name === "--json") o.json = true;
+    else if (name === "--refresh") o.refresh = true;
+    else if (name === "--compact") o.compact = true;
+    else if (name === "--no-local") o.local = false;
+    else if (name === "--no-cache") o.cache = false;
+    else if (name === "--help" || name === "-h") o.help = true;
+    else throw new Error(`unknown argument: ${a}`);
+  }
+  return o;
+}
+
+/** "--since" is a date (YYYY-MM-DD[...]) or a version (1.16.0). */
+export function parseSince(since) {
+  if (!since) return {};
+  if (/^\d{4}-\d{2}-\d{2}/.test(since)) {
+    const ms = Date.parse(since);
+    if (Number.isNaN(ms)) throw new Error(`invalid --since date: ${since}`);
+    return { sinceMs: ms };
+  }
+  if (/^v?\d+(\.\d+)*$/.test(since)) return { sinceVersion: since.replace(/^v/, "") };
+  throw new Error(`--since must be a version (1.16.0) or a date (YYYY-MM-DD): ${since}`);
+}
+
+export function parseReleaseTimes(text) {
+  return String(text ?? "").split("\n").map((l) => l.trim().split(/\s+/))
+    .filter((p) => p.length === 2 && /^\d+$/.test(p[1])).map((p) => ({ v: p[0].replace(/^v/, ""), t: Number(p[1]) }));
+}
+
+function gitTagTimes() {
+  try {
+    const out = execFileSync("git", ["-C", join(here, "..", ".."), "for-each-ref", "refs/tags", "--format=%(refname:short) %(creatordate:unix)"],
+      { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    return parseReleaseTimes(out.split("\n").filter((l) => /^v\d/.test(l)).join("\n"));
+  } catch { return []; }
+}
+
+/** --repo, else forge.yaml project.owner/repo, else the git origin remote. */
+export function resolveRepo(explicit, cwd = process.cwd()) {
+  if (explicit) return explicit;
+  for (const f of ["forge.yaml", ".forge.yaml"]) {
+    const p = join(cwd, f);
+    if (!existsSync(p)) continue;
+    try {
+      const m = /^project:\s*\n((?:[ \t]+.*\n?|\s*\n)*)/m.exec(readFileSync(p, "utf8"));
+      const owner = /^[ \t]+owner:\s*"?([\w.-]+)"?/m.exec(m?.[1] ?? "")?.[1];
+      const repo = /^[ \t]+repo:\s*"?([\w.-]+)"?/m.exec(m?.[1] ?? "")?.[1];
+      if (owner && repo) return `${owner}/${repo}`;
+    } catch { /* fall through */ }
+  }
+  try {
+    const url = execFileSync("git", ["-C", cwd, "remote", "get-url", "origin"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
+    const m = /[:/]([\w.-]+)\/([\w.-]+?)(?:\.git)?$/.exec(url);
+    if (m) return `${m[1]}/${m[2]}`;
+  } catch { /* none */ }
+  return null;
+}
+
+/** Build the report object. deps: { gh, env, localRuns, now } for tests. Never touches the network when `gh` is stubbed. */
+export function buildReport(opts, deps = {}) {
+  const env = deps.env ?? process.env;
+  const repo = opts.repo;
+  const gh = fetchGithubRecords({ repo, refresh: opts.refresh, useCache: opts.cache, gh: deps.gh, env });
+  const model = toModel(repo, gh);
+  const ghKeys = new Set(model.runs.map((r) => `${r.repo}#${r.issue}`));
+  let local = [];
+  if (opts.local) {
+    local = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo === repo && !ghKeys.has(`${r.repo}#${r.issue}`));
+  }
+  let releaseTimes = [];
+  if (opts.releaseTimes) releaseTimes = parseReleaseTimes(readFileSync(opts.releaseTimes, "utf8"));
+  else if (deps.releaseTimes) releaseTimes = deps.releaseTimes;
+  else releaseTimes = gitTagTimes();
+  const report = aggregate({ runs: [...model.runs, ...local], events: model.events, issues: model.issues },
+    { minN: opts.minN, releaseTimes, ...parseSince(opts.since) });
+  return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report };
+}
+
+export async function runStats(argv, deps = {}) {
+  let opts;
+  try { opts = parseArgs(argv); } catch (err) {
+    process.stderr.write(`forgedock stats: ${err.message}\n\n${HELP}`);
+    return 2;
+  }
+  if (opts.help) { process.stdout.write(HELP); return 0; }
+  opts.repo = resolveRepo(opts.repo);
+  if (!opts.repo || !isValidRepo(opts.repo)) {
+    process.stderr.write(`forgedock stats: could not resolve a valid owner/repo (pass --repo owner/repo)\n`);
+    return 2;
+  }
+  try {
+    const report = buildReport(opts, deps);
+    process.stdout.write(`${opts.json ? renderJson(report) : opts.compact ? renderCompact(report) : renderMarkdown(report)}\n`);
+    return 0;
+  } catch (err) {
+    process.stderr.write(`forgedock stats: ${err.message}\n`);
+    return 1;
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  process.exitCode = await runStats(process.argv.slice(2));
+}

package/bin/stats/local-source.mjs

@@ -0,0 +1,79 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Optional extra source for `forgedock stats` (forge#3835): the engine run logs in ~/.forge/runs (FORGE_RUNS_DIR).
+ * Needed only for runs that predate the GitHub `phase_attempt` / `run_terminal` records. Same counting rules as
+ * scripts/cost-report.sh. Old logs lack `ts`, `code`, `version` and `repo`: those fields become null.
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const runsDir = (env = process.env) => env.FORGE_RUNS_DIR || join(homedir(), ".forge", "runs");
+
+function parseLog(text) {
+  const out = [];
+  for (const line of text.split("\n")) {
+    if (!line.trim()) continue;
+    try { const e = JSON.parse(line); if (e && typeof e === "object") out.push(e); } catch { /* truncated or corrupt line */ }
+  }
+  return out;
+}
+
+const n = (x) => (Number.isFinite(x) ? x : 0);
+
+/** Summarise one log: the last RUN_START..end slice. Returns null when there is no RUN_START. */
+export function summariseLog(events, fallbackIssue = null) {
+  let s = -1;
+  events.forEach((e, i) => { if (e.event === "RUN_START") s = i; });
+  if (s < 0) return null;
+  const ev = events.slice(s);
+  const start = ev[0];
+  let t = -1; let r = -1;
+  ev.forEach((e, i) => {
+    if (e.event === "RUN_TERMINAL") t = i;
+    if (e.event === "RUN_REOPEN" || e.event === "RUN_NEXT_CYCLE") r = i;
+  });
+  const attemptPhases = new Set(ev.filter((e) => e.event === "PHASE_ATTEMPT_USAGE").map((e) => e.phase));
+  const phases = {};
+  let usageEvents = 0;
+  for (const e of ev) {
+    const attempt = e.event === "PHASE_ATTEMPT_USAGE";
+    const legacy = (e.event === "PHASE_COMMIT" || e.event === "PHASE_FAILED") && !attemptPhases.has(e.phase);
+    if (!(attempt || legacy) || !e.usage || typeof e.usage !== "object") continue;
+    usageEvents++;
+    const p = (phases[e.phase ?? "other"] ??= { cr: 0, out: 0, usd: null, ms: null, attempts: 0 });
+    p.cr += n(e.usage.cache_read_input_tokens);
+    p.out += n(e.usage.output_tokens);
+    p.attempts++;
+    if (attempt && Number.isFinite(e.duration_ms)) p.ms = (p.ms ?? 0) + e.duration_ms;
+    if (attempt && Number.isFinite(e.cost_usd)) p.usd = (p.usd ?? 0) + e.cost_usd;
+  }
+  const finished = t >= 0 && (r < 0 || t > r);
+  const startTs = start.ts ?? null;
+  const endTs = finished ? ev[t].ts ?? null : null;
+  const a = Date.parse(startTs ?? ""); const b = Date.parse(endTs ?? "");
+  return {
+    source: "local", repo: start.repo ?? null, version: start.version ?? null, issue: start.issue ?? fallbackIssue, run: start.run ?? null,
+    finished, hydrated: start.hydrated === true, usageEvents,
+    reason: finished ? ev[t].reason ?? null : null, code: finished ? ev[t].code ?? null : null,
+    remediated: ev.some((e) => e.event === "PHASE_COMMIT" && e.phase === "remediate"),
+    startTs, endTs, wallMs: finished && !Number.isNaN(a) && !Number.isNaN(b) ? Math.max(0, b - a) : null,
+    phases,
+  };
+}
+
+/** All run summaries in `dir` (a missing dir or an unreadable log yields nothing, never an error). */
+export function readLocalRuns(dir = runsDir()) {
+  if (!existsSync(dir)) return [];
+  const runs = [];
+  for (const f of readdirSync(dir).sort()) {
+    if (!f.endsWith(".jsonl")) continue;
+    try {
+      const m = /^(\d+)\.jsonl$/.exec(f);
+      const sum = summariseLog(parseLog(readFileSync(join(dir, f), "utf8")), m ? Number(m[1]) : null);
+      if (sum) runs.push({ ...sum, mtimeTs: new Date(statSync(join(dir, f)).mtimeMs).toISOString() });
+    } catch { /* unreadable log */ }
+  }
+  return runs;
+}

package/bin/stats/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "forgedock-stats",
+  "private": true,
+  "description": "Directory entry so `node --test bin/stats/` runs the stats suite on Node versions that treat a directory argument as a module path.",
+  "main": "stats.test.mjs"
+}

package/bin/stats/render.mjs

@@ -0,0 +1,82 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/** Rendering for `forgedock stats` (forge#3835): full markdown, compact table (review-pr-staging) and JSON. */
+
+const NA = "n/a";
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+const k = (x) => (isNum(x) ? `${Math.round(x / 1000)}k` : NA);
+const pct = (x) => (isNum(x) ? `${Math.round(x * 100)}%` : NA);
+const usd = (x) => (isNum(x) ? `$${x.toFixed(2)}` : NA);
+const f1 = (x) => (isNum(x) ? (Math.round(x * 10) / 10).toString() : NA);
+const dur = (ms) => {
+  if (!isNum(ms)) return NA;
+  const s = ms / 1000;
+  if (s < 90) return `${Math.round(s)}s`;
+  if (s < 5400) return `${Math.round(s / 60)}m`;
+  return `${f1(s / 3600)}h`;
+};
+const signed = (x, fmt) => (isNum(x) ? `${x > 0 ? "+" : x < 0 ? "-" : ""}${fmt(Math.abs(x))}` : NA);
+const dpp = (x) => signed(x, (v) => `${Math.round(v * 100)}pp`);
+const flag = (g) => (g.lowN ? ` (low-n <${g.minN})` : "");
+const table = (head, rows) => [`| ${head.join(" | ")} |`, `|${head.map(() => "---").join("|")}|`, ...rows.map((r) => `| ${r.join(" | ")} |`)];
+
+export const renderJson = (report) => JSON.stringify(report, null, 2);
+
+const label = (g) => `${g.version} / ${g.repo}`;
+
+function sections(g) {
+  const out = [`## ${label(g)}  n=${g.n}${flag(g)}`, ""];
+  const c = g.cost;
+  out.push("### Cost", ...table(["metric", "value"], [
+    ["cache-read tokens per issue (mean / median)", `${k(c.cacheReadMean)} / ${k(c.cacheReadMedian)}`],
+    ["output tokens per issue (mean)", k(c.outputMean)],
+    ["per phase (mean)", Object.entries(c.perPhase).filter(([, v]) => isNum(v)).map(([p, v]) => `${p} ${k(v)}`).join(", ") || NA],
+    ["cache-read tokens per merged issue", k(c.perMergedIssue)],
+    ["USD per issue (n=" + c.usdN + ")", usd(c.usdMean)],
+  ]), "");
+  const t = g.time;
+  const phaseRows = Object.entries(t.phases).map(([p, v]) => [p, String(v.n), dur(v.p50), dur(v.p90)]);
+  out.push("### Time", ...table(["phase", "n", "p50", "p90"], [...phaseRows, ["wall per issue", String(t.wall.n), dur(t.wall.p50), dur(t.wall.p90)]]), "");
+  const f = g.flow;
+  out.push("### Flow", ...table(["metric", "value"], [
+    ["merged", pct(f.mergedPct)], ["needs-human", pct(f.needsHumanPct)], ["remediation rate", pct(f.remediationPct)],
+    ["top outcome codes (n=" + f.codesN + ")", f.codes.slice(0, 5).map((x) => `${x.code} ${x.count} (${pct(x.pct)})`).join(", ") || NA],
+  ]), "");
+  const r = g.funnel;
+  const sev = Object.entries(r.bySeverity).map(([s, d]) => [s, String(d.total), f1(d.perReview), String(d.ISSUE), String(d.NOTE), String(d.INPR_FIX)]);
+  out.push("### Review funnel", `${r.reviews} reviews, ${r.findings} findings (${f1(r.perReview)} per review). Validated issues: ${r.validated}, false positives: ${r.falsePositive}.`, "",
+    ...(sev.length ? table(["severity", "findings", "per review", "ISSUE", "NOTE", "in-PR fix"], sev) : [`No finding records: ${NA}`]), "");
+  const q = g.gates;
+  const rate = (x) => (x.n ? `${pct(x.rate)} (${x.blocked}/${x.n})` : NA);
+  out.push("### Gates", ...table(["gate", "block rate"], [
+    ["acceptance", rate(q.acceptance)], ["diff size (OVER)", rate(q.diffSize)], ["ci", rate(q.ci)], ["liveness", rate(q.liveness)],
+  ]), `Malformed-check rate: ${pct(q.malformedRate)}. Diff size: COHESIVE ${q.diffSize.cohesive}, OVER ${q.diffSize.over}.`, "");
+  const i = g.issueFlow;
+  const kv = (o) => Object.entries(o).map(([a, b]) => `${a} ${b}`).join(", ") || NA;
+  out.push("### Issue flow", ...table(["metric", "value"], [
+    ["created by source", kv(i.created)], ["closed by kind", kv(i.closed)],
+    ["net flow (created - closed)", isNum(i.net) ? signed(i.net, String) : NA], ["findings per merged issue", f1(i.findingsPerMerge)],
+  ]), "");
+  if (g.delta) {
+    const d = g.delta;
+    out.push(`Delta vs ${d.against}: cache-read/issue ${signed(d.cacheReadMean, k)}, merged ${dpp(d.mergedPct)}, needs-human ${dpp(d.needsHumanPct)}, remediation ${dpp(d.remediationPct)}, wall p50 ${signed(d.wallP50, dur)}`, "");
+  } else out.push(`Delta: ${NA} (no previous version)`, "");
+  return out;
+}
+
+export function renderMarkdown(report) {
+  const out = ["# ForgeDock stats", ""];
+  if (!report.groups.length) out.push("No data: no finished runs, telemetry records or issues were found.", "");
+  for (const g of report.groups) out.push(...sections(g));
+  const e = report.excluded;
+  out.push(`Finished runs only. Excluded: ${e.unfinished} unfinished, ${e.noUsage} without usage data (hydrated or empty). "~" = version estimated from release times; "unknown" = no version available. Groups with n < ${report.minN} are flagged low-n.`);
+  return out.join("\n");
+}
+
+/** One row per (version, repo): the embedded table for the staging review summary. */
+export function renderCompact(report) {
+  if (!report.groups.length) return "No stats data.";
+  const rows = report.groups.map((g) => [g.version, g.repo, String(g.n), k(g.cost.cacheReadMean), pct(g.flow.mergedPct), pct(g.flow.needsHumanPct),
+    pct(g.flow.remediationPct), dur(g.time.wall.p50), f1(g.funnel.perReview), pct(g.gates.acceptance.rate), g.delta ? signed(g.delta.cacheReadMean, k) : NA, g.lowN ? `low-n (<${g.minN})` : "ok"]);
+  return table(["Version", "Repo", "n", "Cache-read/issue", "Merged", "Needs-human", "Remediated", "Wall p50", "Findings/review", "Acceptance block", "Delta tokens", "Confidence"], rows).join("\n");
+}

package/bin/stats/stats.test.mjs

@@ -0,0 +1,224 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+import { aggregate, compareVersions, percentile } from "./aggregate.mjs";
+import { cacheDir } from "./cache.mjs";
+import { fetchGithubRecords, parseTelemetryComments, runsFromRecords, toModel } from "./github-source.mjs";
+import { buildReport, parseArgs, parseSince, runStats } from "./index.mjs";
+import { readLocalRuns } from "./local-source.mjs";
+import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const fx = (f) => join(here, "fixtures", f);
+const load = (f) => JSON.parse(readFileSync(fx(f), "utf8"));
+const ENV = { FORGE_TRAIL_TRUSTED_ASSOCIATIONS: "OWNER,MEMBER,COLLABORATOR" };
+
+// A fake `gh`: lists the fixture issues (honouring `since`), serves comments per number. Counts every call.
+function fakeGh() {
+  const comments = { 201: load("comments-201.json"), 301: load("comments-301.json") };
+  const issues = load("issues.json");
+  const calls = [];
+  const gh = (args) => {
+    calls.push(args.join(" "));
+    const path = args.find((a) => a.startsWith("repos/"));
+    const m = /issues\/(\d+)\/comments$/.exec(path);
+    if (m) return JSON.stringify(comments[m[1]] ?? []);
+    const since = /since=([^&]+)/.exec(path)?.[1];
+    return JSON.stringify(issues.filter((i) => !since || i.updated_at >= since));
+  };
+  return { gh, calls };
+}
+
+let td;
+beforeEach(() => { td = mkdtempSync(join(tmpdir(), "stats-")); });
+afterEach(() => rmSync(td, { recursive: true, force: true }));
+
+describe("telemetry comment parsing", () => {
+  it("concatenates parts, dedupes on eid and ignores untrusted or unrelated comments", () => {
+    const recs = parseTelemetryComments(load("comments-201.json"), ENV);
+    assert.deepEqual(recs.map((r) => r.eid), ["e1", "e2", "e3", "e4", "e5"]);
+    assert.ok(!recs.some((r) => r.eid === "e90"), "untrusted User comment must not be read");
+  });
+  it("skips torn lines and keeps identical-looking events that have different eids", () => {
+    const body = "<!-- FORGE:TELEMETRY v1 -->\n```jsonl\n{\"event\":\"a\",\"eid\":\"1\"}\n{\"event\":\"a\",\"eid\":\"2\"}\n{torn\n";
+    const recs = parseTelemetryComments([{ body, user: { type: "Bot" } }], ENV);
+    assert.equal(recs.length, 2);
+  });
+  it("trusts OWNER/MEMBER/COLLABORATOR, not NONE users", () => {
+    const body = "<!-- FORGE:TELEMETRY v1 -->\n```jsonl\n{\"event\":\"a\",\"eid\":\"1\"}\n```";
+    assert.equal(parseTelemetryComments([{ body, author_association: "MEMBER", user: { type: "User" } }], ENV).length, 1);
+    assert.equal(parseTelemetryComments([{ body, author_association: "NONE", user: { type: "User" } }], ENV).length, 0);
+  });
+});
+
+describe("GitHub-only report (fresh machine)", () => {
+  it("rebuilds cost, time, flow, funnel, gates and issue flow from GitHub alone", () => {
+    const { gh } = fakeGh();
+    const report = buildReport({ repo: "o/r", local: false, cache: false, minN: 10 }, { gh, env: ENV, releaseTimes: [] });
+    const g = report.groups.find((x) => x.version === "1.17.0");
+    assert.equal(g.n, 1);
+    assert.equal(g.cost.cacheReadMean, 3000000);
+    assert.equal(g.cost.usdMean, 1.2);
+    assert.equal(g.time.wall.p50, 900000);
+    assert.equal(g.time.phases.build.p50, 500000);
+    assert.equal(g.flow.mergedPct, 1);
+    assert.deepEqual(g.flow.codes.map((c) => c.code), ["completed"]);
+    assert.equal(g.funnel.reviews, 1);
+    assert.equal(g.funnel.findings, 3);
+    assert.equal(g.funnel.bySeverity.HIGH.total, 2);
+    assert.equal(g.funnel.bySeverity.HIGH.ISSUE, 1);
+    assert.equal(g.funnel.bySeverity.HIGH.INPR_FIX, 1);
+    assert.equal(g.gates.acceptance.rate, 1, "duplicate eid counted once");
+    assert.equal(g.gates.malformedRate, 1);
+    assert.equal(g.gates.ci.rate, 0.5);
+    assert.deepEqual(g.issueFlow.created, { pipeline: 1 });
+    assert.equal(g.issueFlow.net, 0);
+    const u = report.groups.find((x) => x.version === "unknown");
+    assert.deepEqual(u.issueFlow.created, { "review-finding": 2, human: 1 });
+    assert.equal(u.funnel.validated, 1);
+    assert.equal(u.funnel.falsePositive, 1);
+    assert.equal(u.issueFlow.net, 2);
+    assert.equal(u.issueFlow.findingsPerMerge, null);
+    assert.ok(!JSON.stringify(report).includes("999999999"), "untrusted record leaked into the report");
+  });
+  it("matches the run-record field names written by the engine seam", () => {
+    const runs = runsFromRecords(parseTelemetryComments(load("comments-201.json"), ENV));
+    assert.equal(runs.length, 1);
+    assert.equal(runs[0].finished, true);
+    assert.equal(runs[0].phases.review.cr, 500000);
+  });
+});
+
+describe("local runs, joins, version delta and low-n", () => {
+  const runs = () => readLocalRuns(fx("runs"));
+  const agg = (opts = {}) => aggregate({ runs: runs() }, { releaseTimes: [], ...opts });
+
+  it("applies the cost-report counting rules", () => {
+    const r = agg();
+    assert.deepEqual(r.excluded, { unfinished: 1, noUsage: 1 });
+    const g = r.groups.find((x) => x.version === "1.16.0");
+    assert.equal(g.n, 2);
+    assert.equal(g.cost.cacheReadMean, 4500000);
+    assert.equal(g.cost.perPhase.build, 4000000);
+    assert.equal(g.cost.perPhase.remediate, 0);
+    assert.equal(g.cost.perMergedIssue, 9000000);
+    assert.equal(g.flow.mergedPct, 0.5);
+    assert.equal(g.flow.needsHumanPct, 0.5);
+    assert.equal(g.flow.remediationPct, 0.5);
+    assert.equal(g.cost.usdN, 1);
+    assert.equal(g.time.wall.p50, 1500000);
+    const legacy = r.groups.find((x) => x.version === "unknown");
+    assert.equal(legacy.cost.cacheReadMean, 4000000, "legacy PHASE_COMMIT usage counted when no attempt events exist");
+    assert.equal(legacy.flow.codesN, 0, "old logs have no outcome codes");
+  });
+  it("computes the delta against the previous version of the same repo", () => {
+    const g = agg().groups.find((x) => x.version === "1.17.0");
+    assert.equal(g.delta.against, "1.16.0");
+    assert.equal(g.delta.cacheReadMean, 2000000 - 4500000);
+    assert.equal(g.delta.mergedPct, 0.5);
+    assert.equal(agg().groups.find((x) => x.version === "1.16.0").delta, null);
+  });
+  it("flags samples under 10 and honours --min-n", () => {
+    assert.ok(agg().groups.every((g) => g.lowN));
+    assert.equal(agg({ minN: 2 }).groups.find((x) => x.version === "1.16.0").lowN, false);
+    assert.match(renderMarkdown(agg()), /low-n/);
+  });
+  it("agrees with scripts/cost-report.sh on the same logs", { skip: spawnSync("jq", ["--version"]).status !== 0 }, () => {
+    writeFileSync(join(td, "rel.txt"), "");
+    const out = spawnSync("bash", [join(here, "..", "..", "scripts", "cost-report.sh"), "--runs-dir", fx("runs"), "--release-times", join(td, "rel.txt")], { encoding: "utf8" }).stdout;
+    const row = out.split("\n").find((l) => l.startsWith("| 1.16.0 |"));
+    assert.ok(row?.includes("| 4500k |"), row);
+    assert.ok(row?.includes("| 9000k |"), row);
+  });
+  it("--since filters by version or date", () => {
+    assert.deepEqual(agg({ ...parseSince("1.17.0") }).groups.map((g) => g.version), ["1.17.0"]);
+    assert.ok(agg({ ...parseSince("2026-10-03") }).groups.every((g) => g.version !== "1.16.0"));
+  });
+  it("estimates versions of unversioned records from release times", () => {
+    const r = aggregate({ runs: runs().filter((x) => x.issue === 105) }, { releaseTimes: [{ v: "1.15.0", t: 1 }] });
+    assert.equal(r.groups[0].version, "1.15.0~");
+  });
+});
+
+describe("n/a degradation and rendering", () => {
+  it("renders n/a, never NaN or undefined, for missing data and empty input", () => {
+    const old = { source: "local", repo: "o/r", version: "1.0.0", issue: 1, finished: true, hydrated: false, usageEvents: 1, reason: "merged", code: null, remediated: false, phases: { build: { cr: 10, out: 0, usd: null, ms: null, attempts: 1 } } };
+    const r = aggregate({ runs: [old] });
+    const md = renderMarkdown(r);
+    assert.match(md, /n\/a/);
+    assert.doesNotMatch(md, /NaN|undefined|Infinity/);
+    assert.doesNotMatch(renderCompact(r), /NaN|undefined/);
+    assert.match(renderMarkdown(aggregate({})), /No data/);
+    assert.equal(percentile([], 90), null);
+  });
+  it("json output is parseable and carries every section", () => {
+    const { gh } = fakeGh();
+    const report = buildReport({ repo: "o/r", local: false, cache: false, minN: 10 }, { gh, env: ENV, releaseTimes: [] });
+    const parsed = JSON.parse(renderJson(report));
+    for (const k of ["cost", "time", "flow", "funnel", "gates", "issueFlow"]) assert.ok(k in parsed.groups[0]);
+    const md = renderMarkdown(report);
+    for (const h of ["### Cost", "### Time", "### Flow", "### Review funnel", "### Gates", "### Issue flow"]) assert.ok(md.includes(h), h);
+  });
+  it("orders versions numerically", () => {
+    assert.ok(compareVersions("1.9.0", "1.10.0") < 0);
+    assert.ok(compareVersions("1.10.0~", "1.10.0") === 0);
+  });
+});
+
+describe("cache", () => {
+  it("refreshes incrementally from the updated_at watermark and survives deletion or corruption", () => {
+    const env = { ...ENV, FORGE_STATS_CACHE_DIR: join(td, "cache") };
+    const a = fakeGh();
+    const first = fetchGithubRecords({ repo: "o/r", gh: a.gh, env });
+    assert.equal(first.fromCache, false);
+    assert.ok(!a.calls[0].includes("since="));
+    const b = fakeGh();
+    const second = fetchGithubRecords({ repo: "o/r", gh: b.gh, env });
+    assert.equal(second.fromCache, true);
+    assert.match(b.calls[0], /since=2026-10-06T13:00:00Z/);
+    assert.deepEqual(second.records.map((r) => r.eid).sort(), first.records.map((r) => r.eid).sort());
+    // corrupt cache -> treated as empty -> same result from a full fetch
+    writeFileSync(join(cacheDir("o/r", env), "cache.json"), "{not json");
+    const c = fakeGh();
+    const third = fetchGithubRecords({ repo: "o/r", gh: c.gh, env });
+    assert.equal(third.fromCache, false);
+    assert.equal(third.records.length, first.records.length);
+    // --refresh ignores a good cache
+    const d = fakeGh();
+    fetchGithubRecords({ repo: "o/r", gh: d.gh, env, refresh: true });
+    assert.ok(!d.calls[0].includes("since="));
+    assert.equal(toModel("o/r", third).issues.length, 5);
+  });
+  it("rejects an unsafe repo before any gh call", () => {
+    const { gh, calls } = fakeGh();
+    assert.throws(() => fetchGithubRecords({ repo: "o/r; rm -rf /", gh }), /invalid repo/);
+    assert.equal(calls.length, 0);
+  });
+});
+
+describe("cli", () => {
+  it("parses flags and rejects bad input", () => {
+    const o = parseArgs(["--since", "1.2.0", "--repo=o/r", "--json", "--refresh", "--min-n", "3"]);
+    assert.deepEqual([o.since, o.repo, o.json, o.refresh, o.minN], ["1.2.0", "o/r", true, true, 3]);
+    assert.throws(() => parseArgs(["--since"]), /needs a value/);
+    assert.throws(() => parseArgs(["--bogus"]), /unknown argument/);
+    assert.throws(() => parseSince("yesterday"), /--since must be/);
+  });
+  it("--help exits 0 and importing the module did not run it", () => {
+    const r = spawnSync(process.execPath, [join(here, "index.mjs"), "--help"], { encoding: "utf8" });
+    assert.equal(r.status, 0);
+    assert.match(r.stdout, /--since/);
+    assert.match(r.stdout, /--refresh/);
+  });
+  it("runStats returns 2 for an unusable repo", async () => {
+    const real = process.stderr.write;
+    process.stderr.write = () => true;
+    try { assert.equal(await runStats(["--repo", "bad repo"]), 2); } finally { process.stderr.write = real; }
+  });
+});