forgedock 1.30.0 → 1.32.0

package/bin/stats/efficiency.test.mjs

@@ -0,0 +1,266 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { tmpDir } from "../tests/_tmp.mjs";
+import { TRIGGERS } from "../lean/fixround.mjs";
+import {
+  TRIGGER_CAUSE, RETRY_CAUSES, WEEK_MS, codeCause, issueSlices, sliceRetries, issueQuality, acceptanceOf, efficiencyMetrics,
+  batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows,
+} from "./efficiency.mjs";
+import { parseArgs, runStats } from "./index.mjs";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const FX = join(here, "fixtures", "efficiency-20261022");
+const AL = "RapierCraftStudios/AlterLab";
+const B1 = "b20261022-181111-1103229", B2 = "b20261022-183421-1103229";
+const NOW = Date.parse("2026-10-23T00:00:00Z");
+const DAY = 86400000;
+
+const start = (ts, repo = "o/r") => ({ event: "RUN_START", ts, repo });
+const att = (phase, o = {}) => ({ event: "PHASE_ATTEMPT_USAGE", phase, turns: 10, cost_usd: 1, usage: { cache_read_input_tokens: 1000 }, model: "m1", ...o });
+const term = (reason, code, o = {}) => ({ event: "RUN_TERMINAL", reason, code, ...o });
+const probe = (result, at, trusted = true) => ({ body: `<!-- FORGE:PROBE -->\n## Acceptance probe\n\`{"fix":"abc","result":"${result}"}\``, created_at: new Date(at).toISOString(),
+  author_association: trusted ? "OWNER" : "NONE", user: { type: trusted ? "Bot" : "User", login: "x" } });
+
+describe("#4402 retry causes", () => {
+  it("classifies every builder trigger (property: TRIGGERS ⊆ TRIGGER_CAUSE)", () => {
+    for (const t of TRIGGERS) assert.ok(Object.hasOwn(TRIGGER_CAUSE, t), `trigger ${t} has no cause`);
+    for (const c of Object.values(TRIGGER_CAUSE)) assert.ok(c === null || RETRY_CAUSES.includes(c));
+    assert.equal(TRIGGER_CAUSE.post_merge, "code"); assert.equal(TRIGGER_CAUSE.duplicate, "code"); // #4406: builder rework on the merged tree
+  });
+  it("maps lane-rerun codes by family", () => {
+    assert.equal(codeCause("env.tool_missing"), "env");
+    assert.equal(codeCause("followup.unavailable"), "engine");
+    assert.equal(codeCause("engine.error"), "engine");
+    assert.equal(codeCause("conformance.unimplemented"), "code");
+    assert.equal(codeCause("review.failed"), "review");
+    assert.equal(codeCause("weird"), "unknown");
+    assert.equal(codeCause(null), "unknown");
+  });
+  it("counts triggered attempts by cause and legacy attempts beyond plan + build as unknown", () => {
+    const s = issueSlices([start("t"), att("implement", { trigger: "plan" }), att("implement", { trigger: "build" }), att("implement", { trigger: "ci_red" }),
+      att("implement", { trigger: "context_restart" }), att("implement", { trigger: "ci_unavailable" }), att("review", { trigger: "review" }), att("implement", { trigger: "review" })])[0];
+    assert.deepEqual(sliceRetries(s), { env: 1, engine: 1, code: 1, review: 1, unknown: 0 });
+    const legacy = issueSlices([start("t"), att("implement"), att("implement"), att("implement"), att("implement"), att("review")])[0];
+    assert.deepEqual(sliceRetries(legacy), { env: 0, engine: 0, code: 0, review: 0, unknown: 2 });
+  });
+});
+
+describe("#4402 issue quality and acceptance", () => {
+  it("tags root cause + file:line + fix design", () => {
+    assert.equal(issueQuality("## Root Cause\nx at `bin/a.mjs:12`\n## Fix design\ny").tag, "full");
+    assert.equal(issueQuality("**Root cause**: z\n### Proposal\nfoo").tag, "partial");
+    assert.equal(issueQuality("just words, see a.js:3").tag, "partial");
+    assert.equal(issueQuality("plain text").tag, "none");
+    assert.equal(issueQuality("<!-- ## Root Cause a.js:1 ## Fix -->\nnothing").tag, "none");
+    assert.equal(issueQuality("<!-- ## Root Cause a.js:1\n## Fix design\nunclosed").tag, "none");
+    assert.equal(issueQuality(null).tag, "none");
+  });
+  it("is pending under 7 days, then pass only when every record within 7 days passes", () => {
+    const t0 = NOW - WEEK_MS;
+    assert.equal(acceptanceOf([], NOW), "unprobed");
+    assert.equal(acceptanceOf([probe("pass", t0 + 1)], NOW), "pending");
+    assert.equal(acceptanceOf([probe("pass", t0)], NOW), "pass", "exactly 7 days settles");
+    assert.equal(acceptanceOf([probe("pass", t0), probe("fail", t0 + WEEK_MS)], NOW + WEEK_MS), "fail");
+    assert.equal(acceptanceOf([probe("pass", t0), probe("fail", t0 + WEEK_MS + 1)], NOW + WEEK_MS), "pass", "a fail after the window does not count");
+    assert.equal(acceptanceOf([probe("fail", t0, false)], NOW), "unprobed", "untrusted records are ignored");
+  });
+});
+
+describe("#4402 efficiencyMetrics", () => {
+  const logs = new Map([
+    [1, [start("2026-10-01T00:00:00Z"), att("implement", { trigger: "plan" }), att("implement", { trigger: "build" }), term("needs-human", "env.tool_missing"),
+      start("2026-10-02T00:00:00Z"), att("implement", { trigger: "build", model: "m2", turns: 30, usage: { cache_read_input_tokens: 9000 } }), att("review", { cost_usd: 0.5 }),
+      term("merged", "completed", { turns: { size: "S" } }), start("2026-10-09T00:00:00Z"), att("implement", { cost_usd: 99 })]],
+    [2, [start("2026-10-02T00:00:00Z", "x/y"), att("implement", { cost_usd: 50 }), term("merged", "completed"),
+      start("2026-10-03T00:00:00Z"), att("implement", { trigger: "build" }), term("merged", "completed")]],
+  ]);
+  const entries = [
+    { row: { issue: 1, outcome: "merged" }, settleAt: "2026-10-05T00:00:00Z" },
+    { row: { issue: 2, outcome: "merged" }, settleAt: null },
+    { row: { issue: 3, outcome: "needs-human", code: "builder.blocked", detail: "Builder is blocked (promotion): on staging", cost_usd: 0.3 } },
+    { row: { issue: 4, outcome: "needs-human", code: "builder.blocked", detail: "Builder is blocked (credentials): x", cost_usd: 0.3 } },
+    { row: { issue: 5, outcome: "held", prefilter: { parked: true, kind: "awaiting-promotion" }, cost_usd: 0 } },
+  ];
+  const issues = new Map([[1, { body: "## Root cause\na.js:1\n## Fix", labels: [], comments: [probe("pass", NOW - 8 * DAY)] }],
+    [2, { body: "x", labels: ["size:L"], comments: [] }]]);
+  const m = efficiencyMetrics({ entries, logs, issues, repo: "o/r", now: NOW });
+
+  it("sums the cost to merge over earlier slices, bounded by the settle time and the repo", () => {
+    assert.equal(m.merged, 2);
+    assert.deepEqual(m.per_issue.map((p) => [p.issue, p.slices, p.cost_usd, p.implement_turns, p.size]), [[1, 2, 3.5, 50, "S"], [2, 1, 1, 10, "L"]]);
+    assert.equal(m.usd_per_merge, 2.25);
+    assert.equal(m.turns_per_merge, 30);
+    assert.equal(m.context_per_turn, 200);
+    assert.deepEqual(Object.keys(m.by_size), ["S", "L"]);
+    assert.deepEqual(m.by_model.m2, { merges: 1, attempts: 1, turns: 30, cost_usd: 1, usd_per_merge: 1, turns_per_merge: 30, context_per_turn: 300 });
+  });
+  it("counts retries by cause including lane reruns", () => {
+    assert.deepEqual(m.retries, { total: 1, per_merge: 0.5, by_cause: { env: 1, engine: 0, code: 0, review: 0, unknown: 0 }, lane_reruns: 1 });
+  });
+  it("lists no-action lanes: parked and promotion-blocked only", () => {
+    assert.equal(m.no_action.lanes, 2);
+    assert.deepEqual(Object.keys(m.no_action.by_kind).sort(), ["awaiting-promotion", "blocked-promotion"]);
+    assert.equal(m.no_action.cost_usd, 0.3);
+  });
+  it("compares quality tags and reports the acceptance rate", () => {
+    assert.equal(m.quality.full.n, 1);
+    assert.equal(m.quality.none.n, 1);
+    assert.equal(m.quality.partial.n, 0);
+    assert.deepEqual(m.acceptance, { pass: 1, fail: 0, pending: 0, unprobed: 1, probed: 1, pass_rate: 1 });
+  });
+  it("renders n/a, never NaN, for an empty cohort", () => {
+    const lines = formatEfficiencyLines(efficiencyMetrics({}));
+    assert.equal(lines.length, 9);
+    assert.ok(!lines.some((l) => /NaN|undefined/.test(l)), lines.join("\n"));
+    assert.match(lines[0], /^efficiency: merged 0, n\/a\/merge/);
+    assert.equal(lines[6], "acceptance (7d after promotion): n/a");
+    assert.equal(lines[7], "tools: calls/turn p50 n/a, re-reads (>=3x same path)/merge n/a");
+    assert.equal(lines[8], "lanes: light 0 ($0.00), upgraded 0 ($0.00), full 0 ($0.00); upgrade rate n/a");
+  });
+});
+
+describe("#4452 light/full split and upgrade rate", () => {
+  // issue 1: light, merged; 2: light then upgraded in the same run, merged; 3: full, merged; 4: light, failed; 5: upgraded, failed
+  const logs = new Map([
+    [1, [start("2026-10-22T10:00:00Z"), { event: "LIGHT_ELIGIBILITY", light: true }, att("implement", { trigger: "build", cost_usd: 0.9, cost_usd_delta: 0.4 }), { event: "LIGHT_DONE" }, att("review", { cost_usd: 0.05 }), term("merged", "merged")]],
+    [2, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.3 }), { event: "LIGHT_UPGRADE", reasons: [{ code: "diff_lines" }] }, att("implement", { trigger: "plan", cost_usd_delta: 0.5 }), term("merged", "merged")]],
+    [3, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "plan", cost_usd_delta: 0.6 }), att("implement", { trigger: "build", cost_usd: 1.2, cost_usd_delta: 0.4 }), term("merged", "merged")]],
+    [4, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.2 }), { event: "LIGHT_DONE" }, term("failed", "ci.red")]],
+    [5, [start("2026-10-22T10:00:00Z"), att("implement", { trigger: "build", cost_usd_delta: 0.1 }), { event: "LIGHT_UPGRADE" }, term("failed", "conformance.unimplemented")]],
+  ]);
+  const entries = [1, 2, 3, 4, 5].map((issue) => ({ row: { issue, outcome: issue <= 3 ? "merged" : "failed" }, settleAt: null }));
+  const m = efficiencyMetrics({ entries, logs, repo: "o/r" });
+
+  it("tags each lane slice by shape from LIGHT_DONE / LIGHT_UPGRADE", () => {
+    assert.deepEqual(issueSlices(logs.get(1)).map((s) => s.shape), ["light"]);
+    assert.deepEqual(issueSlices(logs.get(2)).map((s) => s.shape), ["upgraded"]);
+    assert.deepEqual(issueSlices(logs.get(3)).map((s) => s.shape), ["full"]);
+  });
+  it("splits every lane (all outcomes) by shape with true #4407 delta costs and computes the upgrade rate", () => {
+    assert.deepEqual(m.lanes, {
+      light: { n: 2, cost_usd: 0.65 }, // 0.4 delta (not the 0.9 running total) + 0.05 review + 0.2
+      upgraded: { n: 2, cost_usd: 0.9 },
+      full: { n: 1, cost_usd: 1 }, // 0.6 + 0.4 delta, not 1.2
+      upgrade_rate: 0.5,
+    });
+    assert.deepEqual(m.per_issue.map((x) => [x.issue, x.shape]), [[1, "light"], [2, "upgraded"], [3, "full"]]);
+  });
+  it("the batch report prints the split and the upgrade rate", () => {
+    const lines = formatEfficiencyLines(m);
+    assert.equal(lines.at(-1), "lanes: light 2 ($0.65), upgraded 2 ($0.90), full 1 ($1.00); upgrade rate 50% (2/4)");
+  });
+});
+
+describe("#4410 tool calls per turn and re-reads", () => {
+  const tools = (calls, turns, rereads) => ({ tools: { calls, turns, per_turn: calls / turns, rereads, top: [] } });
+  const logs = new Map([
+    [1, [start("2026-10-01T00:00:00Z"), att("implement", { trigger: "plan", ...tools(4, 4, 0) }), att("implement", { trigger: "build", ...tools(8, 4, 2) }),
+      att("review", tools(50, 1, 9)), term("merged", "completed")]],
+    [2, [start("2026-10-01T00:00:00Z"), att("implement", { trigger: "build", ...tools(30, 10, 1) }), term("merged", "completed")]],
+    [3, [start("2026-10-01T00:00:00Z"), att("implement", { trigger: "build", ...tools(5, 5, 0) }), term("merged", "completed")]],
+    [4, [start("2026-10-01T00:00:00Z"), att("implement", { trigger: "build" }), term("merged", "completed")]],
+  ]);
+  const entries = [1, 2, 3, 4].map((issue) => ({ row: { issue, outcome: "merged" }, settleAt: null }));
+  const m = efficiencyMetrics({ entries, logs, repo: "o/r", now: NOW });
+
+  it("p50 over lanes with tool data of implement calls / implement tool turns; re-reads per such lane", () => {
+    // lane 1: 12/8 = 1.5 (review excluded), lane 2: 3, lane 3: 1; lane 4 has no tool data
+    assert.equal(m.tool_calls_per_turn_p50, 1.5);
+    assert.equal(m.rereads_per_merge, 1);
+    const p = Object.fromEntries(m.per_issue.map((x) => [x.issue, x]));
+    assert.equal(p[1].implement_tool_calls_per_turn, 1.5); assert.equal(p[1].implement_rereads, 2);
+    assert.equal("implement_tool_calls_per_turn" in p[4], false);
+    assert.equal(formatEfficiencyLines(m).find((l) => l.startsWith("tools:")), "tools: calls/turn p50 1.50, re-reads (>=3x same path)/merge 1.00");
+  });
+  it("null without tool data (older logs)", () => {
+    const e = efficiencyMetrics({ entries: [entries[3]], logs, repo: "o/r", now: NOW });
+    assert.equal(e.tool_calls_per_turn_p50, null); assert.equal(e.rereads_per_merge, null);
+  });
+});
+
+describe("#4402 reproduces the 2026-10-22 AlterLab batches (#4389-#4394)", () => {
+  const window = async (gh = null) => (await windowEfficiency({ batchRoot: join(FX, "batches"), runsDir: join(FX, "runs"), repo: AL, gh, now: NOW, windows: [7 * DAY] }))["7d"];
+  it("implement $85.98 against review $3.13 over the 10 merged lanes, per-issue turns, cache-read and cost", async () => {
+    const m = await window();
+    assert.equal(m.batches, 2);
+    assert.equal(m.merged, 10);
+    assert.equal(m.implement_usd.toFixed(2), "85.98");
+    assert.equal(m.review_usd.toFixed(2), "3.13");
+    assert.equal(Math.round((m.implement_usd / (m.implement_usd + m.review_usd)) * 100), 96);
+    assert.equal(m.implement_turns, 802);
+    assert.equal(m.implement_cache_read, 92215514);
+    assert.equal(m.context_per_turn, 114982);
+    assert.equal(m.turns_per_merge, 80.2);
+    const p = Object.fromEntries(m.per_issue.map((x) => [x.issue, x]));
+    const mm = (x) => `${(x / 1e6).toFixed(1)}M`;
+    assert.deepEqual([p[36629].implement_turns, mm(p[36629].implement_cache_read), p[36629].implement_usd.toFixed(2)], [124, "22.6M", "19.80"]);
+    assert.deepEqual([p[36143].implement_turns, mm(p[36143].implement_cache_read), p[36143].implement_usd.toFixed(2), p[36143].slices], [165, "14.2M", "15.03", 2]);
+    assert.equal(p[36143].retries.code, 1);
+    assert.deepEqual([p[36627].implement_turns, mm(p[36627].implement_cache_read), p[36627].implement_usd.toFixed(2)], [102, "15.8M", "9.20"]);
+    assert.deepEqual([p[36620].implement_turns, mm(p[36620].implement_cache_read)], [91, "12.2M"]);
+    assert.deepEqual([p[36612].implement_turns, p[36612].implement_usd.toFixed(2)], [19, "1.23"]);
+    assert.deepEqual([p[36619].slices, p[36619].retries.env, p[36619].retries.engine], [4, 1, 2]);
+  });
+  it("batch 183421: 32 promotion-blocked no-action lanes for $10.13 and the #4389 outcome table", async () => {
+    const b = await batchEfficiency({ batchDir: join(FX, "batches", B2), runsDir: join(FX, "runs"), now: NOW });
+    assert.equal(b.batch, B2);
+    assert.deepEqual([b.no_action.lanes, b.no_action.cost_usd.toFixed(2), Object.keys(b.no_action.by_kind)], [32, "10.13", ["blocked-promotion"]]);
+    const { readFileSync } = await import("node:fs");
+    const rows = JSON.parse(readFileSync(join(FX, "batches", B2, "report.json"), "utf8")).rows;
+    const by = (o) => { const xs = rows.filter((r) => r.outcome === o); return [xs.length, xs.reduce((a, r) => a + r.cost_usd, 0).toFixed(2)]; };
+    assert.deepEqual([by("merged"), by("dependency-blocked"), by("invalid")], [[7, "52.97"], [3, "1.27"], [2, "0.82"]]);
+    const b1 = await batchEfficiency({ batchDir: join(FX, "batches", B1), runsDir: join(FX, "runs"), now: NOW });
+    assert.equal(b1.merged + b.merged, 10);
+  });
+  it("a gh failure leaves quality and acceptance n/a with a note, never throws", async () => {
+    const m = await window(async () => { throw new Error("HTTP 502"); });
+    assert.equal(m.quality, null);
+    assert.equal(m.acceptance, null);
+    assert.match(formatEfficiencyLines(m).join("\n"), /issue quality: n\/a \(gh failed \(HTTP 502\)\)/);
+  });
+  it("windows exclude batches outside the window and other repos", async () => {
+    const out = await windowEfficiency({ batchRoot: join(FX, "batches"), runsDir: join(FX, "runs"), repo: AL, now: NOW + 30 * DAY, windows: [7 * DAY, 60 * DAY] });
+    assert.deepEqual([out["7d"].batches, out["7d"].merged, out["60d"].batches, out["60d"].merged], [0, 0, 2, 10]);
+    const other = await windowEfficiency({ batchRoot: join(FX, "batches"), runsDir: join(FX, "runs"), repo: "o/r", now: NOW, windows: [7 * DAY] });
+    assert.equal(other["7d"].batches, 0);
+  });
+  it("an issue in two batches counts once, with its latest row", async () => {
+    const root = tmpDir("eff-root-");
+    for (const [id, created, outcome] of [["b1", "2026-10-20T00:00:00Z", "deferred"], ["b2", "2026-10-21T00:00:00Z", "merged"]]) {
+      mkdirSync(join(root, id), { recursive: true });
+      writeFileSync(join(root, id, "plan.json"), JSON.stringify({ v: 1, id, repo: "o/r", created }));
+      writeFileSync(join(root, id, "report.json"), JSON.stringify({ rows: [{ issue: 7, outcome, cost_usd: 1 }] }));
+    }
+    const m = (await windowEfficiency({ batchRoot: root, runsDir: tmpDir("eff-runs-"), repo: "o/r", now: NOW, windows: [7 * DAY] }))["7d"];
+    assert.deepEqual([m.batches, m.merged], [2, 1]);
+  });
+});
+
+describe("#4402 forgedock stats --efficiency", () => {
+  it("parses the flags and rejects --batch/--window without --efficiency", () => {
+    const o = parseArgs(["--efficiency", "--batch", B2, "--window", "7d,30d"]);
+    assert.deepEqual([o.efficiency, o.batch, o.windows], [true, B2, [7 * DAY, 30 * DAY]]);
+    assert.throws(() => parseArgs(["--batch", B2]), /need --efficiency/);
+    assert.throws(() => parseArgs(["--efficiency", "--batch", "../x"]), /invalid --batch/);
+    assert.throws(() => parseWindows("7w"), /--window/);
+  });
+  it("prints the batch and each window", async () => {
+    const out = [], w = process.stdout.write;
+    process.stdout.write = (t) => { out.push(String(t)); return true; };
+    let code;
+    try {
+      code = await runStats(["--efficiency", "--batch", B2, "--window", "7d", "--repo", AL],
+        { batchRoot: join(FX, "batches"), runsDir: join(FX, "runs"), now: () => NOW, gh: async () => { throw new Error("offline"); } });
+    } finally { process.stdout.write = w; }
+    assert.equal(code, 0);
+    const text = out.join("");
+    assert.match(text, new RegExp(`^efficiency \\(batch ${B2}\\): merged 7,`, "m"));
+    assert.match(text, /^efficiency \(7d, 2 batches\): merged 10, \$8\.91\/merge \(implement \$85\.98, review \$3\.13\), turns\/merge 80\.2, context\/turn 114982$/m);
+    assert.match(text, /^no action: 32 lanes, \$10\.13 \(blocked-promotion 32\)$/m);
+  });
+});

package/bin/stats/github-source.mjs

@@ -77,7 +77,7 @@ export function runsFromRecords(records) {
     const key = `${r.repo}#${r.issue}#${r.run ?? ""}`;
     const run = byRun.get(key) ?? { source: "github", repo: r.repo ?? null, version: r.forge_version ?? null, issue: r.issue, run: r.run ?? null,
       finished: false, hydrated: false, usageEvents: 0, reason: null, code: null, remediated: false, startTs: null, endTs: null, wallMs: null,
-      phases: {}, _first: null };
+      phases: {}, builderAttempts: [], _first: null };
     byRun.set(key, run);
     if (!run._first || r.ts < run._first) run._first = r.ts;
     if (r.event === "run_terminal") {
@@ -92,8 +92,10 @@ export function runsFromRecords(records) {
     p.out += num(r.output_tokens) ?? 0;
     p.attempts++;
     if (num(r.duration_ms) != null) p.ms = (p.ms ?? 0) + num(r.duration_ms);
-    if (num(r.cost_usd) != null) p.usd = (p.usd ?? 0) + num(r.cost_usd);
+    const usd = num(r.cost_usd_delta) ?? num(r.cost_usd); // #4407: cost_usd is a running total for a resumed session
+    if (usd != null) p.usd = (p.usd ?? 0) + usd;
     if (id === "remediate") run.remediated = true;
+    if (id === "implement") run.builderAttempts.push({ attempt: num(r.attempt), trigger: r.trigger ?? null, restart_of: r.restart_of ?? null, cost_usd: usd, _ts: r.ts ?? "" }); // #4362
   }
   const latest = new Map();
   for (const run of byRun.values()) {
@@ -102,7 +104,10 @@ export function runsFromRecords(records) {
     const a = run._first ?? "", b = cur?._first ?? "";
     if (!cur || a > b || (a === b && (run.run ?? "") > (cur.run ?? ""))) latest.set(k, run);
   }
-  return [...latest.values()].map(({ _first, ...run }) => ({ ...run, startTs: _first }));
+  // #4362: attempt order (a timeout restart reuses its number: then record time); stable for equal keys
+  const order = (a, b) => (a.attempt ?? 0) - (b.attempt ?? 0) || (a._ts < b._ts ? -1 : a._ts > b._ts ? 1 : 0);
+  return [...latest.values()].map(({ _first, ...run }) => ({ ...run, startTs: _first,
+    builderAttempts: [...run.builderAttempts].sort(order).map(({ _ts, ...a }) => a) }));
 }
 
 /** Gate/review records (everything that is not a run record) become aggregate events. */

package/bin/stats/index.mjs

@@ -9,16 +9,20 @@ import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import { aggregate, DEFAULT_MIN_N } from "./aggregate.mjs";
-import { fetchGithubRecords, isValidRepo, toModel } from "./github-source.mjs";
-import { readLocalRuns, runsDir } from "./local-source.mjs";
+import { aggregate, computeAlerts, DEFAULT_ALERTS, DEFAULT_MIN_N } from "./aggregate.mjs";
+import { defaultGh, fetchGithubRecords, isValidRepo, toModel } from "./github-source.mjs";
+import { readLanes, readLocalLaneRuns, readLocalRuns, runsDir } from "./local-source.mjs";
+import { parseForgeYaml } from "../forge-utils.mjs";
 import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
-import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb } from "./query.mjs";
+import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
+import { batchEfficiency, windowEfficiency, formatEfficiencyLines, parseWindows } from "./efficiency.mjs";
+import { batchRoot } from "../lean/batch.mjs";
 
 const here = dirname(fileURLToPath(import.meta.url));
 
 export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
                       [--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
+       forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--repo <owner/repo>] [--json]
 
 Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
 delta against the previous version and a low-n flag for samples under 10. Missing data prints n/a.
@@ -36,6 +40,10 @@ delta against the previous version and a low-n flag for samples under 10. Missin
   --release-times FILE  "<version> <epoch-seconds>" lines used to estimate versions of unversioned records
                         (default: git tags v* of the ForgeDock checkout)
 
+ALERTS (forge#4354; local run logs only, skipped with --no-local): fail-soft rates over the last 50 lanes of the repo
+(DEPS_PREPARED failed per ecosystem, deferred-to-ci, repeated TOOLCHAIN_WARNING, outcome codes whose share doubled);
+thresholds under forge.yaml stats.alerts.*. Nothing is printed when none fired or under 10 lanes.
+
 Audit queries (forge#4165; JSON, from structure_delta / run_terminal telemetry, no comment bodies read):
   --edge FROM:TO        merged PRs that added the dependency or import edge FROM -> TO (e.g. @s/web:@s/db)
   --milestone M         conformance verdict mix and per-day structure series for milestone M
@@ -43,10 +51,29 @@ Audit queries (forge#4165; JSON, from structure_delta / run_terminal telemetry,
 Context-pack A/B (forge#4275; local run logs only, no GitHub crawl):
   --context-ab          first-pass yield, builder turns, cache tokens and cost per lane type (rerun, child,
                         first) and context_pack mode (on, off, shadow); --json for JSON
+
+Explorer A/B (forge#4418; local run logs only, every run of each log, no GitHub crawl):
+  --explore-ab          builder turns, builder cache-read, merge and conformance rates per size (S, M, L) and
+                        lean.explore mode (on, off), plus the same issues run in both modes; --json for JSON
+
+Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues' bodies, labels and probes):
+  --efficiency          $/merge, turns/merge, context/turn per size and model, retries/merge by cause, no-action
+                        lanes, issue quality vs cost and the acceptance pass rate 7 days after promotion
+  --batch <id>          also report one batch (~/.forge/batches/<id>, FORGEDOCK_BATCH_DIR)
+  --window <list>       trailing windows over the repo's batches (default 7d,30d)
+
+Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
+  --tiers               merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
+                        (mid, mid->top escalated, top) and routing class; --json for JSON
+
+Lane profile baseline (forge#4436; local run logs only, every run of each issue, no GitHub crawl):
+  --lane-profile        first-turn baseline_tokens per lane profile (full, memory-off, lane) and phase, and the
+                        conformance rate per profile; --json for JSON
+  --issues <list>       pin the report to these issues (comma list), for a before/after on the same issue set
 `;
 
 export function parseArgs(argv) {
-  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
+  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, batch: null, windows: null, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
   const val = (i, name) => {
     const v = argv[i + 1];
     if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
@@ -63,6 +90,13 @@ export function parseArgs(argv) {
     else if (name === "--edge") { const v = take(); if (!parseEdge(v)) throw new Error("--edge must be FROM:TO"); o.edge = v; }
     else if (name === "--milestone") o.milestone = take();
     else if (name === "--context-ab") o.contextAb = true;
+    else if (name === "--explore-ab") o.exploreAb = true;
+    else if (name === "--tiers") o.tiers = true;
+    else if (name === "--lane-profile") o.laneProfile = true;
+    else if (name === "--issues") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--issues must be a comma list of issue numbers"); o.issues = v.split(",").map(Number); }
+    else if (name === "--efficiency") o.efficiency = true;
+    else if (name === "--batch") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error(`invalid --batch id: ${v}`); o.batch = v; }
+    else if (name === "--window") o.windows = parseWindows(take());
     else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
     else if (name === "--json") o.json = true;
     else if (name === "--refresh") o.refresh = true;
@@ -72,6 +106,7 @@ export function parseArgs(argv) {
     else if (name === "--help" || name === "-h") o.help = true;
     else throw new Error(`unknown argument: ${a}`);
   }
+  if ((o.batch || o.windows) && !o.efficiency) throw new Error("--batch and --window need --efficiency");
   return o;
 }
 
@@ -153,6 +188,25 @@ export function resolveRepo(explicit, cwd = process.cwd()) {
   return null;
 }
 
+const ALERT_KEYS = { window: "window", min_lanes: "minLanes", deps_failed_pct: "depsFailedPct", deferred_to_ci_pct: "deferredPct",
+  toolchain_warning_pct: "toolchainWarningPct", outcome_share_factor: "outcomeShareFactor" };
+
+/**
+ * forge.yaml `stats.alerts.*` (forge#4354). The one-level parser flattens a nested `alerts:` block under `stats:` to
+ * `stats.<key>`; the flat `stats.alerts_<key>` form also works and wins. Invalid or unreadable values keep DEFAULT_ALERTS.
+ */
+export function readAlertConfig(cwd = process.cwd()) {
+  const out = { ...DEFAULT_ALERTS };
+  try {
+    const s = parseForgeYaml(readFileSync(join(cwd, "forge.yaml"), "utf8"))?.stats ?? {};
+    for (const [key, name] of Object.entries(ALERT_KEYS)) {
+      const raw = String(s[`alerts_${key}`] ?? s[key] ?? "").trim(), n = Number(raw);
+      if (raw !== "" && Number.isFinite(n) && n > 0) out[name] = n;
+    }
+  } catch { /* no forge.yaml: defaults */ }
+  return out;
+}
+
 /** Build the report object. deps: { gh, env, localRuns, now, onProgress, warn, sleep } for tests. Never touches the network when `gh` is stubbed. */
 export async function buildReport(opts, deps = {}) {
   const env = deps.env ?? process.env;
@@ -161,10 +215,29 @@ export async function buildReport(opts, deps = {}) {
   if (opts.releaseTimes) releaseTimes = parseReleaseTimes(readFileSync(opts.releaseTimes, "utf8"));
   else if (deps.releaseTimes) releaseTimes = deps.releaseTimes;
   else releaseTimes = gitTagTimes();
+  if (opts.efficiency) { // forge#4402: local batch dirs + run logs; gh only for merged issues
+    const root = deps.batchRoot ?? batchRoot(env), rd = deps.runsDir ?? runsDir(env), now = deps.now?.() ?? Date.now(), gh = deps.gh ?? defaultGh;
+    const efficiency = {};
+    if (opts.batch) efficiency.batch = await batchEfficiency({ batchDir: join(root, opts.batch), runsDir: rd, repo, gh, now });
+    efficiency.windows = await windowEfficiency({ batchRoot: root, runsDir: rd, repo, gh, now, windows: opts.windows ?? parseWindows("7d,30d") });
+    return { repo, query: { efficiency } };
+  }
   if (opts.contextAb) { // forge#4275: run logs only
     const runs = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
     return { repo, sources: { local: { runs: runs.length } }, query: { context_ab: contextPackAb(runs) } };
   }
+  if (opts.exploreAb) { // forge#4418: run logs only, every RUN_START slice
+    const runs = (deps.localRuns ?? readLocalLaneRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
+    return { repo, sources: { local: { runs: runs.length } }, query: { explore_ab: exploreAb(runs) } };
+  }
+  if (opts.tiers) { // forge#4394: run logs only
+    const runs = (deps.localRuns ?? readLocalRuns(runsDir(env))).filter((r) => r.repo == null || r.repo === repo);
+    return { repo, sources: { local: { runs: runs.length } }, query: { tiers: builderTierReport(runs) } };
+  }
+  if (opts.laneProfile) { // forge#4436: run logs only, every RUN_START slice
+    const runs = (deps.localRuns ?? readLocalRuns(runsDir(env), { allSlices: true })).filter((r) => r.repo == null || r.repo === repo);
+    return { repo, sources: { local: { runs: runs.length } }, query: { lane_profile: laneProfileReport(runs, { issues: opts.issues }) } };
+  }
   const bound = sinceToTimestamp(opts.since, releaseTimes);
   if (bound.note) (deps.warn ?? ((m) => process.stderr.write(`forgedock stats: ${m}\n`)))(bound.note);
   const gh = await fetchGithubRecords({ repo, since: bound.since, refresh: opts.refresh, useCache: opts.cache, gh: deps.gh, env,
@@ -182,7 +255,9 @@ export async function buildReport(opts, deps = {}) {
   }
   const report = aggregate({ runs: [...model.runs, ...local], events: model.events, issues: model.issues },
     { minN: opts.minN, releaseTimes, ...parseSince(opts.since) });
-  return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report };
+  // forge#4354: fail-soft alerts from the local run logs only (no gh call)
+  const alerts = opts.local ? computeAlerts(deps.localLanes ?? readLanes(runsDir(env)), { repo, ...(deps.alertConfig ?? readAlertConfig()) }) : [];
+  return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report, alerts };
 }
 
 export async function runStats(argv, deps = {}) {
@@ -199,7 +274,17 @@ export async function runStats(argv, deps = {}) {
   }
   try {
     const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
+    if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
+    if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
+    if (report.query?.efficiency && !opts.json) {
+      const e = report.query.efficiency;
+      const lines = [...(e.batch ? formatEfficiencyLines(e.batch, `batch ${e.batch.batch ?? opts.batch}`) : []),
+        ...Object.entries(e.windows).flatMap(([w, m]) => formatEfficiencyLines(m, `${w}, ${m.batches} batch${m.batches === 1 ? "" : "es"}`))];
+      process.stdout.write(`${lines.join("\n")}\n`);
+      return 0;
+    }
     if (report.query?.context_ab && !opts.json) { process.stdout.write(`${renderContextAb(report.query.context_ab)}\n`); return 0; }
+    if (report.query?.explore_ab && !opts.json) { process.stdout.write(`${renderExploreAb(report.query.explore_ab)}\n`); return 0; }
     process.stdout.write(`${opts.json || report.query ? renderJson(report) : opts.compact ? renderCompact(report) : renderMarkdown(report)}\n`);
     return 0;
   } catch (err) {