forgedock 1.30.0 → 1.32.0

package/bin/stats/stats.test.mjs

@@ -8,13 +8,13 @@ import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawnSync } from "node:child_process";
-import { aggregate, compareVersions, percentile } from "./aggregate.mjs";
+import { aggregate, compareVersions, computeAlerts, DEFAULT_ALERTS, percentile } from "./aggregate.mjs";
 import { cacheDir } from "./cache.mjs";
 import { fetchGithubRecords, parseTelemetryComments, runsFromRecords, toModel } from "./github-source.mjs";
-import { buildReport, makeProgressPrinter, parseArgs, parseSince, runStats, sinceToTimestamp } from "./index.mjs";
-import { readLocalRuns, summariseLog } from "./local-source.mjs";
-import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
-import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb } from "./query.mjs";
+import { buildReport, makeProgressPrinter, parseArgs, parseSince, readAlertConfig, runStats, sinceToTimestamp } from "./index.mjs";
+import { lanesOfLog, readLanes, readLocalLaneRuns, readLocalRuns, summariseLog } from "./local-source.mjs";
+import { renderAlerts, renderCompact, renderJson, renderMarkdown } from "./render.mjs";
+import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
 
 const here = dirname(fileURLToPath(import.meta.url));
 const fx = (f) => join(here, "fixtures", f);
@@ -538,21 +538,23 @@ describe("context-pack A/B over run logs (forge#4275)", () => {
 
   it("summariseLog carries mode, lane type, builder turns, cache tokens, cost and first-pass", () => {
     const s = summariseLog(log("on", "child"));
-    assert.deepEqual(s.contextPack, { mode: "on", laneType: "child" });
+    assert.deepEqual(s.contextPack, { mode: "on", laneType: "child", ab: false });
     assert.deepEqual([s.builderTurns, s.cacheTokens, s.costUsd, s.firstPass], [10, 2130, 1.1, true]);
     assert.equal(summariseLog(log("on", "child", { builds: 4 })).firstPass, false);
     assert.equal(summariseLog(log("on", "child", { reason: "needs-human" })).firstPass, false);
     const fixed = log("on", "child"); fixed.splice(-1, 0, { event: "CI_FIX_ROUND", trigger: "build", round: 1 });
     assert.equal(summariseLog(fixed).firstPass, false);
     assert.equal(summariseLog([{ event: "RUN_START", issue: 3 }, { event: "RUN_TERMINAL", reason: "merged" }]).contextPack.mode, "off");
+    assert.equal(summariseLog([{ event: "RUN_START", issue: 3 }, { event: "RUN_TERMINAL", reason: "merged" }]).size, null);
+    assert.equal(summariseLog([{ event: "RUN_START", issue: 3 }, { event: "RUN_TERMINAL", reason: "merged", turns: { size: "L", budget: 100, build: 9 } }]).size, "L"); // #4409
   });
 
   it("reports per lane type and mode, never one blended average", () => {
     const runs = [log("on", "first"), log("on", "first", { builds: 4, reviews: 2 }), log("off", "first"), log("shadow", "rerun"),
       [{ event: "RUN_START", issue: 2 }, { event: "RUN_TERMINAL", reason: "merged" }]].map((l) => summariseLog(l));
     const ab = contextPackAb(runs);
-    assert.deepEqual(Object.keys(ab), ["rerun", "child", "first"]);
-    assert.deepEqual(ab.first.on, { n: 2, first_pass_yield: 0.5, builder_turns: 15, cache_tokens: 3195, cost_usd: 1.65 });
+    assert.deepEqual(Object.keys(ab), ["rerun", "child", "first", "recovery"]);
+    assert.deepEqual(ab.first.on, { n: 2, first_pass_yield: 0.5, builder_turns: 15, cache_tokens: 3195, cost_usd: 1.65, merged: 2, turns_per_merge: 15, cache_read_per_merge: 3150, rework_rate: 0 });
     assert.equal(ab.first.off.n, 1); assert.equal(ab.first.off.first_pass_yield, 1);
     assert.equal(ab.rerun.shadow.n, 1); assert.equal(ab.child.on.n, 0); assert.equal(ab.child.on.cost_usd, null);
     assert.match(renderContextAb(ab), /\| first \| on \| 2 \| 0\.5 \| 15 \| 3195 \| 1\.65 \|/);
@@ -564,3 +566,396 @@ describe("context-pack A/B over run logs (forge#4275)", () => {
     assert.equal(rep.query.context_ab.first.on.n, 1);
   });
 });
+
+describe("explorer A/B over run logs (forge#4418)", () => {
+  const use = (phase, turns, cr) => ({ event: "PHASE_ATTEMPT_USAGE", phase, turns, cost_usd: 0.5, usage: { cache_read_input_tokens: cr, cache_creation_input_tokens: 10, output_tokens: 1 } });
+  const slice = (issue, explore, { size = "M", turns = 5, cr = 1000, reason = "merged", triggers = ["plan", "build"], review = true } = {}) => [
+    { event: "RUN_START", issue, repo: "o/r", run: `r${issue}${explore}` },
+    ...triggers.map((t) => ({ ...use("implement", turns, cr), trigger: t })), ...(review ? [use("review", 2, 100)] : []),
+    { event: "RUN_TERMINAL", reason, code: "x", ...(explore ? { explore } : {}), ...(size ? { turns: { size, budget: 100, build: 9 } } : {}) },
+  ];
+
+  it("summariseLog carries the explore tag and the plan + build cache-read", () => {
+    const on = summariseLog(slice(1, "on"));
+    assert.deepEqual([on.explore, on.builderCacheRead, on.builderTurns, on.cacheTokens], ["on", 2000, 10, 2130]);
+    assert.equal(summariseLog(slice(1, "off")).explore, "off");
+    assert.equal(summariseLog(slice(1, null)).explore, null);
+    assert.equal(summariseLog(slice(1, "maybe")).explore, null);
+  });
+
+  it("readLocalLaneRuns keeps every RUN_START slice of a log", () => {
+    const dir = tmpDir("explore-ab-");
+    try {
+      writeFileSync(join(dir, "o__r__7.jsonl"), [...slice(7, "on"), ...slice(7, "off", { reason: "needs-human" })].map((e) => JSON.stringify(e)).join("\n"));
+      const runs = readLocalLaneRuns(dir);
+      assert.deepEqual(runs.map((r) => [r.issue, r.explore, r.reason]), [[7, "on", "merged"], [7, "off", "needs-human"]]);
+      assert.equal(readLocalRuns(dir).length, 1);
+      assert.deepEqual(readLocalLaneRuns(join(dir, "missing")), []);
+    } finally { rmSync(dir, { recursive: true, force: true }); }
+  });
+
+  it("reports per size and mode, pairs same issues and leaves untagged runs out", () => {
+    const runs = [slice(1, "on", { size: "L", turns: 3, cr: 500 }), slice(1, "off", { size: "L", turns: 10, cr: 4000 }),
+      slice(2, "on", { size: "L", turns: 4, cr: 600, triggers: ["plan", "build", "conformance"] }), slice(3, "off", { size: "S", reason: "needs-human", review: false }),
+      slice(4, null), slice(5, "on", { size: null })].map((l) => summariseLog(l));
+    const ab = exploreAb(runs);
+    assert.deepEqual(Object.keys(ab.by_size), ["S", "M", "L", "unsized"]);
+    assert.deepEqual(ab.by_size.L.on, { n: 2, builder_turns: 9, builder_cache_read: 1400, merge_rate: 1, conformance_rate: 0.5, reviewed: 2 });
+    assert.deepEqual(ab.by_size.L.off, { n: 1, builder_turns: 20, builder_cache_read: 8000, merge_rate: 1, conformance_rate: 1, reviewed: 1 });
+    assert.deepEqual([ab.by_size.S.off.merge_rate, ab.by_size.S.off.conformance_rate, ab.by_size.M.on.n, ab.by_size.M.on.builder_turns], [0, null, 0, null]);
+    assert.equal(ab.by_size.unsized.on.n, 1);
+    assert.equal(ab.untagged, 1);
+    assert.equal(ab.paired.issues, 1);
+    assert.deepEqual([ab.paired.on.n, ab.paired.on.builder_cache_read, ab.paired.off.n, ab.paired.off.builder_cache_read], [1, 1000, 1, 8000]);
+    const md = renderExploreAb(ab);
+    assert.match(md, /\| L \| on \| 2 \| 9 \| 1400 \| 1 \| 0\.5 \(2\) \|/);
+    assert.match(md, /\| paired \| off \| 1 \| 20 \| 8000 \| 1 \| 1 \(1\) \|/);
+    assert.match(md, /Same issues in both modes: 1/);
+    assert.match(md, /1 finished run\(s\) without an explore tag/);
+  });
+
+  it("--explore-ab reads run logs only (no GitHub crawl)", async () => {
+    assert.equal(parseArgs(["--explore-ab"]).exploreAb, true);
+    assert.equal(parseArgs([]).exploreAb, false);
+    const rep = await buildReport({ repo: "o/r", exploreAb: true }, { localRuns: [summariseLog(slice(1, "on")), { ...summariseLog(slice(2, "on")), repo: "x/y" }], releaseTimes: [], gh: async () => { throw new Error("no network"); } });
+    assert.equal(rep.query.explore_ab.by_size.M.on.n, 1);
+    assert.equal(rep.sources.local.runs, 1);
+  });
+});
+
+describe("context-pack A/B per merge, rework and recovery vs first (#4444)", () => {
+  const use = (trigger, turns, cr, out) => ({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", trigger, turns, cost_usd: 0.5, usage: { cache_read_input_tokens: cr, cache_creation_input_tokens: 10, output_tokens: out } });
+  const slice = (issue, lt, { reason = "merged", mode = "on", ab = false, calls = [["plan", 2, 100, 10], ["build", 10, 1000, 400]], extra = [] } = {}) => [
+    { event: "RUN_START", issue, repo: "o/r", run: `r${issue}${lt}`, ts: lt === "rerun" ? "2026-10-02T00:00:00Z" : "2026-10-01T00:00:00Z" },
+    ...calls.map((c) => use(...c)), ...extra,
+    { event: "RUN_TERMINAL", reason, code: "x", context_pack: mode, lane_type: lt, ...(ab ? { context_ab: true } : {}) },
+  ];
+
+  it("summariseLog: cache-read alone, the implement attempt without the plan call, the ab tag and the log's first-run baseline", () => {
+    const s = summariseLog([...slice(5, "first"), ...slice(5, "rerun", { ab: true, calls: [["recover", 3, 50, 100]] })]);
+    assert.equal(s.cacheReadTokens, 50);
+    assert.deepEqual(s.implementAttempt, { turns: 3, output_tokens: 100 });
+    assert.deepEqual(s.firstRunAttempt, { turns: 10, output_tokens: 400 });
+    assert.equal(s.contextPack.ab, true);
+    assert.equal(summariseLog(slice(5, "first")).firstRunAttempt, null);
+    assert.deepEqual(summariseLog(slice(6, "first", { calls: [] })).implementAttempt, null);
+  });
+
+  it("per-merge turns and cache-read, rework rate and the same-issue recovery ratio", () => {
+    const runs = [summariseLog(slice(1, "first")), summariseLog(slice(2, "first", { reason: "needs-human" })),
+      summariseLog(slice(3, "first", { extra: [{ event: "PHASE_COMMIT", phase: "remediate" }] })),
+      summariseLog([...slice(4, "first"), ...slice(4, "rerun", { calls: [["recover", 5, 0, 100]] })]),
+      summariseLog(slice(1, "rerun", { calls: [["recover", 2, 0, 200]] }))];
+    const ab = contextPackAb(runs);
+    // three runs: 12 builder turns each, 1100 cache-read each, two merged
+    assert.deepEqual([ab.first.on.merged, ab.first.on.turns_per_merge, ab.first.on.cache_read_per_merge, ab.first.on.rework_rate], [2, 18, 1650, 0.3333]);
+    assert.equal(ab.rerun.on.rework_rate, 1);
+    assert.equal(ab.child.on.turns_per_merge, null);
+    // issue 4: 5/10 turns, 100/400 output (own log); issue 1: 2/10 turns, 200/400 output (earlier first run in `runs`)
+    assert.deepEqual(ab.recovery, { pairs: 2, mean_turn_ratio: 0.35, mean_output_ratio: 0.375 });
+    const md = renderContextAb(ab);
+    assert.match(md, /\| Merged \| Turns\/merge \| Cache-read\/merge \| Rework rate \|/);
+    assert.match(md, /\| first \| on \| 3 \| .* \| 2 \| 18 \| 1650 \| 0\.3333 \|/);
+    assert.match(md, /Recovery vs first run \(same issue\): 2 pair\(s\), implement turns ratio 0\.35, output tokens ratio 0\.375/);
+    assert.deepEqual(contextPackAb([]).recovery, { pairs: 0, mean_turn_ratio: null, mean_output_ratio: null });
+  });
+});
+
+describe("builder model tiers over run logs (forge#4394)", () => {
+  const use = (phase, trigger, usd = 1) => ({ event: "PHASE_ATTEMPT_USAGE", phase, trigger, cost_usd: usd, usage: { cache_read_input_tokens: 1, output_tokens: 1 } });
+  const slice = (issue, { tier = "mid", cls = "small", escalated = false, reason = "merged", triggers = ["plan", "build"], review = true, shadow } = {}) => [
+    { event: "RUN_START", issue, repo: "o/r", run: `r${issue}` },
+    ...triggers.map((t) => use("implement", t)), ...(review ? [use("review", "review", 0.5)] : []),
+    { event: "RUN_TERMINAL", reason, code: "x", ...(shadow ? { shadow_would: shadow } : {}), builder_tier: tier, builder_class: cls, builder_escalated: escalated, builder_model: tier === "mid" ? "claude-sonnet-5-5" : "default" },
+  ];
+
+  it("summariseLog reads builderRoute from RUN_TERMINAL, else the BUILDER_ROUTE/BUILDER_ESCALATED events; null before routing", () => {
+    const s = summariseLog(slice(1, { escalated: true, triggers: ["plan", "build", "conformance"], shadow: "conformance.scope" }));
+    assert.deepEqual(s.builderRoute, { tier: "mid", class: "small", escalated: true, model: "claude-sonnet-5-5" });
+    assert.equal(s.conformanceRound, true); assert.equal(s.shadowWould, "conformance.scope"); assert.equal(s.reachedReview, true);
+    const evs = [{ event: "RUN_START", issue: 2 }, { event: "BUILDER_ROUTE", tier: "mid", class: "docs-config-test", model: "m" }, { event: "BUILDER_ESCALATED", from: "m", to: "default", trigger: "ci_red" }, { event: "RUN_TERMINAL", reason: "merged" }];
+    assert.deepEqual(summariseLog(evs).builderRoute, { tier: "mid", class: "docs-config-test", escalated: true, model: "m" });
+    const old = summariseLog([{ event: "RUN_START", issue: 3 }, { event: "RUN_TERMINAL", reason: "merged" }]);
+    assert.equal(old.builderRoute, null); assert.equal(old.conformanceRound, false); assert.equal(old.shadowWould, null);
+  });
+
+  it("reports merge, conformance and post-merge-rework rates per tier and class; unrouted runs are left out", () => {
+    const reworked = [...slice(5, { tier: "top", cls: "large" }), ...slice(5, { tier: "top", cls: "large", reason: "needs-human" })]; // a later run of a merged issue
+    const runs = [slice(1), slice(2, { triggers: ["plan", "build", "review"] }), slice(3, { reason: "needs-human", review: false }),
+      slice(4, { escalated: true, triggers: ["plan", "build", "precheck"] }), reworked, [{ event: "RUN_START", issue: 9 }, { event: "RUN_TERMINAL", reason: "merged" }]].map((l) => summariseLog(l));
+    const rep = builderTierReport(runs);
+    assert.equal(rep.unrouted, 1);
+    assert.deepEqual(rep.rows.map((g) => [g.tier, g.class]), [["mid", "small"], ["mid->top", "small"], ["top", "large"]]);
+    const [mid, esc, top] = rep.rows;
+    assert.deepEqual([mid.n, mid.merge_rate, mid.reviewed, mid.conformance_rate, mid.merges, mid.post_merge_rework_rate, mid.first_pass_yield], [3, 0.6667, 2, 0.5, 2, 0, 0.3333]);
+    assert.deepEqual([esc.n, esc.merge_rate, esc.conformance_rate], [1, 1, 1]);
+    assert.deepEqual([top.n, top.merge_rate, top.merges, top.post_merge_rework_rate], [1, 0, 1, 1]);
+    const md = renderTiers(rep);
+    assert.match(md, /\| mid \| small \| 3 \| 0\.6667 \| 0\.5 \(2\) \| 0 \(2\) \| 0\.3333 \|/);
+    assert.match(md, /\| top \| large \| 1 \| 0 \| .* \| 1 \(1\) \|/);
+    assert.match(md, /1 finished run\(s\) without a builder route/);
+    assert.match(renderTiers(builderTierReport([])), /\| none \|/);
+  });
+
+  it("--tiers reads run logs only (no GitHub crawl); other reports carry no tiers", async () => {
+    assert.equal(parseArgs(["--tiers"]).tiers, true); assert.equal(parseArgs([]).tiers, false);
+    const rep = await buildReport({ repo: "o/r", tiers: true }, { localRuns: [summariseLog(slice(1))], releaseTimes: [], gh: async () => { throw new Error("no network"); } });
+    assert.equal(rep.query.tiers.rows[0].n, 1);
+  });
+});
+
+describe("lane profile report (#4436)", () => {
+  const use = (phase, profile, baseline, trigger = "build") => ({ event: "PHASE_ATTEMPT_USAGE", phase, trigger, cost_usd: 1, usage: { cache_read_input_tokens: 1, output_tokens: 1 },
+    ...(profile ? { lane_profile: profile } : {}), ...(baseline != null ? { baseline_tokens: baseline } : {}) });
+  const slice = (issue, profile, { impl = [40000], review = 20000, shadow, triggers } = {}) => [
+    { event: "RUN_START", issue, repo: "o/r", run: `r${issue}${profile}` },
+    ...impl.map((b, i) => use("implement", profile, b, triggers?.[i])), ...(review != null ? [use("review", profile, review, "review")] : []),
+    { event: "RUN_TERMINAL", reason: "merged", ...(shadow ? { shadow_would: shadow } : {}) },
+  ];
+
+  it("summariseLog keeps the slice's lane profile and per-phase baselines; resumed sessions add none", () => {
+    const s = summariseLog(slice(1, "lane", { impl: [30000, null] }));
+    assert.equal(s.laneProfile, "lane");
+    assert.deepEqual(s.baselines, { implement: [30000], review: [20000] });
+    const mixed = summariseLog([{ event: "RUN_START", issue: 2 }, use("implement", "memory-off", 1), use("review", "lane", 2), { event: "RUN_TERMINAL", reason: "merged" }]);
+    assert.equal(mixed.laneProfile, "mixed");
+    const old = summariseLog([{ event: "RUN_START", issue: 3 }, use("implement", null, null), { event: "RUN_TERMINAL", reason: "merged" }]);
+    assert.equal(old.laneProfile, null); assert.deepEqual(old.baselines, {});
+  });
+
+  it("readLocalRuns keeps one summary per log by default and every RUN_START slice with allSlices", () => {
+    const dir = tmpDir("lane-profile-");
+    try {
+      writeFileSync(join(dir, "7.jsonl"), [...slice(7, "memory-off", { impl: [60000] }), ...slice(7, "lane", { impl: [25000] })].map((e) => JSON.stringify(e)).join("\n"));
+      const last = readLocalRuns(dir);
+      assert.equal(last.length, 1); assert.equal(last[0].laneProfile, "lane");
+      const all = readLocalRuns(dir, { allSlices: true });
+      assert.deepEqual(all.map((r) => [r.issue, r.laneProfile, r.baselines.implement[0]]), [[7, "memory-off", 60000], [7, "lane", 25000]]);
+    } finally { rmSync(dir, { recursive: true, force: true }); }
+  });
+
+  it("compares baseline and conformance per profile on a pinned issue set", () => {
+    const runs = [
+      slice(1, "memory-off", { impl: [60000, 70000] }), slice(2, "memory-off", { impl: [50000], shadow: "conformance.scope" }),
+      slice(1, "lane", { impl: [20000, 30000], review: 10000 }), slice(2, "lane", { impl: [24000], review: 12000, triggers: ["conformance"] }),
+      slice(3, "lane", { impl: [99999] }), [{ event: "RUN_START", issue: 4 }, use("implement", null, null), { event: "RUN_TERMINAL", reason: "merged" }],
+      [{ event: "RUN_START", issue: 5 }, use("implement", "lane", 1)], // unfinished: left out
+    ].map((l) => summariseLog(l));
+    const all = laneProfileReport(runs);
+    assert.deepEqual(all.profiles.map((p) => [p.profile, p.n]), [["memory-off", 2], ["lane", 3], ["unknown", 1]]);
+    assert.deepEqual(all.issues_in_all, []);
+    const rep = laneProfileReport(runs, { issues: [1, 2] });
+    assert.deepEqual(rep.profiles.map((p) => p.profile), ["memory-off", "lane"]);
+    const [before, after] = rep.profiles;
+    assert.deepEqual(before.phases, [{ phase: "implement", sessions: 3, median: 60000, mean: 60000 }, { phase: "review", sessions: 2, median: 20000, mean: 20000 }]);
+    assert.deepEqual(after.phases[0], { phase: "implement", sessions: 3, median: 24000, mean: 24667 });
+    assert.deepEqual([before.reviewed, before.conformance_rate, before.shadow_blocks], [2, 0.5, 1]);
+    assert.deepEqual([after.reviewed, after.conformance_rate, after.shadow_blocks], [2, 0.5, 0]);
+    assert.deepEqual(rep.issues_in_all, [1, 2]);
+    const md = renderLaneProfiles(rep);
+    assert.match(md, /\| lane \| implement \| 3 \| 24000 \| 24667 \|/);
+    assert.match(md, /\| memory-off \| 2 \| 0\.5 \(2\) \| 1 \|/);
+    assert.match(md, /every profile: #1, #2/);
+    assert.match(renderLaneProfiles(laneProfileReport([])), /\| none \|/);
+  });
+
+  it("--lane-profile reads run logs only (no GitHub crawl); --issues takes a comma list", async () => {
+    assert.equal(parseArgs(["--lane-profile"]).laneProfile, true); assert.equal(parseArgs([]).laneProfile, false);
+    assert.deepEqual(parseArgs(["--issues", "4,12"]).issues, [4, 12]); assert.deepEqual(parseArgs(["--issues=7"]).issues, [7]);
+    assert.throws(() => parseArgs(["--issues", "4,x"]), /--issues must be a comma list/);
+    const rep = await buildReport({ repo: "o/r", laneProfile: true, issues: [1] }, { localRuns: [summariseLog(slice(1, "lane")), summariseLog(slice(2, "lane"))],
+      releaseTimes: [], gh: async () => { throw new Error("no network"); } });
+    assert.deepEqual(rep.query.lane_profile.profiles.map((p) => [p.profile, p.n]), [["lane", 1]]);
+  });
+});
+
+describe("rework by trigger and first-pass rate (#4362)", () => {
+  const use = (attempt, trigger, usd, phase = "implement") => ({ event: "PHASE_ATTEMPT_USAGE", phase, attempt, ...(trigger ? { trigger } : {}), cost_usd: usd, usage: { cache_read_input_tokens: 10, output_tokens: 1 } });
+  const log = (issue, attempts) => [{ event: "RUN_START", issue, repo: "o/r", run: `r${issue}`, version: "1.30.0", ts: "2026-10-20T00:00:00Z" }, ...attempts,
+    { event: "RUN_TERMINAL", reason: "merged", ts: "2026-10-20T01:00:00Z" }];
+  const runs = [
+    log(1, [use(1, "plan", 1), use(2, "build", 3)]),
+    log(2, [use(1, "plan", 1), use(2, "build", 3), use(3, "precheck", 2), use(4, "review", 4), use(1, "review", 0.2, "review")]),
+    log(3, [use(1, "plan", 1), use(2, "build", 3), use(3, "timeout_restart", 1), use(4, "recovery", 1)]),
+    log(4, [use(1, null, 1), use(2, null, 3), use(3, null, 5)]), // written before the field existed
+    log(5, [use(1, "plan", 1)]), // closed at the plan: never reached the build
+  ].map((l) => summariseLog(l));
+
+  it("summariseLog keeps builder attempts in order with their trigger (null when untagged)", () => {
+    assert.deepEqual(runs[1].builderAttempts.map((a) => a.trigger), ["plan", "build", "precheck", "review"]);
+    assert.deepEqual(runs[3].builderAttempts.map((a) => [a.attempt, a.trigger, a.cost_usd]), [[1, null, 1], [2, null, 3], [3, null, 5]]);
+  });
+
+  it("aggregate counts rework by trigger (untagged as unknown, never dropped) and the first-pass rate with counts", () => {
+    const g = aggregate({ runs }).groups[0];
+    assert.deepEqual(g.rework.byTrigger, { precheck: { n: 1, usd: 2 }, review: { n: 1, usd: 4 }, unknown: { n: 3, usd: 9 } });
+    assert.deepEqual([g.rework.n, g.rework.usd], [5, 15]);
+    assert.deepEqual(g.rework.attempts.build, { n: 3, usd: 9 });
+    assert.deepEqual(g.firstPass, { rate: 2 / 3, firstPass: 2, n: 3, unknown: 1 });
+    const md = renderMarkdown(aggregate({ runs }));
+    assert.match(md, /first-pass rate \| 67% \(2\/3 lanes that reached the build, 1 unknown\)/);
+    assert.match(md, /### Rework by trigger/);
+    assert.match(md, /\| unknown \| 3 \| \$9\.00 \|/);
+    assert.match(md, /\| total \| 5 \| \$15\.00 \|/);
+  });
+
+  it("GitHub phase_attempt records carry the trigger into builderAttempts, ordered by attempt", () => {
+    const rec = (attempt, trigger, ts) => ({ event: "phase_attempt", repo: "o/r", issue: 9, run: "r_9", phase: "implement", attempt, ...(trigger ? { trigger } : {}), cost_usd: 1, output_tokens: 1, ts });
+    const [run] = runsFromRecords([rec(3, "ci_red", "t3"), rec(1, "plan", "t1"), rec(2, "build", "t2"), rec(2, "timeout_restart", "t2b"), rec(4, null, "t4"),
+      { event: "run_terminal", repo: "o/r", issue: 9, run: "r_9", reason: "merged", ts: "t5" }]);
+    assert.deepEqual(run.builderAttempts.map((a) => [a.attempt, a.trigger]), [[1, "plan"], [2, "build"], [2, "timeout_restart"], [3, "ci_red"], [4, null]]);
+  });
+});
+
+describe("per-attempt cost deltas (#4407)", () => {
+  const use = (attempt, usd, delta, phase = "implement") => ({ event: "PHASE_ATTEMPT_USAGE", phase, attempt, trigger: attempt === 1 ? "plan" : "build", cost_usd: usd,
+    ...(delta === undefined ? {} : { cost_usd_delta: delta, cost_usd_session: usd }), usage: { cache_read_input_tokens: 10, output_tokens: 1 } });
+  const log = (attempts) => [{ event: "RUN_START", issue: 1, repo: "o/r", run: "r1", version: "1.30.0", ts: "2026-10-20T00:00:00Z" }, ...attempts,
+    { event: "RUN_TERMINAL", reason: "merged", ts: "2026-10-20T01:00:00Z" }];
+
+  it("a local run sums cost_usd_delta when present, else cost_usd (old logs)", () => {
+    const r = summariseLog(log([use(1, 2, 2), use(2, 5, 3), use(1, 0.5, 0.5, "review")]));
+    assert.equal(r.costUsd, 5.5);
+    assert.equal(r.phases.implement.usd, 5);
+    assert.deepEqual(r.builderAttempts.map((a) => a.cost_usd), [2, 3]);
+    const old = summariseLog(log([use(1, 2), use(2, 5)]));
+    assert.equal(old.costUsd, 7); assert.deepEqual(old.builderAttempts.map((a) => a.cost_usd), [2, 5]);
+  });
+
+  it("GitHub phase_attempt records sum cost_usd_delta when present, else cost_usd", () => {
+    const rec = (attempt, usd, delta) => ({ event: "phase_attempt", repo: "o/r", issue: 9, run: "r_9", phase: "implement", attempt, cost_usd: usd, ...(delta === undefined ? {} : { cost_usd_delta: delta }), output_tokens: 1, ts: `t${attempt}` });
+    const [run] = runsFromRecords([rec(1, 2, 2), rec(2, 5, 3), rec(3, 4), { event: "run_terminal", repo: "o/r", issue: 9, run: "r_9", reason: "merged", ts: "t9" }]);
+    assert.equal(run.phases.implement.usd, 9);
+    assert.deepEqual(run.builderAttempts.map((a) => a.cost_usd), [2, 3, 4]);
+  });
+});
+
+describe("first-pass rate with and without matching pitfalls (#4375)", () => {
+  const use = (attempt, trigger) => ({ event: "PHASE_ATTEMPT_USAGE", phase: "implement", attempt, trigger, cost_usd: 1, usage: { cache_read_input_tokens: 10, output_tokens: 1 } });
+  const log = (issue, attempts, { term = {}, pack = null } = {}) => [{ event: "RUN_START", issue, repo: "o/r", run: `r${issue}`, version: "1.30.0", ts: "2026-10-20T00:00:00Z" },
+    ...(pack ? [{ event: "CONTEXT_PACK", mode: "shadow", pitfalls: pack }] : []), ...attempts, { event: "RUN_TERMINAL", reason: "merged", ts: "2026-10-20T01:00:00Z", ...term }];
+  const clean = [use(1, "plan"), use(2, "build")], rework = [...clean, use(3, "precheck")];
+
+  it("summariseLog reads pitfalls_matched from RUN_TERMINAL, else the CONTEXT_PACK pitfalls, else null", () => {
+    assert.equal(summariseLog(log(1, clean, { term: { pitfalls_matched: 2 } })).pitfallsMatched, 2);
+    assert.equal(summariseLog(log(1, clean, { pack: [{ sig: "a", match: "files" }, { sig: "b", match: "repo" }] })).pitfallsMatched, 1);
+    assert.equal(summariseLog(log(1, clean)).pitfallsMatched, null, "older logs: unknown");
+  });
+
+  it("aggregate splits the first-pass rate by matching pitfalls and leaves unknown runs out; render prints both rows", () => {
+    const runs = [
+      log(1, clean, { term: { pitfalls_matched: 1 } }), log(2, clean, { term: { pitfalls_matched: 3 } }), log(3, rework, { term: { pitfalls_matched: 1 } }),
+      log(4, rework, { term: { pitfalls_matched: 0 } }), log(5, clean, { term: { pitfalls_matched: 0 } }), log(6, rework, { term: { pitfalls_matched: 0 } }),
+      log(7, clean), // no pitfall field: unknown
+    ].map((l) => summariseLog(l));
+    const g = aggregate({ runs }).groups[0];
+    assert.deepEqual(g.firstPassByPitfalls, { with: { rate: 2 / 3, firstPass: 2, n: 3 }, without: { rate: 1 / 3, firstPass: 1, n: 3 } });
+    assert.equal(g.firstPass.n, 7, "the overall rate still counts every lane");
+    const md = renderMarkdown(aggregate({ runs }));
+    assert.match(md, /first-pass with matching pitfalls \| 67% \(2\/3\)/);
+    assert.match(md, /first-pass without matching pitfalls \| 33% \(1\/3\)/);
+    const none = renderMarkdown(aggregate({ runs: [log(8, clean)].map((l) => summariseLog(l)) }));
+    assert.ok(!/matching pitfalls/.test(none), "no pitfall data: no rows");
+  });
+});
+
+describe("fail-soft alerts (#4354)", () => {
+  const ts = (i) => new Date(Date.UTC(2026, 9, 1) + i * 60000).toISOString();
+  // One lane per call; `extra` are the events after RUN_START.
+  const lane = (i, extra = [], { repo = "o/r", code = "completed", reason = "merged" } = {}) => [
+    { event: "RUN_START", ts: ts(i), repo, issue: 100 + i }, ...extra, ...(code ? [{ event: "RUN_TERMINAL", reason, code }] : [])];
+  const deps = (eco, status, why = null) => ({ event: "DEPS_PREPARED", eco, status, reason: why });
+  const lanes = (n, f, o) => Array.from({ length: n }, (_, i) => lanesOfLog(lane(i, f(i), typeof o === "function" ? o(i) : o))[0]);
+
+  it("DEPS_PREPARED failed above 20% per ecosystem, with rate, sample and an example reason; healthy ecosystems stay quiet", () => {
+    const l = lanes(10, (i) => [deps("pnpm", i < 3 ? "failed" : "hit", "tool not found: spawn pnpm ENOENT"), deps("uv", "hit")]);
+    const a = computeAlerts(l, { repo: "o/r" });
+    assert.deepEqual(a.map((x) => [x.event, x.count, x.sample]), [["DEPS_PREPARED failed (pnpm)", 3, 10]]);
+    assert.equal(a[0].example, "tool not found: spawn pnpm ENOENT");
+    assert.deepEqual(computeAlerts(lanes(10, (i) => [deps("uv", i < 2 ? "failed" : "hit")]), { repo: "o/r" }), [], "exactly 20% is not above");
+    const line = renderAlerts(a)[0];
+    assert.match(line, /^DEPS_PREPARED failed \(pnpm\): 30% \(3\/10 lanes, threshold 20%\), e\.g\. "tool not found: spawn pnpm ENOENT"$/);
+  });
+
+  it("deferred-to-ci above 30% of lanes and a TOOLCHAIN_WARNING repeated on more than 50% of lanes", () => {
+    const pre = (deferred) => ({ event: "PRECHECK_COMMANDS", round: 0, results: [{ command: "npm test", rc: 0, ms: 5 }, ...(deferred ? [{ command: "npx tsc", status: "deferred-to-ci" }] : [])] });
+    const warn = { event: "TOOLCHAIN_WARNING", warning: "host node v26 does not satisfy the .nvmrc pin \"22\"" };
+    const a = computeAlerts(lanes(10, (i) => [pre(i < 4), ...(i < 6 ? [warn] : [])]), { repo: "o/r" });
+    assert.deepEqual(a.map((x) => [x.event, x.count, x.sample, x.example]), [["deferred-to-ci", 4, 10, "npx tsc"], ["TOOLCHAIN_WARNING", 6, 10, warn.warning]]);
+    assert.deepEqual(computeAlerts(lanes(10, (i) => [pre(i < 3), ...(i < 5 ? [warn] : [])]), { repo: "o/r" }), [], "30% and 50% are not above");
+  });
+
+  it("an outcome code whose share doubled against the previous window; needs a full previous window; completed never alerts", () => {
+    // 20 lanes, window 10: previous window has 1 ci.red (10%), current has 3 (30%)
+    const code = (i) => ({ code: [0, 12, 15, 18].includes(i) ? "ci.red" : "completed", reason: [0, 12, 15, 18].includes(i) ? "needs-human" : "merged" });
+    const l = lanes(20, () => [], code);
+    const a = computeAlerts(l, { repo: "o/r", window: 10 });
+    assert.deepEqual(a.map((x) => [x.event, x.count, x.sample, x.prevRate, x.example]), [["outcome ci.red", 3, 10, 0.1, "#118 needs-human"]]);
+    assert.match(renderAlerts(a)[0], /^outcome ci\.red: 30% \(3\/10 lanes, previous window 10%, threshold 2x\), e\.g\. "#118 needs-human"$/);
+    assert.deepEqual(computeAlerts(l.slice(5), { repo: "o/r", window: 10 }), [], "a previous window under min_lanes never compares");
+    assert.deepEqual(computeAlerts(lanes(20, () => [], (i) => ({ code: i < 15 ? "invalid" : "completed" })), { repo: "o/r", window: 10 }), []);
+  });
+
+  it("rolling window per repo, ordered by start: old failures and other repos fall out; under 10 lanes or no logs gives nothing", () => {
+    const bad = lanes(10, () => [deps("pnpm", "failed", "x")]);
+    const good = lanes(50, () => [deps("pnpm", "hit")]).map((l, i) => ({ ...l, startTs: ts(100 + i) }));
+    assert.deepEqual(computeAlerts([...good, ...bad], { repo: "o/r" }), [], "the 50 newest lanes are healthy");
+    assert.equal(computeAlerts(bad, { repo: "o/r" }).length, 1);
+    assert.deepEqual(computeAlerts(bad, { repo: "x/y" }), []);
+    assert.deepEqual(computeAlerts(bad.slice(1), { repo: "o/r" }), [], "9 lanes is too few");
+    assert.deepEqual(computeAlerts([], { repo: "o/r" }), []);
+    assert.deepEqual(readLanes(join(td, "missing")), []);
+  });
+
+  it("readLanes splits every RUN_START slice of every log (local files only) and keeps the repo", () => {
+    const dir = join(td, "runs"); mkdirSync(dir);
+    const text = (evs) => evs.map((e) => JSON.stringify(e)).join("\n") + "\n";
+    writeFileSync(join(dir, "7.jsonl"), text([...lane(0, [deps("cargo", "failed", "lock drift")]), ...lane(1, [{ event: "WORKTREE_SPARSE_FALLBACK", reason: "no cone" }], { code: null })]) + "{torn");
+    writeFileSync(join(dir, "8.jsonl"), text(lane(2, [], { repo: "a/b" })));
+    const got = readLanes(dir);
+    assert.deepEqual(got.map((l) => [l.repo, l.issue, l.code, l.deps.length, l.sparseFallback]),
+      [["o/r", 100, "completed", 1, null], ["o/r", 101, null, 0, "no cone"], ["a/b", 102, "completed", 0, null]]);
+  });
+
+  it("thresholds come from forge.yaml stats.alerts.* (nested block and flat form) and change the result", () => {
+    assert.deepEqual(readAlertConfig(join(td, "none")), DEFAULT_ALERTS);
+    writeFileSync(join(td, "forge.yaml"), "stats:\n  alerts:\n    min_lanes: 5\n    deps_failed_pct: 50\n    alerts_window: 8\n    window: 30\n    deferred_to_ci_pct: nope\n");
+    const c = readAlertConfig(td);
+    assert.deepEqual(c, { ...DEFAULT_ALERTS, minLanes: 5, depsFailedPct: 50, window: 8 });
+    const l = lanes(6, (i) => [deps("pnpm", i < 2 ? "failed" : "hit", "x")]);
+    assert.deepEqual(computeAlerts(l, { repo: "o/r" }), [], "default min_lanes 10");
+    assert.deepEqual(computeAlerts(l, { repo: "o/r", ...c }), [], "33% is under the configured 50%");
+    assert.equal(computeAlerts(l, { repo: "o/r", ...c, depsFailedPct: 20 }).length, 1);
+  });
+
+  it("stats report carries alerts and renders an ALERTS section only when one fired; --no-local skips them; no gh call for alerts", async () => {
+    const { gh } = fakeGh();
+    const localLanes = lanes(10, () => [deps("pnpm", "failed", "spawn pnpm ENOENT")]);
+    const rep = await buildReport({ repo: "o/r", local: true, cache: false, minN: 10 }, { gh, env: { ...ENV, FORGE_RUNS_DIR: join(td, "none") }, releaseTimes: [], localRuns: [], localLanes, alertConfig: {} });
+    assert.equal(rep.alerts.length, 1);
+    assert.match(renderMarkdown(rep), /## ALERTS\n\n- DEPS_PREPARED failed \(pnpm\): 100% \(10\/10 lanes/);
+    assert.equal(JSON.parse(renderJson(rep)).alerts[0].event, "DEPS_PREPARED failed (pnpm)");
+    const off = await buildReport({ repo: "o/r", local: false, cache: false, minN: 10 }, { gh, env: ENV, releaseTimes: [], localLanes });
+    assert.deepEqual(off.alerts, []);
+    assert.doesNotMatch(renderMarkdown(off), /ALERTS/);
+    // computed from local run logs alone: no gh in scope at all
+    assert.equal(computeAlerts(localLanes, { repo: "o/r" }).length, 1);
+  });
+});
+
+describe("env.tool_missing per release (forge#4414)", () => {
+  // The #4393 replay criterion is read as outcome-code counts per engine version; pin that the report surfaces them.
+  const run = (issue, version, reason, code) => ({ issue, repo: "o/r", version, reason, code, finished: true, usageEvents: 1, endTs: "2026-10-23T10:00:00Z", phases: { implement: { cr: 1, out: 1, ms: 1 } } });
+  const runs = [run(1, "1.30.0", "needs-human", "env.tool_missing"), run(2, "1.30.0", "merged", "completed"),
+    run(3, "1.31.0", "merged", "completed"), run(4, "1.31.0", "needs-human", "dependency.open")];
+  const codesOf = (g) => Object.fromEntries(g.flow.codes.map((c) => [c.code, c.count]));
+
+  it("counts env.tool_missing per version and drops pre-release lanes with sinceVersion", () => {
+    const all = aggregate({ runs }, { releaseTimes: [] });
+    assert.equal(codesOf(all.groups.find((g) => g.version === "1.30.0"))["env.tool_missing"], 1);
+    const after = aggregate({ runs }, { releaseTimes: [], sinceVersion: "1.31.0" });
+    assert.deepEqual(after.groups.map((g) => g.version), ["1.31.0"]);
+    assert.equal(codesOf(after.groups[0])["env.tool_missing"], undefined);
+    assert.equal(after.groups[0].flow.codesN, 2);
+  });
+});

package/commands/orchestrate.md

@@ -100,7 +100,9 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
   (live lanes are re-adopted, nothing is launched twice). The runner is hosted by the detached `forgedock schedd` daemon,
   which resumes every daemon-hosted batch that has no report yet when it starts, so a closed session needs no manual
   `--resume`; a batch started without the daemon still needs `forgedock orchestrate --resume <id>`. Change K of a running batch with
-  `forgedock orchestrate --set-parallel <id> K`. The runner requeues `deferred` lanes by code (host busy after its
+  `forgedock orchestrate --set-parallel <id> K`. Issues filed while a batch runs join it with
+  `forgedock orchestrate --add <id> N...` (or `--resume <id> N...`), under its K and DAG; never start a second batch
+  for them. Its pre-filter still closes already-fixed ones. The runner requeues `deferred` lanes by code (host busy after its
   retry-in, a usage limit holds admissions for that account (backend identity) until the reset, across every batch on
   the host; `forgedock limits show|clear <id>|--all` lists or drops holds; transient merge/push/idle failures back off; non-host
   deferrals stop after `orchestration.max_deferrals`), relaunches an `engine-error` once, and stops admitting after

package/commands/review-pr-agents/protocols.md

@@ -210,7 +210,7 @@ Append this block at the very end of your comment (after the `---` footer line,
 8. **HTML comments**: The block is invisible in rendered markdown but parseable by the review system
 9. **Agent marker**: Include exactly one `<!-- FORGE:REVIEW-AGENT:{domain} -->` marker in the persisted body, where `{domain}` is the marker domain the orchestrator passes you (the canonical value from the roster-to-marker table in `review-pr.md`, e.g. `infra` and `scraper`; never a lowercased display name such as `infrastructure` or `scraping`).
 10. **Reviewed SHA**: Include exactly one line `Reviewed-SHA: [REVIEW_SHA]` (the full 40-hex head you were pinned to) in the persisted body, outside the marker. `/review-pr` scopes its completion wait and idempotent re-entry to this line, so a comment without the current head does not count as a completed review. Do NOT put the SHA inside the `<!-- FORGE:REVIEW-AGENT:{domain} -->` marker.
-11. **Failure scenario and scope (provenance inputs)**: every finding's body section (the prose above the findings block) MUST carry a `**Failure scenario**` line — concrete inputs leading to a wrong output or behaviour — and a `**Scope**: changed` or `**Scope**: pre-existing` line (`changed` = the cited line is added or modified by this PR; `pre-existing` = it was already there). `/review-pr` §6B.4 reads these: a finding without a failure scenario is demoted to a note (unless CRITICAL/HIGH, any confidence), and a MEDIUM or lower finding whose cited line is not in the PR diff is routed as pre-existing (CRITICAL/HIGH never are; see §5). The `**Scope**` tag is informational and never gates routing. Speculative hardening with no failure scenario is a note, not a finding. The one-line `<!-- FINDING:... -->` grammar above is unchanged. <!-- Added: forge#3452 -->
+11. **Failure scenario and scope (provenance inputs)**: every finding's body section (the prose above the findings block) MUST carry a `**Failure scenario**` line — concrete inputs leading to a wrong output or behaviour — and a `**Scope**: changed` or `**Scope**: pre-existing` line (`changed` = the cited line is added or modified by this PR; `pre-existing` = it was already there). `/review-pr` §6B.4 reads these: a finding without a failure scenario is demoted to a note (unless CRITICAL/HIGH, any confidence, or CONFIRMED MEDIUM; a plain `Failure scenario:`, `Scenario:`, `Repro:` or `Steps to reproduce:` line also counts as the scenario), and a MEDIUM or lower finding whose cited line is not in the PR diff is routed as pre-existing (CRITICAL/HIGH never are; see §5). The `**Scope**` tag is informational and never gates routing. Speculative hardening with no failure scenario is a note, not a finding. The one-line `<!-- FINDING:... -->` grammar above is unchanged. <!-- Added: forge#3452 -->
 12. **Fix efficacy (linked-issue contradiction)**: when a finding says the PR's linked issue is NOT fixed, or that the PR re-introduces the defect it claims to fix, its body section MUST carry a `**Fix efficacy**: fails` (the fix does not work) or `**Fix efficacy**: regresses` (the PR re-introduces the defect) line, and the prose may include the token on its own line (`FIX_EFFICACY: fails|regresses`; line-anchored, a mid-sentence mention does not count). Opt-in on the tag, never on keywords. `/review-pr` §6B.5 passes it to `scripts/classify-finding.sh --fix-efficacy`, which exempts a MEDIUM finding from the fix-PR damper. Use it only for the linked issue's own fix; never for unrelated defects. The one-line `<!-- FINDING:... -->` grammar is unchanged. <!-- Added: forge#3693 -->
 
 ### Domain Prefixes