forgedock 1.36.0 → 1.37.0

package/bin/stats/compounding.mjs

@@ -0,0 +1,157 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * #4696: compounding metrics for `forgedock stats --compounding`: does a repo get safer the longer ForgeDock works on it.
+ * Deterministic (0 LLM calls): guards (#4692 FORGE:GUARD comments), `reopened` issue events, git blame, and gate events
+ * from the FORGE:TELEMETRY records. The report core is pure; `gatherCompounding` supplies the GitHub and git inputs.
+ *   reintroduction: a guarded fix whose issue was reopened after the lane closed it (a), or whose merge commit is the blame
+ *     of the old-side lines of a later guarded fix within 90 days (b; squash and rebase merges, where merge_sha is the commit)
+ *   reopen_after_close: lane-closed (merged) issues that were reopened afterwards
+ *   guard / decision counts: gate events guard_run, guard_broken, supersedes_accepted, decision_superseded,
+ *     decision_violation (#4693, #4695); null (n/a) until a repo has recorded any of that kind
+ * Grouped as a rolling window, per release and per lane-age bucket (lane rank = merge order of the repo's lanes).
+ */
+import { execFileSync } from "node:child_process";
+import { collectGuards } from "../lean/guards.mjs";
+import { parsePages } from "./github-source.mjs";
+
+export const REINTRO_DAYS = 90;
+export const DEFAULT_BUCKETS = [100, 500];
+export const DEFAULT_WINDOW_MS = 30 * 86400000;
+export const COUNTED_EVENTS = ["guard_run", "guard_broken", "supersedes_accepted", "decision_superseded", "decision_violation"];
+const DAY = 86400000;
+const ratio = (a, b) => (b ? Math.round((a / b) * 1000) / 1000 : null);
+const ms = (ts) => { const t = Date.parse(ts ?? ""); return Number.isFinite(t) ? t : null; };
+
+export const bucketLabel = (rank, edges) => {
+  if (rank == null) return "unranked";
+  let lo = 1;
+  for (const e of edges) { if (rank <= e) return `${lo}-${e}`; lo = e + 1; }
+  return `${lo}+`;
+};
+
+/** Guards whose fixed lines a later guarded fix changed within 90 days: [{fix, by}] (fix = earlier guard's issue). Git probes may return null (unresolved). */
+export function blameLineage(guards, git) {
+  const t = new Map(), hits = [];
+  const time = (g) => { if (!t.has(g.merge_sha)) t.set(g.merge_sha, git.time(g.merge_sha)); return t.get(g.merge_sha); };
+  const byCommit = new Map(guards.filter((g) => g.merge_sha).map((g) => [g.merge_sha, g]));
+  let unresolved = 0;
+  for (const g2 of guards) {
+    if (!g2.merge_sha) { unresolved++; continue; }
+    const ranges = git.oldRanges(g2.merge_sha);
+    if (!ranges) { unresolved++; continue; }
+    const seen = new Set();
+    for (const r of ranges) for (const sha of git.blame(`${g2.merge_sha}^`, r.file, r.start, r.end) ?? []) {
+      const g1 = byCommit.get(sha);
+      if (!g1 || g1.issue === g2.issue || seen.has(g1.issue)) continue;
+      const a = time(g1), b = time(g2);
+      if (a == null || b == null || b < a || b - a > REINTRO_DAYS * 86400) continue;
+      seen.add(g1.issue); hits.push({ fix: g1.issue, by: g2.issue });
+    }
+  }
+  return { hits, unresolved };
+}
+
+/** Metrics of one group of lane entries and the gate events falling in it. */
+function summarize(entries, events, have, minN) {
+  const guarded = entries.filter((e) => e.guarded), lanes = entries.filter((e) => e.merged);
+  const count = (k) => (have[k] ? events.filter((e) => e.event === k).length : null);
+  return {
+    lanes: lanes.length,
+    reopen_after_close: { reopened: lanes.filter((e) => e.reopened).length, rate: ratio(lanes.filter((e) => e.reopened).length, lanes.length) },
+    reintroduction: { guarded: guarded.length, reintroduced: guarded.filter((e) => e.reintroduced).length, rate: ratio(guarded.filter((e) => e.reintroduced).length, guarded.length) },
+    guard: { tests_run: count("guard_run"), broken_caught: count("guard_broken"), supersedes_accepted: count("supersedes_accepted") },
+    decisions: { superseded: count("decision_superseded"), violations: count("decision_violation") },
+    low_n: Math.max(lanes.length, guarded.length) < minN,
+  };
+}
+
+/**
+ * Pure report. guards: parsed FORGE:GUARD objects; reopens: [{issue, ts}]; runs: toModel runs; events: toModel gate events;
+ * git: {time(sha)->epoch s|null, oldRanges(sha)->[{file,start,end}]|null, blame(rev,file,start,end)->[sha]|null}.
+ */
+export function compoundingReport({ guards = [], reopens = [], runs = [], events = [], git, windowMs = DEFAULT_WINDOW_MS, buckets = DEFAULT_BUCKETS, now = Date.now(), minN = 10 }) {
+  const merged = runs.filter((r) => r.reason === "merged").sort((a, b) => (ms(a.endTs) ?? 0) - (ms(b.endTs) ?? 0) || a.issue - b.issue);
+  const byIssue = new Map(merged.map((r, i) => [r.issue, { issue: r.issue, merged: true, rank: i + 1, version: r.version ?? null, at: ms(r.endTs), guarded: false, reopened: false, reintroduced: false }]));
+  const gset = new Map();
+  for (const g of guards) if (!gset.has(g.issue)) gset.set(g.issue, g);
+  const uniq = [...gset.values()];
+  for (const g of uniq) {
+    const e = byIssue.get(g.issue) ?? { issue: g.issue, merged: false, rank: null, version: null, at: null, guarded: false, reopened: false, reintroduced: false };
+    e.guarded = true;
+    const secs = git?.time && g.merge_sha ? git.time(g.merge_sha) : null;
+    if (e.at == null && secs != null) e.at = secs * 1000;
+    byIssue.set(g.issue, e);
+  }
+  for (const e of byIssue.values()) { // (a) and reopen-after-close: a reopen after the lane closed the issue
+    e.reopened = reopens.some((r) => r.issue === e.issue && (e.at == null || (ms(r.ts) ?? 0) > e.at));
+    if (e.guarded && e.reopened) e.reintroduced = true;
+  }
+  const lineage = git && uniq.length ? blameLineage(uniq, git) : { hits: [], unresolved: uniq.length };
+  for (const h of lineage.hits) if (byIssue.has(h.fix)) byIssue.get(h.fix).reintroduced = true;
+  const entries = [...byIssue.values()];
+  const have = Object.fromEntries(COUNTED_EVENTS.map((k) => [k, events.some((e) => e.event === k)]));
+  const evMs = (e) => ms(e.ts);
+  const out = { rolling: null, by_release: {}, by_lane_age: {}, sources: { guards: uniq.length, reopens: reopens.length, blame_hits: lineage.hits.length, blame_unresolved: lineage.unresolved }, buckets };
+  out.rolling = { window_days: Math.round(windowMs / DAY), ...summarize(entries.filter((e) => e.at != null && now - e.at <= windowMs && e.at <= now),
+    events.filter((e) => evMs(e) != null && now - evMs(e) <= windowMs && evMs(e) <= now), have, minN) };
+  const versions = [...new Set([...entries.map((e) => e.version), ...events.map((e) => e.version)].filter(Boolean))].sort((a, b) => a.localeCompare(b, undefined, { numeric: true }));
+  for (const v of versions) out.by_release[v] = summarize(entries.filter((e) => e.version === v), events.filter((e) => e.version === v), have, minN);
+  const labels = [...new Set(entries.map((e) => bucketLabel(e.rank, buckets)))];
+  const order = (l) => (l === "unranked" ? Infinity : Number.parseInt(l, 10));
+  for (const l of labels.sort((a, b) => order(a) - order(b))) {
+    const inB = entries.filter((e) => bucketLabel(e.rank, buckets) === l);
+    const evs = events.filter((e) => { const r = byIssue.get(e.issue); return r && bucketLabel(r.rank, buckets) === l; });
+    out.by_lane_age[l] = summarize(inB, evs, have, minN);
+  }
+  return out;
+}
+
+const pct = (x) => (x == null ? "n/a" : `${Math.round(x * 1000) / 10}%`);
+const n = (x) => (x == null ? "n/a" : String(x));
+function line(label, s) {
+  return `  ${label}: ${s.lanes} lanes, reintroduction ${pct(s.reintroduction.rate)} (${s.reintroduction.reintroduced}/${s.reintroduction.guarded}), reopen-after-close ${pct(s.reopen_after_close.rate)}, `
+    + `guard runs ${n(s.guard.tests_run)} broken ${n(s.guard.broken_caught)} supersedes ${n(s.guard.supersedes_accepted)}, decisions superseded ${n(s.decisions.superseded)} violations ${n(s.decisions.violations)}${s.low_n ? " (low n)" : ""}`;
+}
+
+export function renderCompounding(r) {
+  const rows = [`compounding: ${r.sources.guards} guards, ${r.sources.reopens} reopen events, ${r.sources.blame_hits} blame hits (${r.sources.blame_unresolved} guards unresolved)`,
+    line(`rolling ${r.rolling.window_days}d`, r.rolling), "by lane age:", ...Object.entries(r.by_lane_age).map(([k, s]) => line(k, s))];
+  const rel = Object.entries(r.by_release);
+  return [...rows, "by release:", ...(rel.length ? rel.map(([k, s]) => line(k, s)) : ["  n/a"])].join("\n");
+}
+
+const tryGit = (cwd, args) => { try { return execFileSync("git", ["-C", cwd, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], maxBuffer: 64 * 1024 * 1024 }); } catch { return null; } };
+
+/** git probes over a local clone; every miss (unknown sha, shallow clone) is null, never a thrown error. */
+export function gitProbes(cwd) {
+  return {
+    time: (sha) => { const t = Number(tryGit(cwd, ["show", "-s", "--format=%ct", sha])?.trim()); return Number.isFinite(t) && t > 0 ? t : null; },
+    oldRanges: (sha) => {
+      const d = tryGit(cwd, ["diff", "-U0", "--no-renames", `${sha}^`, sha]);
+      if (d == null) return null;
+      const out = []; let file = null;
+      for (const l of d.split("\n")) {
+        if (l.startsWith("--- ")) { file = l === "--- /dev/null" ? null : l.replace(/^--- a\//, ""); continue; }
+        const m = /^@@ -(\d+)(?:,(\d+))? \+/.exec(l);
+        if (m && file && m[2] !== "0") out.push({ file, start: Number(m[1]), end: Number(m[1]) + (m[2] === undefined ? 1 : Number(m[2])) - 1 });
+      }
+      return out;
+    },
+    blame: (rev, file, start, end) => {
+      const o = tryGit(cwd, ["blame", "--porcelain", "-L", `${start},${end}`, rev, "--", file]);
+      return o == null ? null : [...new Set(o.split("\n").filter((l) => /^[0-9a-f]{40} \d+ \d+/.test(l)).map((l) => l.slice(0, 40)))];
+    },
+  };
+}
+
+/** GitHub + git inputs for compoundingReport: guards from FORGE:GUARD comments, reopen events from one paginated events call. */
+export async function gatherCompounding({ repo, gh, cwd = process.cwd() }) {
+  const guards = await collectGuards({ gh, repo });
+  let reopens = [];
+  try {
+    reopens = parsePages(await gh(["api", "--paginate", `repos/${repo}/issues/events?per_page=100`]))
+      .filter((e) => e?.event === "reopened" && Number.isSafeInteger(e.issue?.number)).map((e) => ({ issue: e.issue.number, ts: e.created_at }));
+  } catch { /* events unreadable: reopen metrics read 0 reopens, shown with the count in sources */ }
+  return { guards, reopens, git: gitProbes(cwd) };
+}

package/bin/stats/compounding.test.mjs

@@ -0,0 +1,74 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { compoundingReport, renderCompounding, bucketLabel, gitProbes } from "./compounding.mjs";
+import { parseArgs } from "./index.mjs";
+
+const NOW = Date.parse("2026-10-24T00:00:00Z"), DAY = 86400000, S = (d) => Math.floor((NOW - d * DAY) / 1000);
+const run = (issue, daysAgo, version = "1.0.0") => ({ issue, reason: "merged", version, endTs: new Date(NOW - daysAgo * DAY).toISOString() });
+const guard = (issue, sha) => ({ v: 1, issue, merge_sha: sha, tests: [{ path: "t.mjs", ids: [] }], hunks: [] });
+// shas: a (issue 1, 40d ago), b (issue 2, 10d ago), c (issue 3, 1d ago)
+const TIMES = { a: S(40), b: S(10), c: S(1) };
+const fakeGit = (blames = {}) => ({ time: (s) => TIMES[s] ?? null, oldRanges: (s) => [{ file: "f.mjs", start: 1, end: 3 }], blame: (rev) => blames[rev] ?? [] });
+const base = { runs: [run(1, 40), run(2, 10), run(3, 1)], guards: [guard(1, "a"), guard(2, "b"), guard(3, "c")], now: NOW, minN: 1 };
+
+test("reintroduction (b): a later fix whose old lines blame to an earlier fix within 90 days", () => {
+  const r = compoundingReport({ ...base, git: fakeGit({ "b^": ["a"] }) });
+  assert.deepEqual(r.rolling.reintroduction, { guarded: 2, reintroduced: 0, rate: 0 }); // 30d window holds fixes 2, 3; the broken fix is #1 (40d ago)
+  assert.equal(r.sources.blame_hits, 1);
+  const all = Object.values(r.by_lane_age)[0];
+  assert.deepEqual(all.reintroduction, { guarded: 3, reintroduced: 1, rate: 0.333 });
+});
+
+test("reintroduction (b): older than 90 days or unknown time does not count; unresolved is reported, not zeroed", () => {
+  const far = { ...base, guards: [guard(1, "a"), guard(2, "b")], runs: [run(1, 200), run(2, 10)] };
+  const r = compoundingReport({ ...far, git: { ...fakeGit({ "b^": ["a"] }), time: (s) => ({ a: S(200), b: S(10) })[s] ?? null } });
+  assert.equal(r.sources.blame_hits, 0);
+  const u = compoundingReport({ ...base, git: { time: () => null, oldRanges: () => null, blame: () => null } });
+  assert.equal(u.sources.blame_unresolved, 3);
+  assert.equal(u.sources.blame_hits, 0);
+});
+
+test("reintroduction (a) and reopen-after-close: reopen after the lane closed the issue", () => {
+  const reopens = [{ issue: 2, ts: new Date(NOW - 5 * DAY).toISOString() }, { issue: 3, ts: new Date(NOW - 3 * DAY).toISOString() /* before close? no: close was 1d ago */ }];
+  const r = compoundingReport({ ...base, reopens, git: fakeGit() });
+  assert.equal(r.rolling.reopen_after_close.reopened, 1); // issue 2 reopened after its close; issue 3's reopen predates its close
+  assert.equal(r.rolling.reintroduction.reintroduced, 1);
+  assert.equal(r.rolling.lanes, 2);
+});
+
+test("guard and decision counts are n/a until a kind is recorded, then counted per window", () => {
+  const none = compoundingReport({ ...base, git: fakeGit() });
+  assert.equal(none.rolling.guard.broken_caught, null);
+  assert.equal(none.rolling.decisions.violations, null);
+  const events = [{ event: "guard_broken", issue: 2, ts: new Date(NOW - DAY).toISOString(), version: "1.0.0" }, { event: "guard_broken", issue: 1, ts: new Date(NOW - 60 * DAY).toISOString(), version: "1.0.0" }];
+  const r = compoundingReport({ ...base, events, git: fakeGit() });
+  assert.equal(r.rolling.guard.broken_caught, 1);
+  assert.equal(r.rolling.guard.tests_run, null);
+  assert.equal(r.by_release["1.0.0"].guard.broken_caught, 2);
+});
+
+test("lane-age buckets by merge order and rendering", () => {
+  assert.deepEqual([1, 100, 101, 500, 501, null].map((n) => bucketLabel(n, [100, 500])), ["1-100", "1-100", "101-500", "101-500", "501+", "unranked"]);
+  const runs = Array.from({ length: 4 }, (_, i) => run(i + 1, 40 - i));
+  const r = compoundingReport({ runs, guards: [], buckets: [2], now: NOW, minN: 1, git: fakeGit() });
+  assert.deepEqual(Object.keys(r.by_lane_age), ["1-2", "3+"]);
+  assert.equal(r.by_lane_age["3+"].lanes, 2);
+  assert.match(renderCompounding(r), /by lane age:\n  1-2: 2 lanes, reintroduction n\/a \(0\/0\)/);
+});
+
+test("gitProbes degrades to null outside a repo or for an unknown sha", () => {
+  const g = gitProbes("/nonexistent-dir-for-compounding");
+  assert.equal(g.time("deadbeef"), null);
+  assert.equal(g.oldRanges("deadbeef"), null);
+  assert.equal(g.blame("deadbeef^", "x", 1, 2), null);
+});
+
+test("parseArgs: --compounding takes --window and --bucket; --bucket alone is rejected", () => {
+  const o = parseArgs(["--compounding", "--window", "30d", "--bucket", "50,200"]);
+  assert.equal(o.compounding, true);
+  assert.deepEqual(o.bucket, [50, 200]);
+  assert.throws(() => parseArgs(["--bucket", "10"]), /needs --compounding/);
+  assert.throws(() => parseArgs(["--compounding", "--bucket", "9,5"]), /increasing/);
+});

package/bin/stats/index.mjs

@@ -19,6 +19,7 @@ import { batchEfficiency, windowEfficiency, compareEfficiency, formatEfficiencyL
 import { releaseMetrics, renderReleaseMetrics, metricsComment, postMetrics } from "./release.mjs";
 import { experimentReport, renderExperiment } from "./experiment.mjs";
 import { readExperiment } from "../experiment.mjs";
+import { compoundingReport, gatherCompounding, renderCompounding, DEFAULT_BUCKETS, DEFAULT_WINDOW_MS } from "./compounding.mjs";
 import { poolUtilisation, formatPoolLines } from "./pool.mjs";
 import { batchRoot } from "../lean/batch.mjs";
 
@@ -29,6 +30,7 @@ export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <ow
        forgedock stats --experiment <id> [--require-met] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --by version [--post-metrics <pr>] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --pool [--window 7d,30d] [--repo <owner/repo>] [--json]
+       forgedock stats --compounding [--window 30d] [--bucket 100,500] [--since <date>] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
 
 Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
@@ -84,6 +86,13 @@ Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues
                         their delta and full-lane cohorts matched per size (forge#4491)
   --offline             skip the GitHub crawl for merged issues (issue quality and acceptance print n/a)
 
+Compounding (forge#4696; GitHub + git of the current clone, no model):
+  --compounding         reintroduction rate (guarded fix reopened, or its lines blamed by a later guarded fix within 90 days),
+                        reopen-after-close rate, guard activity (guard_run, guard_broken, supersedes_accepted) and decision
+                        churn (decision_superseded, decision_violation), as a rolling window, per release and per lane-age
+                        bucket; n/a where no data is recorded. Pair with --efficiency --compare for $/merge and turns/merge
+  --bucket <list>       lane-age bucket upper bounds (default 100,500: lanes 1-100, 101-500, 501+)
+
 Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
   --tiers               merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
                         (mid, mid->top escalated, top) and routing class; --json for JSON
@@ -98,7 +107,7 @@ Lane profile baseline (forge#4436; local run logs only, every run of each issue,
 export function experimentExit(opts, report) { return opts.requireMet && report.targets_met !== true ? 1 : 0; }
 
 export function parseArgs(argv) {
-  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, by: null, experiment: null, emitEvent: false, requireMet: false, postMetrics: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
+  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, compounding: false, bucket: null, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, by: null, experiment: null, emitEvent: false, requireMet: false, postMetrics: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
   const val = (i, name) => {
     const v = argv[i + 1];
     if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
@@ -117,6 +126,8 @@ export function parseArgs(argv) {
     else if (name === "--context-ab") o.contextAb = true;
     else if (name === "--explore-ab") o.exploreAb = true;
     else if (name === "--tiers") o.tiers = true;
+    else if (name === "--compounding") o.compounding = true;
+    else if (name === "--bucket") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--bucket must be a comma list of lane counts"); const b = v.split(",").map(Number); if (b.some((x, i) => i && x <= b[i - 1])) throw new Error("--bucket must be increasing"); o.bucket = b; }
     else if (name === "--lane-profile") o.laneProfile = true;
     else if (name === "--issues") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--issues must be a comma list of issue numbers"); o.issues = v.split(",").map(Number); }
     else if (name === "--efficiency") o.efficiency = true;
@@ -144,7 +155,8 @@ export function parseArgs(argv) {
     else throw new Error(`unknown argument: ${a}`);
   }
   if (o.batch && !o.efficiency) throw new Error("--batch and --window need --efficiency");
-  if (o.windows && !o.efficiency && !o.pool) throw new Error("--batch and --window need --efficiency");
+  if (o.windows && !o.efficiency && !o.pool && !o.compounding) throw new Error("--batch and --window need --efficiency");
+  if (o.bucket && !o.compounding) throw new Error("--bucket needs --compounding");
   if ((o.compare || o.offline) && !o.efficiency) throw new Error("--compare and --offline need --efficiency");
   if (o.postMetrics && !o.by) throw new Error("--post-metrics needs --by version");
   return o;
@@ -302,6 +314,12 @@ export async function buildReport(opts, deps = {}) {
     return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache } }, query };
   }
   const model = toModel(repo, gh);
+  if (opts.compounding) { // forge#4696: GitHub records + guards + git, no model
+    const inputs = await (deps.gatherCompounding ?? gatherCompounding)({ repo, gh: deps.gh ?? defaultGh });
+    const query = { compounding: compoundingReport({ ...inputs, runs: model.runs, events: model.events, windowMs: opts.windows?.[0] ?? DEFAULT_WINDOW_MS,
+      buckets: opts.bucket ?? DEFAULT_BUCKETS, now: deps.now?.() ?? Date.now(), minN: opts.minN }) };
+    return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache } }, query };
+  }
   const ghKeys = new Set(model.runs.map((r) => `${r.repo}#${r.issue}`));
   let local = [];
   if (opts.local) {
@@ -348,6 +366,7 @@ export async function runStats(argv, deps = {}) {
     if (report.query?.experiment) { process.stdout.write(`${opts.json ? renderJson(report) : renderExperiment(report.query.experiment)}\n`); return experimentExit(opts, report.query.experiment); }
     if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
     if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
+    if (report.query?.compounding && !opts.json) { process.stdout.write(`${renderCompounding(report.query.compounding)}\n`); return 0; }
     if (report.query?.pool && !opts.json) { process.stdout.write(`${formatPoolLines(report.query.pool).join("\n")}\n`); return 0; }
     if (report.query?.efficiency && !opts.json) {
       const e = report.query.efficiency;

package/commands/bootstrap.md

@@ -18,18 +18,20 @@ BOOT_TPL="$(forge_root)/templates/bootstrap"; VALIDATE=$(forge_resolve bootstrap
 ```
 
 1. **Read the input.** Use the free text, or read each doc at the given path(s).
-2. **Interview** from `interview.md`. Cover every aspect it lists, briefly and conversationally. Skip what the input already answers.
-3. **Fill the blanks.** Give every unanswered aspect a sensible default with a one-line rationale tied to what the user said, using the closest `stack-profiles.md` preset. Show them all in ONE editable "Here's what I assumed" list and apply the user's edits.
-4. **Ask only genuine doubts** (they change architecture or cost: native vs PWA, data residency, self-host vs managed DB, product name, repo owner). Ask with AskUserQuestion, at most 4 multiple-choice questions per batch, each with a recommended option. Everything else stays a default. Headless (no AskUserQuestion, e.g. `npx forgedock bootstrap`): take each recommended option and list it under "Here's what I assumed".
+2. **Interview** from `interview.md`. Cover each aspect briefly; skip what the input answers.
+3. **Fill the blanks.** Default every unanswered aspect with a one-line rationale, using the closest `stack-profiles.md` preset. Show them in ONE editable "Here's what I assumed" list and apply the user's edits.
+4. **Ask only genuine doubts** (architecture or cost: native vs PWA, data residency, self-host vs managed DB, product name, repo owner). Ask with AskUserQuestion, at most 4 multiple-choice questions per batch, each with a recommended option. Everything else stays a default. Headless (no AskUserQuestion, e.g. `npx forgedock bootstrap`): take each recommended option and list it under "Here's what I assumed".
 5. **Spike doc.** Write `technical-spike.md` to `--spike` (default `docs/engineering/technical-spike.md`); set the plan's `spikeDoc` to that path. Fill every section and give each decision an ADR-style `**Rationale**:` line.
 6. **Plan JSON** in `$TMPDIR` (shape: header of `bootstrap-validate.mjs` and `bootstrap-create.mjs`):
-   - Milestones in build order, foundation first.
+   - Milestones as dependency layers, foundation first: each depends only on earlier ones (#4659).
+   - A thin deployable slice (CI deploy workflow, staging env, seed) in milestone 1 or 2, never last.
    - A `SPIKE` parent issue.
    - Sub-issues from `issue-body.md` with planned paths, `[type:*]` tags, `priority` and `component`.
    - `"ui": true` issues carry the full design spec. `"hero": true` issues route to `/design`.
-   - One `needs-human-issue.md` issue per credential, account or external input. Code issues reference it as `#{{KEY}}` but never depend on it, and name the mock that keeps dev and CI green.
+   - One `needs-human-issue.md` issue per credential, account or external input, with `gates:` (#4660). Code issues reference it as `#{{KEY}}`, never depend on it, and name the mock that keeps dev and CI green.
    - Sub-issues point at the spike via `**Parent**` only, never `Depends on`.
    - `guardrails` and `domainRules` from the interview.
-7. **Validate.** Run `node "$VALIDATE" <plan>`. Fix the plan until it exits 0. Never create from an invalid plan.
-8. **Create.** Run `node "$CREATE" <plan>`, adding `--dry-run` when the input has it (preview only, GitHub untouched). The script reuses existing issues by title and parent, so on an error re-run the same command to resume.
-9. **Hand off.** Print the script's hand-off verbatim: what was created (links), what needs a human (links), and the `/orchestrate` command per milestone, in order.
+7. **Validate.** Run `node "$VALIDATE" <plan>`. Fix the plan until it exits 0. Heed `deploy-last` warnings. Never create from an invalid plan.
+8. **Create.** Run `node "$CREATE" <plan>`, adding `--dry-run` when the input has it (preview only, GitHub untouched). It reuses existing issues, so on an error re-run it to resume.
+9. **Hand off.** Print the script's hand-off verbatim: created (links), needs a human (links), and the `/orchestrate` command per milestone, in order.
+10. Later: `npx forgedock plan validate --live` (#4659).

package/commands/deploy-info.md

@@ -325,6 +325,7 @@ else
   # Step 3: Count commits queued (commits in staging not yet in main)
   QUEUED_COMMITS=$(git rev-list --count origin/{DEFAULT_BRANCH}..origin/{STAGING_BRANCH} 2>/dev/null || echo "?")
 fi
+INTEGRATION_RED=$(gh issue list {GH_FLAG} --label integration-red --json number,title --jq '.[]|"#\(.number) \(.title)"') # forge#4667
 ```
 
 Train state is included in the Phase 5 output summary.
@@ -339,10 +340,12 @@ Train state is included in the Phase 5 output summary.
 **Commits**: {N} | **PRs**: {N} | **Issues closed**: {N} | **Files changed**: {N}
 
 ### Deploy Train State
+**Gate**: `node ${FORGEDOCK_HOME:-${CLAUDE_PLUGIN_ROOT}}/scripts/readiness.mjs --target {preview|staging|prod}`; NOT DEPLOYABLE (open prerequisites) means never call the target deployable (#4660).
 **Train PR**: #{TRAIN_PR_NUMBER} (SHA: {TRAIN_PR_SHA}, opened: {TRAIN_PR_CREATED})
 **Status**: {TRAIN_STATUS}
 {If HELD: "**Held by**: {count} open finding(s) — fix and push to unblock:\n{HELD_BY}"}
 {If CLEAR: "**Ready to merge**: no open findings on the train PR"}
+{If INTEGRATION_RED: "**Ship-gate risk**: red integration run:\n{INTEGRATION_RED}"}
 {If NO_TRAIN: "No open staging→main PR — run /review-pr-staging to open one"}
 **Queued commits**: {QUEUED_COMMITS} (commits in staging not yet in main)
 

package/commands/issue.md

@@ -85,6 +85,8 @@ Duplicates: the script's token-overlap dedup is authoritative. Before drafting,
 
 ### 3. Draft
 
+**External prerequisite** (credential, account, sign-off): label `needs-human:external` and put `gates: preview|staging-deploy|prod-deploy|feature-flag:<name>` and `owner:` in the body (#4660).
+
 **Title**: `{prefix}({scope}): {specific description}`, max 80 characters, no vague verbs ("improve", "update", "handle") without specifics.
 
 **Body** (pipeline template; `## Problem`, `## Affected Files` and `## Acceptance Criteria` are mandatory, the rest optional; in greenfield mode Affected Files may list *planned* paths inside the spike's declared repo layout):

package/commands/milestone.md

@@ -96,6 +96,8 @@ Parse `$ARGUMENTS` to determine which action to take:
 | `drift <slug>` | → **Milestone Drift** |
 | `close <slug>` | → **Close Milestone** |
 
+Unattended multi-milestone runs use the CLI, not this spec: `forgedock milestone train <slug...> [--parallel-milestones N]` (resume with `--resume <id>`, add a milestone with `--insert <id> <slug>`). It keeps one state file, reuses the lifecycle above and never edits `forge.yaml`. See `devdocs/decisions/4658-milestone-train.md`.
+
 ---
 
 ## Action: Create Milestone
@@ -356,6 +358,10 @@ else
 fi
 ```
 
+### Step 6C: Re-validate dependency order
+
+After a milestone is added, the train order changes or an issue is decomposed, run `npx forgedock plan validate --live` (#4659). Fix every `invalid` edge (a dependency in a later milestone) before continuing; `held` edges are reported only.
+
 ### Step 7: Report
 
 ```
@@ -444,37 +450,28 @@ git fetch origin milestone/{slug}
 OPEN_COUNT=$(gh issue list --milestone "{TITLE}" --state open --json number --jq '. | length')
 ```
 
-If there are open issues, warn the user:
+If there are open issues, this is a hard gate (interactive and unattended; unattended runs fail instead of asking). Every open issue must be **carried over** with a target milestone *and* a class (`engineering|external|post-deploy|wont-do`), or closed first. The class comes from labels (`carry:<class>`, `needs-human:external`, `awaiting-promotion|awaiting-precondition|awaiting-measurement`, `wont-do`; else `engineering`); the target is a `carry-to:<milestone title>` label or the user's answer. Run the check with `preflightCarryOver` (`bin/lean/lifecycle.mjs`):
+
 ```
-⚠ Milestone "{TITLE}" still has {OPEN_COUNT} open issues:
-{list them}
+⚠ Milestone "{TITLE}" still has {OPEN_COUNT} open issues without a carry-over target:
+{list them with their class}
 
-Ship anyway? Open issues will remain assigned to the milestone.
+Name a target milestone for each (or close it). Ship is blocked until every open issue has one.
 ```
 
-Wait for confirmation.
+An issue that is `needs-human:external`, flag/legal-gated (`flag-gated`, `legal-gated`) or depends on an issue that is not on the target base is **never** demoted to fast lane: it is carried to its target milestone.
 
 ### Step 2: Sync milestone branch with staging
 
-Before shipping, ensure the milestone branch is up to date with `staging` (picks up any fast-lane fixes that accumulated since the milestone branch was created):
+Before shipping, bring the milestone branch up to date with `staging` (picks up fast-lane fixes that accumulated since it was cut). `forgedock milestone-sync` does it deterministically (#4663): a milestone already `shipped:*`/`deployed:default`/`closed` is refused, a branch already containing staging is a no-op, append-only registry conflicts are resolved by the lane base-merge resolver (`lean.base_merge_*`), and a protected branch (ruleset or branch protection, or a push the rules reject) is synced through the single `sync/milestone-{slug}` PR with auto-merge armed (merged directly when CLEAN). A `ship` sync is never debounced.
 
 ```bash
 cd {REPO_PATH}
-git fetch origin {STAGING_BRANCH} milestone/{slug}
-# Never mutate the main checkout: sync in a detached worktree at the milestone tip
-SYNC_WT=$(mktemp -d)
-git worktree add --detach "$SYNC_WT" origin/milestone/{slug}
-if git -C "$SYNC_WT" merge origin/{STAGING_BRANCH} --no-edit; then
-  git -C "$SYNC_WT" push origin HEAD:refs/heads/milestone/{slug}
-  git worktree remove --force "$SYNC_WT"
-else
-  # Conflicts: report to user and STOP — do not auto-resolve
-  git -C "$SYNC_WT" merge --abort 2>/dev/null || true
-  git worktree remove --force "$SYNC_WT"
-  echo "STOP: merge conflict syncing milestone/{slug} with {STAGING_BRANCH}"; exit 1
-fi
+forgedock milestone-sync {slug} --reason ship --base {STAGING_BRANCH} --repo {owner}/{repo} || { echo "STOP: milestone sync failed or conflicts"; exit 1; }
 ```
 
+If the sync went through a PR (`pr-created`/`pr-updated`), wait for it to merge before Step 2.5.
+
 ### Step 2.5: Pre-Merge Hunk-Loss Audit (MANDATORY before creating PR)
 
 **WHY THIS EXISTS**: When a long-lived milestone branch is merged into staging, git hunks that exist in staging but were NOT touched by the milestone can be silently dropped. This happened in a 20h window that produced 13 regression-fix PRs (27% of total pipeline throughput) restoring lost code: postgres tuning params, completion_refund sweep, credit_ledger columns, browser flags, pool guards, Redis config, crawl TTL refresh, challenge_strategy guard. Each regression required investigation → new issue → new PR → review → merge — consuming nearly a third of pipeline capacity on damage repair instead of forward progress.
@@ -730,24 +727,28 @@ Run /milestone ship {slug} again when the milestone is ready for another shippin
 ```bash
 gh pr merge {PR_NUMBER} --merge
 
-# Step 6A: Promote remaining open issues to fast lane
-# Query open issues still on this milestone (per_page=100 to avoid silent truncation beyond default 30)
-OPEN_ISSUES=$(gh api "repos/{owner}/{repo}/issues?milestone={MS_NUMBER}&state=open&per_page=100" \
-  --jq '.[].number')
+# Step 6A: Carry over remaining open issues (forge#4657). Never clear the milestone: every open issue moves to its
+# target milestone (Step 1 gate) and keeps its class label. Only an issue whose preflightCarryOver entry has demote=true
+# and whose target is "fast lane" may lose its milestone; external, gated and off-base-dependent issues are never demoted.
+OPEN_ISSUES=$(gh api "repos/{owner}/{repo}/issues?milestone={MS_NUMBER}&state=open&per_page=100" --jq '.[].number')
 OPEN_COUNT=$(echo "$OPEN_ISSUES" | grep -c '[0-9]' 2>/dev/null || echo 0)
-
-if [ "$OPEN_COUNT" -gt 0 ]; then
-  echo "Promoting $OPEN_COUNT open issue(s) to fast lane (removing milestone assignment)..."
-  for ISSUE_NUM in $OPEN_ISSUES; do
-    gh api "repos/{owner}/{repo}/issues/$ISSUE_NUM" -X PATCH --field milestone=null
-    echo "  Demoted #$ISSUE_NUM → fast lane"
-  done
-else
-  echo "No open issues remain on the milestone — nothing to demote."
-fi
-
-# Step 6B: Close the milestone
-gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field state="closed"
+for ISSUE_NUM in $OPEN_ISSUES; do
+  gh issue edit "$ISSUE_NUM" --milestone "{TARGET_TITLE_OF_ISSUE}" --add-label "carry:{CLASS_OF_ISSUE}"  # <!-- allowlist:check-command-side-effects -->
+  echo "  Carried #$ISSUE_NUM -> {TARGET_TITLE_OF_ISSUE} ({CLASS_OF_ISSUE})"
+done
+
+# Step 6B: Record the lifecycle. The merge is on staging, not yet on the default branch: write
+# a FORGE:LIFECYCLE record ({"v":1,"state":"shipped:staging","sha":"<merge commit>"}) in the milestone description and
+# close the milestone only when forge.yaml `milestones.close_on` (default `default`) is reached. With close_on=default the
+# milestone stays open until the staging -> default merge is observed (deployed:default), then it closes.
+MERGE_SHA=$(gh pr view {PR_NUMBER} --json mergeCommit --jq '.mergeCommit.oid')
+DESC=$(gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --jq '.description // ""')
+TAG="FORGE:LIFECYCLE"
+BLOCK="<!-- $TAG {\"v\":1,\"state\":\"shipped:staging\",\"sha\":\"$MERGE_SHA\"} -->"
+gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field description="$(printf '%s\n\n%s' "$(printf '%s' "$DESC" | sed "/$TAG/d")" "$BLOCK")"
+# close_on=staging only (and only when no issue is left open on the milestone):
+# gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field state="closed"
+# Orchestrate-driven runners get the same transition from bin/lean/lifecycle.mjs advanceMilestone at batch end.
 
 # Step 6C: Delete the milestone branch
 git push origin --delete milestone/{slug}
@@ -758,8 +759,8 @@ Report:
 ```
 ## Shipped: {MILESTONE_TITLE}
 
-Merged to staging. Milestone branch deleted.
-{If OPEN_COUNT > 0: "{OPEN_COUNT} open issue(s) moved to fast lane: #{list}. They will target staging on next /work-on run."}
+Merged to staging (lifecycle: shipped:staging). Milestone branch deleted. Milestone closes at `milestones.close_on` ({close_on}).
+{If OPEN_COUNT > 0: "{OPEN_COUNT} open issue(s) carried over: #{list} -> {target} ({class}). None were demoted to fast lane."}
 {If OPEN_COUNT == 0: "No open issues remaining — clean close."}
 
 To deploy: merge `staging → main` via GitHub web UI when ready.
@@ -769,27 +770,15 @@ To deploy: merge `staging → main` via GitHub web UI when ready.
 
 ## Action: Sync Milestone Branch
 
-Syncs the milestone branch with the latest default branch to pick up fast-lane fixes. Run periodically on long-lived milestones.
+Syncs the milestone branch with the latest default branch to pick up fast-lane fixes. Sync **on demand only**: before ship, before a batch, or when a conflict is expected. Never on every base move, and never a milestone whose lifecycle is `shipped:*` (the command refuses it; a shipped milestone is also never the "active" one to sync, `isSyncable` in `bin/lean/milestonesync.mjs`).
 
 ```bash
 cd {REPO_PATH}
 DEFAULT_BRANCH=$(yq '.branches.default' "$CONFIG_FILE")
-git fetch origin $DEFAULT_BRANCH milestone/{slug}
-# Never mutate the main checkout: sync in a detached worktree at the milestone tip
-SYNC_WT=$(mktemp -d)
-git worktree add --detach "$SYNC_WT" origin/milestone/{slug}
-if git -C "$SYNC_WT" merge origin/$DEFAULT_BRANCH --no-edit; then
-  git -C "$SYNC_WT" push origin HEAD:refs/heads/milestone/{slug}
-  git worktree remove --force "$SYNC_WT"
-else
-  # If conflicts, report to user and STOP
-  git -C "$SYNC_WT" merge --abort 2>/dev/null || true
-  git worktree remove --force "$SYNC_WT"
-  echo "STOP: merge conflict syncing milestone/{slug} with $DEFAULT_BRANCH"; exit 1
-fi
+forgedock milestone-sync {slug} --reason manual --base "$DEFAULT_BRANCH" --repo {owner}/{repo}
 ```
 
-Report: "Synced milestone/{slug} with latest `{DEFAULT_BRANCH}`. {N} new commits incorporated."
+Protected branches get ONE sync PR (`sync/milestone-{slug}`) whose head is force-pushed in place; a base that moved again inside `milestones.sync_debounce` (default 10m) reports `debounced` and leaves the PR (and its CI) alone. Report the printed status.
 
 ---
 
@@ -828,5 +817,5 @@ This automatically makes the issue a feature-lane issue — next time `/work-on`
 
 - **Milestone already exists**: Check first with `gh api repos/{owner}/{repo}/milestones --jq '.[] | select(.title == "{TITLE}")'`
 - **Branch already exists**: Reuse it, don't recreate
-- **Merge conflicts on sync/ship**: Report to user, do NOT auto-resolve. List conflicting files.
+- **Merge conflicts on sync/ship**: append-only registry conflicts matching `lean.base_merge_*` are resolved deterministically by `milestone-sync`; any other conflict is reported with its files and the branch is left untouched. Do NOT hand-resolve.
 - **No issues in milestone**: Warn user — empty milestones are valid but unusual

package/commands/orchestrate/phase-4-execution.md

@@ -1581,6 +1581,8 @@ The merged-trail hold on #${N} was released by a human (needs-human cleared or t
   return 0
 }
 
+# gh failures are unknown, not state (forge#4662): in every hold/settle/ship read, a failed gh call (HTTP 401, 5xx, timeout) is never
+# "not closed" or "no PR". Keep the previous classification, back off, and after 5 consecutive failures post one alert and stop that loop.
 # classify_predecessor_state <issue> (forge#3168): the helpers hold_merged_trail, resolve_orch_login,
 # trail_escalation_state and release_merged_trail above MUST be declared in the SAME shell before this function runs
 # (reverify_merged_trail is nested below). It has exactly ONE guarded write (the DRY_RUN-guarded hold_merged_trail

package/commands/orchestrate.md

@@ -105,7 +105,7 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
   for them. After a decision (a `FORGE:DECISION` posted and `needs-human` removed), a blocked issue that already ended in the batch
   (`needs-human`, `invalid`, `skipped`, `dependency-*`, `ci-pending`) returns to it with
   `forgedock orchestrate --requeue <id> N... --reason "..."` (refused while it is labelled `needs-human` or closed; skipped dependents
-  return to pending with it; a finished batch is reopened); `--add` skips such an issue as `already in batch`. Its pre-filter still closes already-fixed ones. The runner requeues `deferred` lanes by code (host busy and a full sandbox pool after their
+  return to pending with it; a finished batch is reopened); `--add` on a `skipped` issue re-queues it (other ended issues still report `already in batch`). A dependency-skipped issue shows as `waiting on #X` while the batch runs (the blocker is re-checked each pass; merged or closed re-admits it, closed as not planned keeps it skipped) and ends `skipped` only if the blocker never resolved. Its pre-filter still closes already-fixed ones. The runner requeues `deferred` lanes by code (host busy and a full sandbox pool after their
   retry-in, uncapped, a usage limit holds admissions for that account (backend identity) until the reset, across every batch on
   the host; `forgedock limits show|clear <id>|--all` lists or drops holds; transient merge/push/idle failures back off; other
   deferrals stop after `orchestration.max_deferrals`), relaunches an `engine-error` once, and stops admitting after
@@ -193,6 +193,13 @@ Escalate only these:
 Collect escalations into one question list and ask it once, at the end of the batch. Never ask per issue, and keep every
 other lane running meanwhile.
 
+## 3b. Settle scope (#4665)
+
+When settle begins, freeze the milestone's issue set (`scripts/settle-scope.mjs` `freezeScope`). Issues filed later are
+classified by `classifyNew`: decomposition children of a frozen issue stay in the milestone and join the measured set;
+deferred follow-ups go to the next milestone or the backlog; review findings and others count as growth only.
+`convergence` measures the frozen set plus those children; a STOP prints `formatStopReport` (growth and sources).
+
 ## 4. Report
 
 When all lanes have finished, print one table: issue, outcome, PR, minutes (issues the pre-filter closed