forgedock 1.35.0 → 1.37.0

package/bin/stats/experiment.mjs

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * forge#4629: `forgedock stats --experiment <id>`. Finished runs tagged with the experiment id, grouped by arm; each metric
+ * is null below the declared min n (release.mjs rule), a target is null when a value it needs is null, never a pass.
+ */
+import { aggregate } from "./aggregate.mjs";
+import { releaseMetrics } from "./release.mjs";
+import { benchRecord } from "../experiment.mjs";
+
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+
+/** One target against the per-arm metrics: absolute limit, or limit x the other arm's value. */
+export function evalTarget(t, byArm) {
+  const value = byArm[t.arm]?.[t.metric], base = t.vs ? byArm[t.vs]?.[t.metric] : 1;
+  const limit = isNum(base) ? t.limit * base : null;
+  const pass = isNum(value) && isNum(limit) ? (t.op === "<=" ? value <= limit : value >= limit) : null;
+  return { pass, value: isNum(value) ? value : null, limit };
+}
+
+export function experimentReport(runs, exp) {
+  const mine = runs.filter((r) => r.finished && r.experiment === exp.id);
+  const arms = {}, byArm = {};
+  for (const arm of exp.arms) {
+    const rs = mine.filter((r) => r.arm === arm).map((r) => ({ ...r, version: "0.0.0" })); // one group per arm
+    const g = aggregate({ runs: rs }, { minN: exp.minN }).groups[0];
+    const m = g ? releaseMetrics([g], { minN: exp.minN }).versions[0] : null;
+    byArm[arm] = Object.fromEntries(exp.metrics.map((k) => [k, m?.[k] ?? null]));
+    arms[arm] = { n: rs.length, ...byArm[arm] };
+  }
+  const targets = Object.fromEntries(exp.targets.map((t) => [t.text, evalTarget(t, byArm)]));
+  const verdicts = Object.values(targets);
+  return { id: exp.id, minN: exp.minN, ...benchRecord({ source: "stats", arms, targets }),
+    targets_met: verdicts.length > 0 && verdicts.every((v) => v.pass === true) };
+}
+
+const fmt = (v) => (v == null ? "n/a" : Number.isInteger(v) ? String(v) : v.toFixed(2));
+export function renderExperiment(r) {
+  const keys = Object.keys(Object.values(r.arms)[0] ?? {}).filter((k) => k !== "n");
+  const line = (c) => `| ${c.join(" | ")} |`;
+  const rows = Object.entries(r.arms).map(([a, v]) => line([a, String(v.n), ...keys.map((k) => fmt(v[k]))]));
+  const tg = Object.entries(r.targets).map(([t, v]) => `${t}: ${v.pass == null ? "n/a" : v.pass ? "met" : "missed"} (${fmt(v.value)} vs ${fmt(v.limit)})`);
+  return [`Experiment ${r.id} (min n ${r.minN}; n/a below it)`, line(["arm", "n", ...keys]), line(["---", "---", ...keys.map(() => "---")]), ...rows, ...tg, `targets met: ${r.targets_met}`].join("\n");
+}

package/bin/stats/github-source.mjs

@@ -77,9 +77,10 @@ export function runsFromRecords(records) {
     const key = `${r.repo}#${r.issue}#${r.run ?? ""}`;
     const run = byRun.get(key) ?? { source: "github", repo: r.repo ?? null, version: r.forge_version ?? null, issue: r.issue, run: r.run ?? null,
       finished: false, hydrated: false, usageEvents: 0, reason: null, code: null, remediated: false, startTs: null, endTs: null, wallMs: null,
-      phases: {}, builderAttempts: [], _first: null };
+      phases: {}, builderAttempts: [], experiment: null, arm: null, _first: null };
     byRun.set(key, run);
     if (!run._first || r.ts < run._first) run._first = r.ts;
+    if (r.experiment != null && r.arm != null) { run.experiment = String(r.experiment); run.arm = String(r.arm); } // #4629
     if (r.event === "run_terminal") {
       run.finished = true; run.reason = r.reason ?? null; run.code = r.code ?? null; run.endTs = r.ts ?? null; run.wallMs = num(r.wall_ms);
       continue;

package/bin/stats/index.mjs

@@ -16,6 +16,10 @@ import { parseForgeYaml } from "../forge-utils.mjs";
 import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
 import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
 import { batchEfficiency, windowEfficiency, compareEfficiency, formatEfficiencyLines, formatCompareLines, parseWindows } from "./efficiency.mjs";
+import { releaseMetrics, renderReleaseMetrics, metricsComment, postMetrics } from "./release.mjs";
+import { experimentReport, renderExperiment } from "./experiment.mjs";
+import { readExperiment } from "../experiment.mjs";
+import { compoundingReport, gatherCompounding, renderCompounding, DEFAULT_BUCKETS, DEFAULT_WINDOW_MS } from "./compounding.mjs";
 import { poolUtilisation, formatPoolLines } from "./pool.mjs";
 import { batchRoot } from "../lean/batch.mjs";
 
@@ -23,7 +27,10 @@ const here = dirname(fileURLToPath(import.meta.url));
 
 export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
                       [--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
+       forgedock stats --experiment <id> [--require-met] [--min-n N] [--repo <owner/repo>] [--json]
+       forgedock stats --by version [--post-metrics <pr>] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --pool [--window 7d,30d] [--repo <owner/repo>] [--json]
+       forgedock stats --compounding [--window 30d] [--bucket 100,500] [--since <date>] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
 
 Version-over-version report: Cost, Time, Flow, Review funnel, Gates and Issue flow per (version, repo), with a
@@ -42,6 +49,14 @@ delta against the previous version and a low-n flag for samples under 10. Missin
   --release-times FILE  "<version> <epoch-seconds>" lines used to estimate versions of unversioned records
                         (default: git tags v* of the ForgeDock checkout)
 
+Per-release regression rule (forge#4628):
+  --require-met         with --experiment: exit 1 unless every target is met (#4634; for a named post-deploy check)
+  --experiment <id>     arms of the forge.yaml experiment: section (arms, ratio, metrics, targets, min_n) against its targets; n/a below min n
+  --emit-event          with --experiment: emit one experiment.verdict event (met or missed; docs/EVENTS.md)
+  --by version          median cost per run, wall p50, first-pass rate and needs-human rate of each release against the
+                        previous one; a metric under --min-n runs is null and its verdict null, never a pass
+  --post-metrics <pr>   post (or update) the <!-- FORGE:METRICS v1 --> snapshot comment on that release PR
+
 ALERTS (forge#4354; local run logs only, skipped with --no-local): fail-soft rates over the last 50 lanes of the repo
 (DEPS_PREPARED failed per ecosystem, deferred-to-ci, repeated TOOLCHAIN_WARNING, outcome codes whose share doubled);
 thresholds under forge.yaml stats.alerts.*. Nothing is printed when none fired or under 10 lanes.
@@ -71,6 +86,13 @@ Efficiency (forge#4402; local batch dirs and run logs, gh only for merged issues
                         their delta and full-lane cohorts matched per size (forge#4491)
   --offline             skip the GitHub crawl for merged issues (issue quality and acceptance print n/a)
 
+Compounding (forge#4696; GitHub + git of the current clone, no model):
+  --compounding         reintroduction rate (guarded fix reopened, or its lines blamed by a later guarded fix within 90 days),
+                        reopen-after-close rate, guard activity (guard_run, guard_broken, supersedes_accepted) and decision
+                        churn (decision_superseded, decision_violation), as a rolling window, per release and per lane-age
+                        bucket; n/a where no data is recorded. Pair with --efficiency --compare for $/merge and turns/merge
+  --bucket <list>       lane-age bucket upper bounds (default 100,500: lanes 1-100, 101-500, 501+)
+
 Builder model tiers (forge#4394; local run logs only, no GitHub crawl):
   --tiers               merge, conformance and post-merge-rework rates, first-pass yield and cost per builder tier
                         (mid, mid->top escalated, top) and routing class; --json for JSON
@@ -81,8 +103,11 @@ Lane profile baseline (forge#4436; local run logs only, every run of each issue,
   --issues <list>       pin the report to these issues (comma list), for a before/after on the same issue set
 `;
 
+/** #4634: 0, or 1 when --require-met and the experiment's targets are not all met (a named post-deploy check gates on it). */
+export function experimentExit(opts, report) { return opts.requireMet && report.targets_met !== true ? 1 : 0; }
+
 export function parseArgs(argv) {
-  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
+  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, compounding: false, bucket: null, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, by: null, experiment: null, emitEvent: false, requireMet: false, postMetrics: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
   const val = (i, name) => {
     const v = argv[i + 1];
     if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
@@ -101,6 +126,8 @@ export function parseArgs(argv) {
     else if (name === "--context-ab") o.contextAb = true;
     else if (name === "--explore-ab") o.exploreAb = true;
     else if (name === "--tiers") o.tiers = true;
+    else if (name === "--compounding") o.compounding = true;
+    else if (name === "--bucket") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--bucket must be a comma list of lane counts"); const b = v.split(",").map(Number); if (b.some((x, i) => i && x <= b[i - 1])) throw new Error("--bucket must be increasing"); o.bucket = b; }
     else if (name === "--lane-profile") o.laneProfile = true;
     else if (name === "--issues") { const v = take(); if (!/^[1-9]\d*(,[1-9]\d*)*$/.test(v)) throw new Error("--issues must be a comma list of issue numbers"); o.issues = v.split(",").map(Number); }
     else if (name === "--efficiency") o.efficiency = true;
@@ -112,8 +139,13 @@ export function parseArgs(argv) {
       try { if (!v || !Object.keys(parseSince(v)).length) throw bad(); } catch { throw bad(); }
       o.compare = v;
     }
+    else if (name === "--by") { const v = take(); if (v !== "version") throw new Error("--by must be version"); o.by = v; }
+    else if (name === "--experiment") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error("--experiment must be an experiment id"); o.experiment = v; }
+    else if (name === "--emit-event") o.emitEvent = true;
+    else if (name === "--post-metrics") { const v = take(); if (!/^[1-9]\d*$/.test(v)) throw new Error("--post-metrics must be a PR number"); o.postMetrics = Number(v); }
+    else if (name === "--require-met") o.requireMet = true;
     else if (name === "--offline") o.offline = true;
-    else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
+    else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); o.minNSet = true; }
     else if (name === "--json") o.json = true;
     else if (name === "--refresh") o.refresh = true;
     else if (name === "--compact") o.compact = true;
@@ -123,8 +155,10 @@ export function parseArgs(argv) {
     else throw new Error(`unknown argument: ${a}`);
   }
   if (o.batch && !o.efficiency) throw new Error("--batch and --window need --efficiency");
-  if (o.windows && !o.efficiency && !o.pool) throw new Error("--batch and --window need --efficiency");
+  if (o.windows && !o.efficiency && !o.pool && !o.compounding) throw new Error("--batch and --window need --efficiency");
+  if (o.bucket && !o.compounding) throw new Error("--bucket needs --compounding");
   if ((o.compare || o.offline) && !o.efficiency) throw new Error("--compare and --offline need --efficiency");
+  if (o.postMetrics && !o.by) throw new Error("--post-metrics needs --by version");
   return o;
 }
 
@@ -280,6 +314,12 @@ export async function buildReport(opts, deps = {}) {
     return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache } }, query };
   }
   const model = toModel(repo, gh);
+  if (opts.compounding) { // forge#4696: GitHub records + guards + git, no model
+    const inputs = await (deps.gatherCompounding ?? gatherCompounding)({ repo, gh: deps.gh ?? defaultGh });
+    const query = { compounding: compoundingReport({ ...inputs, runs: model.runs, events: model.events, windowMs: opts.windows?.[0] ?? DEFAULT_WINDOW_MS,
+      buckets: opts.bucket ?? DEFAULT_BUCKETS, now: deps.now?.() ?? Date.now(), minN: opts.minN }) };
+    return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache } }, query };
+  }
   const ghKeys = new Set(model.runs.map((r) => `${r.repo}#${r.issue}`));
   let local = [];
   if (opts.local) {
@@ -287,6 +327,12 @@ export async function buildReport(opts, deps = {}) {
   }
   const report = aggregate({ runs: [...model.runs, ...local], events: model.events, issues: model.issues },
     { minN: opts.minN, releaseTimes, ...parseSince(opts.since) });
+  if (opts.experiment) { // forge#4629
+    const exp = deps.experiment ?? readExperiment();
+    if (!exp || exp.id !== opts.experiment) throw new Error(`experiment ${opts.experiment} is not declared in forge.yaml (experiment: id, arms, ...)`);
+    return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, query: { experiment: experimentReport([...model.runs, ...local], { ...exp, minN: opts.minNSet ? opts.minN : exp.minN }) } };
+  }
+  if (opts.by) return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, query: { release_metrics: releaseMetrics(report.groups, { minN: opts.minN, repo }) } };
   // forge#4354: fail-soft alerts from the local run logs only (no gh call)
   const alerts = opts.local ? computeAlerts(deps.localLanes ?? readLanes(runsDir(env)), { repo, ...(deps.alertConfig ?? readAlertConfig()) }) : [];
   return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report, alerts };
@@ -299,6 +345,10 @@ export async function runStats(argv, deps = {}) {
     return 2;
   }
   if (opts.help) { process.stdout.write(HELP); return 0; }
+  if (opts.requireMet && !opts.experiment) {
+    process.stderr.write(`forgedock stats: --require-met needs --experiment <id>\n`);
+    return 2;
+  }
   opts.repo = resolveRepo(opts.repo);
   if (!opts.repo || !isValidRepo(opts.repo)) {
     process.stderr.write(`forgedock stats: could not resolve a valid owner/repo (pass --repo owner/repo)\n`);
@@ -306,8 +356,17 @@ export async function runStats(argv, deps = {}) {
   }
   try {
     const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
+    if (report.query?.release_metrics) {
+      const rm = report.query.release_metrics;
+      if (opts.postMetrics) await postMetrics({ gh: deps.gh ?? defaultGh, repo: opts.repo, pr: opts.postMetrics, body: metricsComment(rm) });
+      process.stdout.write(`${opts.json ? renderJson(report) : renderReleaseMetrics(rm)}\n`);
+      return 0;
+    }
+    if (report.query?.experiment && opts.emitEvent) await (deps.emitExperiment ?? (async (r, x) => (await import("../engine-cli.mjs")).experimentEventSink(r, x)))(opts.repo, report.query.experiment);
+    if (report.query?.experiment) { process.stdout.write(`${opts.json ? renderJson(report) : renderExperiment(report.query.experiment)}\n`); return experimentExit(opts, report.query.experiment); }
     if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
     if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
+    if (report.query?.compounding && !opts.json) { process.stdout.write(`${renderCompounding(report.query.compounding)}\n`); return 0; }
     if (report.query?.pool && !opts.json) { process.stdout.write(`${formatPoolLines(report.query.pool).join("\n")}\n`); return 0; }
     if (report.query?.efficiency && !opts.json) {
       const e = report.query.efficiency;

package/bin/stats/release.mjs

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * forge#4628: per-release regression rule over the `aggregate()` groups. Each metric is null when its own sample is
+ * below minN; a verdict is null when either side is null. There is no "pass" value, only regressed|improved|flat|null.
+ */
+import { DEFAULT_MIN_N, UNKNOWN, compareVersions } from "./aggregate.mjs";
+
+export const METRICS_MARKER = "<!-- FORGE:METRICS v1 -->";
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+// lower is better for all but the first-pass rate
+const METRICS = [
+  { key: "usd_median", label: "median cost/run", lowerBetter: true, flat: 0.05 },
+  { key: "wall_p50_ms", label: "wall p50", lowerBetter: true, flat: 0.05 },
+  { key: "first_pass_rate", label: "first-pass rate", lowerBetter: false, flat: 0.02 },
+  { key: "needs_human_rate", label: "needs-human rate", lowerBetter: true, flat: 0.02 },
+];
+
+/** The metric values of one group, each null below minN (its own n). */
+function valuesOf(g, minN) {
+  const pick = (n, v) => (isNum(n) && n >= minN && isNum(v) ? v : null);
+  return {
+    n: g.n,
+    usd_median: pick(g.cost?.usdN, g.cost?.usdMedian),
+    wall_p50_ms: pick(g.time?.wall?.n, g.time?.wall?.p50),
+    first_pass_rate: pick(g.firstPass?.n, g.firstPass?.rate),
+    needs_human_rate: pick(g.flow?.n, g.flow?.needsHumanPct),
+  };
+}
+
+/** Relative change for cost and time, absolute for rates; flat inside the tolerance. */
+function verdict(m, cur, prev) {
+  if (!isNum(cur) || !isNum(prev)) return null;
+  const rate = m.key.endsWith("_rate");
+  const d = rate ? cur - prev : prev === 0 ? (cur === 0 ? 0 : Infinity) : (cur - prev) / prev;
+  if (Math.abs(d) <= m.flat) return "flat";
+  return (d < 0) === m.lowerBetter ? "improved" : "regressed";
+}
+
+/** Known versions of one repo, oldest first, each with its metrics and the verdicts against the previous one. */
+export function releaseMetrics(groups, { minN = DEFAULT_MIN_N, repo = null } = {}) {
+  const known = groups.filter((g) => g.version !== UNKNOWN && (repo == null || g.repo === repo))
+    .sort((a, b) => compareVersions(a.version, b.version));
+  const versions = known.map((g, i) => {
+    const metrics = valuesOf(g, minN), prev = i ? valuesOf(known[i - 1], minN) : null;
+    const verdicts = Object.fromEntries(METRICS.map((m) => [m.key, prev ? verdict(m, metrics[m.key], prev[m.key]) : null]));
+    return { version: g.version, against: i ? known[i - 1].version : null, ...metrics, verdicts };
+  });
+  return { repo, minN, versions };
+}
+
+const fmt = (key, v) => (v == null ? "n/a" : key === "usd_median" ? `$${v.toFixed(2)}` : key === "wall_p50_ms" ? `${(v / 60000).toFixed(1)}m` : `${(v * 100).toFixed(0)}%`);
+
+export function renderReleaseMetrics(rm) {
+  const rows = rm.versions.map((v) => [v.version, String(v.n), ...METRICS.map((m) => `${fmt(m.key, v[m.key])} (${v.verdicts[m.key] ?? "n/a"})`)]);
+  const head = ["version", "n", ...METRICS.map((m) => m.label)];
+  const line = (r) => `| ${r.join(" | ")} |`;
+  return [`Release metrics (min n ${rm.minN}; n/a below it)`, line(head), line(head.map(() => "---")), ...rows.map(line)].join("\n");
+}
+
+/** The FORGE:METRICS comment: marker, table and the snapshot as fenced JSON. */
+export function metricsComment(rm) {
+  return `${METRICS_MARKER}\n${renderReleaseMetrics(rm)}\n\n\`\`\`json\n${JSON.stringify(rm)}\n\`\`\`\n`;
+}
+
+/** Create the FORGE:METRICS comment on the PR, or update the existing one (argv-based gh). */
+export async function postMetrics({ gh, repo, pr, body }) {
+  const raw = await gh(["api", "--paginate", `repos/${repo}/issues/${pr}/comments`]);
+  const pages = String(raw ?? "").trim().replace(/\]\s*\[/g, ",");
+  const list = pages ? JSON.parse(pages.startsWith("[") ? pages : `[${pages}]`) : [];
+  const mine = list.filter((c) => typeof c?.body === "string" && c.body.startsWith(METRICS_MARKER)).pop();
+  if (mine) await gh(["api", "-X", "PATCH", `repos/${repo}/issues/comments/${mine.id}`, "-f", `body=${body}`]);
+  else await gh(["api", "-X", "POST", `repos/${repo}/issues/${pr}/comments`, "-f", `body=${body}`]);
+  return mine ? "updated" : "created";
+}

package/bin/stats/release.test.mjs

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { aggregate } from "./aggregate.mjs";
+import { parseArgs } from "./index.mjs";
+import { METRICS_MARKER, metricsComment, postMetrics, releaseMetrics, renderReleaseMetrics } from "./release.mjs";
+
+const trig = (fp) => [{ trigger: "build" }, ...(fp ? [] : [{ trigger: "review" }])];
+function runs(version, n, { usd, wall, fp, human }) {
+  return Array.from({ length: n }, (_, i) => ({
+    source: "github", repo: "o/r", version, issue: i + (version === "1.2.0" ? 100 : 0),
+    finished: true, usageEvents: 1, reason: i < human ? "needs-human" : "merged", wallMs: wall, endTs: "2026-10-01T00:00:00Z",
+    phases: { build: { usd, cr: 1, ms: 1 } }, builderAttempts: trig(i < fp),
+  }));
+}
+const groupsFor = (all, minN = 10) => aggregate({ runs: all }, { minN }).groups;
+
+describe("releaseMetrics (#4628)", () => {
+  const prev = runs("1.1.0", 12, { usd: 2, wall: 600000, fp: 6, human: 1 });
+  it("compares each metric with the previous release and never reports a pass", () => {
+    const cur = runs("1.2.0", 12, { usd: 3, wall: 300000, fp: 11, human: 1 });
+    const rm = releaseMetrics(groupsFor([...prev, ...cur]), { minN: 10 });
+    const v = rm.versions[1];
+    assert.equal(v.against, "1.1.0");
+    assert.deepEqual(v.verdicts, { usd_median: "regressed", wall_p50_ms: "improved", first_pass_rate: "improved", needs_human_rate: "flat" });
+    assert.equal(rm.versions[0].verdicts.usd_median, null);
+    assert.ok(!JSON.stringify(rm).includes("pass\""));
+  });
+  it("is null below min n, and the verdict is null when either side is", () => {
+    const cur = runs("1.2.0", 3, { usd: 9, wall: 1, fp: 0, human: 3 });
+    const rm = releaseMetrics(groupsFor([...prev, ...cur]), { minN: 10 });
+    const v = rm.versions[1];
+    assert.deepEqual([v.usd_median, v.wall_p50_ms, v.first_pass_rate, v.needs_human_rate], [null, null, null, null]);
+    assert.deepEqual(Object.values(v.verdicts), [null, null, null, null]);
+    assert.match(renderReleaseMetrics(rm), /n\/a/);
+  });
+  it("skips unknown versions", () => {
+    const rm = releaseMetrics(groupsFor(runs(null, 12, { usd: 1, wall: 1, fp: 1, human: 0 })), { minN: 1 });
+    assert.deepEqual(rm.versions, []);
+  });
+});
+
+describe("FORGE:METRICS comment (#4628)", () => {
+  const rm = releaseMetrics([], { minN: 10, repo: "o/r" });
+  it("carries the marker and the JSON snapshot", () => {
+    const b = metricsComment(rm);
+    assert.ok(b.startsWith(METRICS_MARKER));
+    assert.deepEqual(JSON.parse(/```json\n(.*)\n```/s.exec(b)[1]), rm);
+  });
+  it("creates the comment, then updates it", async () => {
+    const calls = [];
+    const mk = (list) => (args) => { calls.push(args.join(" ")); return args[0] === "api" && args.length === 3 ? JSON.stringify(list) : ""; };
+    assert.equal(await postMetrics({ gh: mk([{ id: 1, body: "x" }]), repo: "o/r", pr: 7, body: "b" }), "created");
+    assert.match(calls.at(-1), /-X POST repos\/o\/r\/issues\/7\/comments/);
+    assert.equal(await postMetrics({ gh: mk([{ id: 5, body: `${METRICS_MARKER}\nold` }]), repo: "o/r", pr: 7, body: "b" }), "updated");
+    assert.match(calls.at(-1), /-X PATCH repos\/o\/r\/issues\/comments\/5/);
+  });
+});
+
+describe("--by / --post-metrics parsing (#4628)", () => {
+  it("accepts version and a PR number; rejects the rest", () => {
+    assert.equal(parseArgs(["--by", "version", "--post-metrics", "12"]).postMetrics, 12);
+    assert.throws(() => parseArgs(["--by", "repo"]), /--by must be version/);
+    assert.throws(() => parseArgs(["--post-metrics", "12"]), /needs --by version/);
+    assert.throws(() => parseArgs(["--by", "version", "--post-metrics", "x"]), /PR number/);
+  });
+});

package/commands/bootstrap.md

@@ -18,18 +18,20 @@ BOOT_TPL="$(forge_root)/templates/bootstrap"; VALIDATE=$(forge_resolve bootstrap
 ```
 
 1. **Read the input.** Use the free text, or read each doc at the given path(s).
-2. **Interview** from `interview.md`. Cover every aspect it lists, briefly and conversationally. Skip what the input already answers.
-3. **Fill the blanks.** Give every unanswered aspect a sensible default with a one-line rationale tied to what the user said, using the closest `stack-profiles.md` preset. Show them all in ONE editable "Here's what I assumed" list and apply the user's edits.
-4. **Ask only genuine doubts** (they change architecture or cost: native vs PWA, data residency, self-host vs managed DB, product name, repo owner). Ask with AskUserQuestion, at most 4 multiple-choice questions per batch, each with a recommended option. Everything else stays a default. Headless (no AskUserQuestion, e.g. `npx forgedock bootstrap`): take each recommended option and list it under "Here's what I assumed".
+2. **Interview** from `interview.md`. Cover each aspect briefly; skip what the input answers.
+3. **Fill the blanks.** Default every unanswered aspect with a one-line rationale, using the closest `stack-profiles.md` preset. Show them in ONE editable "Here's what I assumed" list and apply the user's edits.
+4. **Ask only genuine doubts** (architecture or cost: native vs PWA, data residency, self-host vs managed DB, product name, repo owner). Ask with AskUserQuestion, at most 4 multiple-choice questions per batch, each with a recommended option. Everything else stays a default. Headless (no AskUserQuestion, e.g. `npx forgedock bootstrap`): take each recommended option and list it under "Here's what I assumed".
 5. **Spike doc.** Write `technical-spike.md` to `--spike` (default `docs/engineering/technical-spike.md`); set the plan's `spikeDoc` to that path. Fill every section and give each decision an ADR-style `**Rationale**:` line.
 6. **Plan JSON** in `$TMPDIR` (shape: header of `bootstrap-validate.mjs` and `bootstrap-create.mjs`):
-   - Milestones in build order, foundation first.
+   - Milestones as dependency layers, foundation first: each depends only on earlier ones (#4659).
+   - A thin deployable slice (CI deploy workflow, staging env, seed) in milestone 1 or 2, never last.
    - A `SPIKE` parent issue.
    - Sub-issues from `issue-body.md` with planned paths, `[type:*]` tags, `priority` and `component`.
    - `"ui": true` issues carry the full design spec. `"hero": true` issues route to `/design`.
-   - One `needs-human-issue.md` issue per credential, account or external input. Code issues reference it as `#{{KEY}}` but never depend on it, and name the mock that keeps dev and CI green.
+   - One `needs-human-issue.md` issue per credential, account or external input, with `gates:` (#4660). Code issues reference it as `#{{KEY}}`, never depend on it, and name the mock that keeps dev and CI green.
    - Sub-issues point at the spike via `**Parent**` only, never `Depends on`.
    - `guardrails` and `domainRules` from the interview.
-7. **Validate.** Run `node "$VALIDATE" <plan>`. Fix the plan until it exits 0. Never create from an invalid plan.
-8. **Create.** Run `node "$CREATE" <plan>`, adding `--dry-run` when the input has it (preview only, GitHub untouched). The script reuses existing issues by title and parent, so on an error re-run the same command to resume.
-9. **Hand off.** Print the script's hand-off verbatim: what was created (links), what needs a human (links), and the `/orchestrate` command per milestone, in order.
+7. **Validate.** Run `node "$VALIDATE" <plan>`. Fix the plan until it exits 0. Heed `deploy-last` warnings. Never create from an invalid plan.
+8. **Create.** Run `node "$CREATE" <plan>`, adding `--dry-run` when the input has it (preview only, GitHub untouched). It reuses existing issues, so on an error re-run it to resume.
+9. **Hand off.** Print the script's hand-off verbatim: created (links), needs a human (links), and the `/orchestrate` command per milestone, in order.
+10. Later: `npx forgedock plan validate --live` (#4659).

package/commands/deploy-info.md

@@ -325,6 +325,7 @@ else
   # Step 3: Count commits queued (commits in staging not yet in main)
   QUEUED_COMMITS=$(git rev-list --count origin/{DEFAULT_BRANCH}..origin/{STAGING_BRANCH} 2>/dev/null || echo "?")
 fi
+INTEGRATION_RED=$(gh issue list {GH_FLAG} --label integration-red --json number,title --jq '.[]|"#\(.number) \(.title)"') # forge#4667
 ```
 
 Train state is included in the Phase 5 output summary.
@@ -339,10 +340,12 @@ Train state is included in the Phase 5 output summary.
 **Commits**: {N} | **PRs**: {N} | **Issues closed**: {N} | **Files changed**: {N}
 
 ### Deploy Train State
+**Gate**: `node ${FORGEDOCK_HOME:-${CLAUDE_PLUGIN_ROOT}}/scripts/readiness.mjs --target {preview|staging|prod}`; NOT DEPLOYABLE (open prerequisites) means never call the target deployable (#4660).
 **Train PR**: #{TRAIN_PR_NUMBER} (SHA: {TRAIN_PR_SHA}, opened: {TRAIN_PR_CREATED})
 **Status**: {TRAIN_STATUS}
 {If HELD: "**Held by**: {count} open finding(s) — fix and push to unblock:\n{HELD_BY}"}
 {If CLEAR: "**Ready to merge**: no open findings on the train PR"}
+{If INTEGRATION_RED: "**Ship-gate risk**: red integration run:\n{INTEGRATION_RED}"}
 {If NO_TRAIN: "No open staging→main PR — run /review-pr-staging to open one"}
 **Queued commits**: {QUEUED_COMMITS} (commits in staging not yet in main)
 

package/commands/issue.md

@@ -85,6 +85,8 @@ Duplicates: the script's token-overlap dedup is authoritative. Before drafting,
 
 ### 3. Draft
 
+**External prerequisite** (credential, account, sign-off): label `needs-human:external` and put `gates: preview|staging-deploy|prod-deploy|feature-flag:<name>` and `owner:` in the body (#4660).
+
 **Title**: `{prefix}({scope}): {specific description}`, max 80 characters, no vague verbs ("improve", "update", "handle") without specifics.
 
 **Body** (pipeline template; `## Problem`, `## Affected Files` and `## Acceptance Criteria` are mandatory, the rest optional; in greenfield mode Affected Files may list *planned* paths inside the spike's declared repo layout):

package/commands/milestone.md

@@ -96,6 +96,8 @@ Parse `$ARGUMENTS` to determine which action to take:
 | `drift <slug>` | → **Milestone Drift** |
 | `close <slug>` | → **Close Milestone** |
 
+Unattended multi-milestone runs use the CLI, not this spec: `forgedock milestone train <slug...> [--parallel-milestones N]` (resume with `--resume <id>`, add a milestone with `--insert <id> <slug>`). It keeps one state file, reuses the lifecycle above and never edits `forge.yaml`. See `devdocs/decisions/4658-milestone-train.md`.
+
 ---
 
 ## Action: Create Milestone
@@ -356,6 +358,10 @@ else
 fi
 ```
 
+### Step 6C: Re-validate dependency order
+
+After a milestone is added, the train order changes or an issue is decomposed, run `npx forgedock plan validate --live` (#4659). Fix every `invalid` edge (a dependency in a later milestone) before continuing; `held` edges are reported only.
+
 ### Step 7: Report
 
 ```
@@ -444,37 +450,28 @@ git fetch origin milestone/{slug}
 OPEN_COUNT=$(gh issue list --milestone "{TITLE}" --state open --json number --jq '. | length')
 ```
 
-If there are open issues, warn the user:
+If there are open issues, this is a hard gate (interactive and unattended; unattended runs fail instead of asking). Every open issue must be **carried over** with a target milestone *and* a class (`engineering|external|post-deploy|wont-do`), or closed first. The class comes from labels (`carry:<class>`, `needs-human:external`, `awaiting-promotion|awaiting-precondition|awaiting-measurement`, `wont-do`; else `engineering`); the target is a `carry-to:<milestone title>` label or the user's answer. Run the check with `preflightCarryOver` (`bin/lean/lifecycle.mjs`):
+
 ```
-⚠ Milestone "{TITLE}" still has {OPEN_COUNT} open issues:
-{list them}
+⚠ Milestone "{TITLE}" still has {OPEN_COUNT} open issues without a carry-over target:
+{list them with their class}
 
-Ship anyway? Open issues will remain assigned to the milestone.
+Name a target milestone for each (or close it). Ship is blocked until every open issue has one.
 ```
 
-Wait for confirmation.
+An issue that is `needs-human:external`, flag/legal-gated (`flag-gated`, `legal-gated`) or depends on an issue that is not on the target base is **never** demoted to fast lane: it is carried to its target milestone.
 
 ### Step 2: Sync milestone branch with staging
 
-Before shipping, ensure the milestone branch is up to date with `staging` (picks up any fast-lane fixes that accumulated since the milestone branch was created):
+Before shipping, bring the milestone branch up to date with `staging` (picks up fast-lane fixes that accumulated since it was cut). `forgedock milestone-sync` does it deterministically (#4663): a milestone already `shipped:*`/`deployed:default`/`closed` is refused, a branch already containing staging is a no-op, append-only registry conflicts are resolved by the lane base-merge resolver (`lean.base_merge_*`), and a protected branch (ruleset or branch protection, or a push the rules reject) is synced through the single `sync/milestone-{slug}` PR with auto-merge armed (merged directly when CLEAN). A `ship` sync is never debounced.
 
 ```bash
 cd {REPO_PATH}
-git fetch origin {STAGING_BRANCH} milestone/{slug}
-# Never mutate the main checkout: sync in a detached worktree at the milestone tip
-SYNC_WT=$(mktemp -d)
-git worktree add --detach "$SYNC_WT" origin/milestone/{slug}
-if git -C "$SYNC_WT" merge origin/{STAGING_BRANCH} --no-edit; then
-  git -C "$SYNC_WT" push origin HEAD:refs/heads/milestone/{slug}
-  git worktree remove --force "$SYNC_WT"
-else
-  # Conflicts: report to user and STOP — do not auto-resolve
-  git -C "$SYNC_WT" merge --abort 2>/dev/null || true
-  git worktree remove --force "$SYNC_WT"
-  echo "STOP: merge conflict syncing milestone/{slug} with {STAGING_BRANCH}"; exit 1
-fi
+forgedock milestone-sync {slug} --reason ship --base {STAGING_BRANCH} --repo {owner}/{repo} || { echo "STOP: milestone sync failed or conflicts"; exit 1; }
 ```
 
+If the sync went through a PR (`pr-created`/`pr-updated`), wait for it to merge before Step 2.5.
+
 ### Step 2.5: Pre-Merge Hunk-Loss Audit (MANDATORY before creating PR)
 
 **WHY THIS EXISTS**: When a long-lived milestone branch is merged into staging, git hunks that exist in staging but were NOT touched by the milestone can be silently dropped. This happened in a 20h window that produced 13 regression-fix PRs (27% of total pipeline throughput) restoring lost code: postgres tuning params, completion_refund sweep, credit_ledger columns, browser flags, pool guards, Redis config, crawl TTL refresh, challenge_strategy guard. Each regression required investigation → new issue → new PR → review → merge — consuming nearly a third of pipeline capacity on damage repair instead of forward progress.
@@ -730,24 +727,28 @@ Run /milestone ship {slug} again when the milestone is ready for another shippin
 ```bash
 gh pr merge {PR_NUMBER} --merge
 
-# Step 6A: Promote remaining open issues to fast lane
-# Query open issues still on this milestone (per_page=100 to avoid silent truncation beyond default 30)
-OPEN_ISSUES=$(gh api "repos/{owner}/{repo}/issues?milestone={MS_NUMBER}&state=open&per_page=100" \
-  --jq '.[].number')
+# Step 6A: Carry over remaining open issues (forge#4657). Never clear the milestone: every open issue moves to its
+# target milestone (Step 1 gate) and keeps its class label. Only an issue whose preflightCarryOver entry has demote=true
+# and whose target is "fast lane" may lose its milestone; external, gated and off-base-dependent issues are never demoted.
+OPEN_ISSUES=$(gh api "repos/{owner}/{repo}/issues?milestone={MS_NUMBER}&state=open&per_page=100" --jq '.[].number')
 OPEN_COUNT=$(echo "$OPEN_ISSUES" | grep -c '[0-9]' 2>/dev/null || echo 0)
-
-if [ "$OPEN_COUNT" -gt 0 ]; then
-  echo "Promoting $OPEN_COUNT open issue(s) to fast lane (removing milestone assignment)..."
-  for ISSUE_NUM in $OPEN_ISSUES; do
-    gh api "repos/{owner}/{repo}/issues/$ISSUE_NUM" -X PATCH --field milestone=null
-    echo "  Demoted #$ISSUE_NUM → fast lane"
-  done
-else
-  echo "No open issues remain on the milestone — nothing to demote."
-fi
-
-# Step 6B: Close the milestone
-gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field state="closed"
+for ISSUE_NUM in $OPEN_ISSUES; do
+  gh issue edit "$ISSUE_NUM" --milestone "{TARGET_TITLE_OF_ISSUE}" --add-label "carry:{CLASS_OF_ISSUE}"  # <!-- allowlist:check-command-side-effects -->
+  echo "  Carried #$ISSUE_NUM -> {TARGET_TITLE_OF_ISSUE} ({CLASS_OF_ISSUE})"
+done
+
+# Step 6B: Record the lifecycle. The merge is on staging, not yet on the default branch: write
+# a FORGE:LIFECYCLE record ({"v":1,"state":"shipped:staging","sha":"<merge commit>"}) in the milestone description and
+# close the milestone only when forge.yaml `milestones.close_on` (default `default`) is reached. With close_on=default the
+# milestone stays open until the staging -> default merge is observed (deployed:default), then it closes.
+MERGE_SHA=$(gh pr view {PR_NUMBER} --json mergeCommit --jq '.mergeCommit.oid')
+DESC=$(gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --jq '.description // ""')
+TAG="FORGE:LIFECYCLE"
+BLOCK="<!-- $TAG {\"v\":1,\"state\":\"shipped:staging\",\"sha\":\"$MERGE_SHA\"} -->"
+gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field description="$(printf '%s\n\n%s' "$(printf '%s' "$DESC" | sed "/$TAG/d")" "$BLOCK")"
+# close_on=staging only (and only when no issue is left open on the milestone):
+# gh api repos/{owner}/{repo}/milestones/{MILESTONE_NUMBER} --method PATCH --field state="closed"
+# Orchestrate-driven runners get the same transition from bin/lean/lifecycle.mjs advanceMilestone at batch end.
 
 # Step 6C: Delete the milestone branch
 git push origin --delete milestone/{slug}
@@ -758,8 +759,8 @@ Report:
 ```
 ## Shipped: {MILESTONE_TITLE}
 
-Merged to staging. Milestone branch deleted.
-{If OPEN_COUNT > 0: "{OPEN_COUNT} open issue(s) moved to fast lane: #{list}. They will target staging on next /work-on run."}
+Merged to staging (lifecycle: shipped:staging). Milestone branch deleted. Milestone closes at `milestones.close_on` ({close_on}).
+{If OPEN_COUNT > 0: "{OPEN_COUNT} open issue(s) carried over: #{list} -> {target} ({class}). None were demoted to fast lane."}
 {If OPEN_COUNT == 0: "No open issues remaining — clean close."}
 
 To deploy: merge `staging → main` via GitHub web UI when ready.
@@ -769,27 +770,15 @@ To deploy: merge `staging → main` via GitHub web UI when ready.
 
 ## Action: Sync Milestone Branch
 
-Syncs the milestone branch with the latest default branch to pick up fast-lane fixes. Run periodically on long-lived milestones.
+Syncs the milestone branch with the latest default branch to pick up fast-lane fixes. Sync **on demand only**: before ship, before a batch, or when a conflict is expected. Never on every base move, and never a milestone whose lifecycle is `shipped:*` (the command refuses it; a shipped milestone is also never the "active" one to sync, `isSyncable` in `bin/lean/milestonesync.mjs`).
 
 ```bash
 cd {REPO_PATH}
 DEFAULT_BRANCH=$(yq '.branches.default' "$CONFIG_FILE")
-git fetch origin $DEFAULT_BRANCH milestone/{slug}
-# Never mutate the main checkout: sync in a detached worktree at the milestone tip
-SYNC_WT=$(mktemp -d)
-git worktree add --detach "$SYNC_WT" origin/milestone/{slug}
-if git -C "$SYNC_WT" merge origin/$DEFAULT_BRANCH --no-edit; then
-  git -C "$SYNC_WT" push origin HEAD:refs/heads/milestone/{slug}
-  git worktree remove --force "$SYNC_WT"
-else
-  # If conflicts, report to user and STOP
-  git -C "$SYNC_WT" merge --abort 2>/dev/null || true
-  git worktree remove --force "$SYNC_WT"
-  echo "STOP: merge conflict syncing milestone/{slug} with $DEFAULT_BRANCH"; exit 1
-fi
+forgedock milestone-sync {slug} --reason manual --base "$DEFAULT_BRANCH" --repo {owner}/{repo}
 ```
 
-Report: "Synced milestone/{slug} with latest `{DEFAULT_BRANCH}`. {N} new commits incorporated."
+Protected branches get ONE sync PR (`sync/milestone-{slug}`) whose head is force-pushed in place; a base that moved again inside `milestones.sync_debounce` (default 10m) reports `debounced` and leaves the PR (and its CI) alone. Report the printed status.
 
 ---
 
@@ -828,5 +817,5 @@ This automatically makes the issue a feature-lane issue — next time `/work-on`
 
 - **Milestone already exists**: Check first with `gh api repos/{owner}/{repo}/milestones --jq '.[] | select(.title == "{TITLE}")'`
 - **Branch already exists**: Reuse it, don't recreate
-- **Merge conflicts on sync/ship**: Report to user, do NOT auto-resolve. List conflicting files.
+- **Merge conflicts on sync/ship**: append-only registry conflicts matching `lean.base_merge_*` are resolved deterministically by `milestone-sync`; any other conflict is reported with its files and the branch is left untouched. Do NOT hand-resolve.
 - **No issues in milestone**: Warn user — empty milestones are valid but unusual

package/commands/orchestrate/phase-4-execution.md

@@ -1581,6 +1581,8 @@ The merged-trail hold on #${N} was released by a human (needs-human cleared or t
   return 0
 }
 
+# gh failures are unknown, not state (forge#4662): in every hold/settle/ship read, a failed gh call (HTTP 401, 5xx, timeout) is never
+# "not closed" or "no PR". Keep the previous classification, back off, and after 5 consecutive failures post one alert and stop that loop.
 # classify_predecessor_state <issue> (forge#3168): the helpers hold_merged_trail, resolve_orch_login,
 # trail_escalation_state and release_merged_trail above MUST be declared in the SAME shell before this function runs
 # (reverify_merged_trail is nested below). It has exactly ONE guarded write (the DRY_RUN-guarded hold_merged_trail