forgedock 1.35.0 → 1.36.0

package/bin/lean/sandbox-provider.contract.test.mjs

@@ -328,3 +328,31 @@ describe("remote capacity (GET /status, #4540)", () => {
     assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
   });
 });
+
+describe("remote stop at the checkpoint (#4612)", () => {
+  it("httpTransport.stop: POST .../stop, 2xx true, 404/405 false, other statuses throw", async () => {
+    const calls = [];
+    const mk = (status) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url, init) => { calls.push([init.method, url]); return { status, ok: status < 400, text: async () => "", headers: { get: () => null } }; } });
+    assert.equal(await mk(202).stop("a b"), true);
+    assert.deepEqual(calls[0], ["POST", "https://pool.example/v1/sandboxes/a%20b/stop"]);
+    assert.equal(await mk(404).stop("x"), false);
+    assert.equal(await mk(405).stop("x"), false);
+    await assert.rejects(mk(500).stop("x"), /stop: HTTP 500/);
+  });
+
+  it("provider.stop and providerLanes.stopLane signal the transport, never destroy; no transport stop = false", async () => {
+    const root = tmpDir("remote-stop-");
+    const batch = openBatch(root, "bt");
+    const { provider, transport } = remoteFixture(batch);
+    const stopped = [];
+    const lanes = providerLanes({ provider, laneArgs: () => [], ledgerFor: () => null });
+    assert.equal(await lanes.stopLane(1, { handle: { id: "sbx" } }), false, "the transport has no stop");
+    transport.stop = async (id) => { stopped.push(id); return true; };
+    assert.equal(await lanes.stopLane(1, { handle: { id: "sbx" } }), true);
+    assert.equal(await lanes.stopLane(1, {}), false, "no handle");
+    transport.stop = async () => { throw new Error("boom"); };
+    assert.equal(await provider.stop({ id: "sbx" }), false);
+    assert.deepEqual(stopped, ["sbx"]);
+    assert.deepEqual(transport.destroyed, []);
+  });
+});

package/bin/lean/sandbox-remote.mjs

@@ -44,6 +44,7 @@ export function isActionsEndpoint(endpoint) {
  * GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
  * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
  * 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
+ * Optional `POST /sandboxes/:id/stop` (#4612) asks the pool to SIGTERM the lane so its teardown checkpoints and it reports `deferred run.interrupted`; 2xx = sent, 404/405 = unsupported or gone (never a DELETE fallback).
  * Optional `GET /status -> {free: N}` is the pool's free slots (#4540); anything else (404, error, malformed) is unknown (null).
  */
 export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
@@ -73,6 +74,7 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
     async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
     async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
     async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
+    async stop(id) { const r = await call("POST", `${p(id)}/stop`); if (r.ok) return true; if (r.status === 404 || r.status === 405) return false; throw fail("stop", r); },
     async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
     async capacity() {
       // `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
@@ -167,6 +169,11 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
     async capacity() {
       try { return (await transport.capacity?.()) ?? null; } catch { return null; }
     },
+    /** #4612: a graceful stop at the lane's checkpoint; false when unsupported or failed. Never destroys the sandbox. */
+    async stop(h) {
+      if (typeof transport.stop !== "function") return false;
+      try { return (await transport.stop(h.id)) === true; } catch { return false; }
+    },
     async teardown(h) {
       await minutesOf(h);
       try { return (await transport.destroy(h.id)) !== false; } catch { return false; }

package/bin/lean/schedd.mjs

@@ -11,9 +11,10 @@
  *   {op:"attach", id, resume?}    → (`resume` also clears a pending cancel, #4592) log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
  *                                   ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
  *   {op:"set-parallel", id, k}    → {type:"ok", parallel}
+ *   {op:"requeue", id, issues, reason, by} → {type:"ok", requeued, refused, hosted, reopened} (#4550): see batch.requeue; a finished batch is hosted again
  *   {op:"add", id, issues}        → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
  *                                   hosted runner reads on its next pass and an unhosted batch on its next resume
- *   {op:"cancel", id}             → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
+ *   {op:"cancel", id, stopRemote?}           → {type:"ok", cancelled:true, hosted} (#4376): writes the batch's `cancel` file; that batch
  *                                   admits nothing more, its running local lanes get SIGTERM (their teardown saves the work)
  *                                   and the rest end held; no other batch and not the daemon. A finished, unhosted batch:
  *                                   {type:"ok", cancelled:false, finished:true}, nothing written
@@ -41,6 +42,7 @@ import { batchRoot, listBatches, openBatch } from "./batch.mjs";
 export const IDLE_MS = 10 * 60 * 1000;
 export const STOP_MS = 10 * 1000;
 export const LOCK_CHECK_MS = 5 * 1000;
+export const DUE_MS = 15 * 60 * 1000; // #4627: how often the due callback (windowed post-deploy monitoring) runs while the daemon is up
 
 export function scheddPaths(dir = schedDir()) {
   return { dir, sock: join(dir, "schedd.sock"), lock: join(dir, "schedd.lock"), log: join(dir, "schedd.log") };
@@ -83,7 +85,7 @@ function takeLock(lock, pid, alive) {
  */
 export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(process.env.FORGEDOCK_SCHEDD_IDLE_MS) || IDLE_MS,
   root = batchRoot(), log = (m) => console.log(m), pid = process.pid, alive = pidAlive, resumeOnStart = true, stopMs = STOP_MS,
-  lockCheckMs = LOCK_CHECK_MS } = {}) {
+  lockCheckMs = LOCK_CHECK_MS, due = null, dueMs = Number(process.env.FORGEDOCK_SCHEDD_DUE_MS) || DUE_MS } = {}) {
   const p = scheddPaths(dir);
   mkdirSync(dir, { recursive: true });
   const lock = takeLock(p.lock, pid, alive);
@@ -142,6 +144,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const { added, skipped } = b.addIssues(req.issues ?? []);
       return send({ type: "ok", added, skipped, hosted: hosted.has(req.id) }, true);
     }
+    if (req.op === "requeue") {
+      // #4550: the runner applies the entries on its next pass; a finished, unhosted batch is reopened (hosted as a resume).
+      const b = openBatch(root, req.id);
+      if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
+      const { requeued, refused } = b.requeue(req.issues ?? [], { reason: req.reason, by: req.by });
+      const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
+      if (reopen) await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} });
+      return send({ type: "ok", requeued, refused, hosted: hosted.has(req.id), reopened: reopen }, true);
+    }
     if (req.op === "hold" || req.op === "release") {
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -152,7 +163,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       if (!hosted.has(req.id) && b.readReport()) return send({ type: "ok", cancelled: false, finished: true }, true);
-      b.requestCancel({ reason: "operator" });
+      b.requestCancel({ reason: "operator", stopRemote: req.stopRemote === true });
       return send({ type: "ok", cancelled: true, hosted: hosted.has(req.id) }, true);
     }
     const sub = (m) => send(m, m.type === "done" || m.type === "error");
@@ -173,7 +184,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
       const b = openBatch(root, req.id);
       if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
       const rep = b.readReport();
-      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length) {
+      if (rep && !(rep.rows ?? []).some((r) => r.outcome === "held") && !b.unrunAdds().length && !b.unrunRequeues().length) {
         return send({ type: "done", replay: true, result: { rows: rep.rows ?? [], crossLane: rep.crossLane ?? [], batch: b.id, ok: !!rep.ok }, report: rep.report ?? null }, true);
       }
       const ent = await host({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, sub);
@@ -212,6 +223,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     if (closing) return closing;
     clearInterval(idleTimer);
     clearInterval(lockTimer);
+    if (dueTimer) clearInterval(dueTimer);
     for (const c of conns) c.destroy();
     const stopping = [...runs.keys()];
     for (const ac of runs.values()) ac.abort();
@@ -241,6 +253,14 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
     else if (!existsSync(p.sock)) { log(`schedd: lost socket ${p.sock}, exiting`); close(); }
   }, lockCheckMs);
   lockTimer.unref?.();
+  // #4627: windowed post-deploy checks. One run at a time; a failure is logged and never stops the daemon.
+  let dueRunning = false;
+  const dueTimer = due ? setInterval(async () => {
+    if (closing || dueRunning) return;
+    dueRunning = true;
+    try { await due(); } catch (e) { log(`schedd: due run failed: ${e?.message ?? e}`); } finally { dueRunning = false; }
+  }, dueMs) : null;
+  dueTimer?.unref?.();
 
   // Daemon restart: resume every daemon-hosted batch that never wrote its report; live lanes are re-adopted.
   if (resumeOnStart) {

package/bin/lean/schedd.test.mjs

@@ -208,6 +208,32 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     } finally { await h.cleanup(); }
   });
 
+  it("#4550: --requeue on a finished batch with a report reopens it through the daemon and runs the issue again; --add still skips it", async () => {
+    const h = host();
+    try {
+      await h.daemon();
+      const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
+      await withRoot(h, () => capture(async () => {
+        writeFileSync(h.release, "");
+        const first = await orchestrateFromCli(["1", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
+        assert.deepEqual(first.rows.map((r) => [r.issue, r.outcome]), [[1, "merged"]]);
+        const [id] = listBatches(h.root);
+        const b = openBatch(h.root, id);
+        // The lane ended needs-human and the operator decided it: the finished batch holds that terminal row.
+        b.store.save({ ...b.store.load(), results: { 1: { issue: 1, outcome: "needs-human", pr: null, cost_usd: 0, minutes: 0 } } });
+        const rep = b.readReport();
+        b.writeReport({ ...rep, rows: [{ issue: 1, outcome: "needs-human" }] });
+        assert.deepEqual((await orchestrateFromCli(["--add", id, "1"], client)).skipped.map((x) => x.reason), ["already in batch"]);
+        const res = await orchestrateFromCli(["--requeue", id, "1", "--reason", "decided"], client);
+        assert.deepEqual(res.requeued, [1]);
+        await until(() => h.launched().length === 2);
+        await until(() => b.store.load()?.results?.[1]?.outcome === "merged" && b.readReport()?.rows?.[0]?.outcome === "merged");
+        assert.deepEqual(listBatches(h.root), [id], "no second batch");
+        assert.equal(b.readReport().rows[0].requeued.reason, "decided");
+      }));
+    } finally { await h.cleanup(); }
+  });
+
   it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
     const h = host();
     try {
@@ -487,3 +513,31 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
     assert.deepEqual(env, { NODE_OPTIONS: "x", PATH: "/c", CLAUDE_CONFIG_DIR: "/acct-b", FORGEDOCK_SCHED_DIR: "/s" });
   });
 });
+
+describe("schedd due timer (#4627)", () => {
+  const boot = async (opts) => startDaemon({ dir: tmpDir("schedd-due-"), root: tmpDir("schedd-due-b-"), runBatch: async () => ({}), resumeOnStart: false, log: () => {}, ...opts });
+
+  it("runs the due callback on its interval, never overlapped, and survives a throwing callback", async () => {
+    let calls = 0, active = 0, overlap = false;
+    const logs = [];
+    const d = await boot({ dueMs: 15, log: (m) => logs.push(m), due: async () => {
+      calls++; active++; if (active > 1) overlap = true;
+      await new Promise((r) => setTimeout(r, 40));
+      active--; if (calls === 1) throw new Error("boom");
+    } });
+    try {
+      await until(() => calls >= 3);
+      assert.equal(overlap, false);
+      assert.ok(logs.some((m) => /due run failed: boom/.test(m)));
+      assert.ok(existsSync(d.sock));
+    } finally { await d.close(); }
+    const n = calls;
+    await new Promise((r) => setTimeout(r, 80));
+    assert.equal(calls, n, "the timer is cleared on close");
+  });
+
+  it("without a due callback there is no timer", async () => {
+    const d = await boot({ dueMs: 10 });
+    await d.close();
+  });
+});

package/bin/stats/aggregate.mjs

@@ -86,7 +86,7 @@ function costSection(runs) {
     n, cacheReadMean: mean(cr), cacheReadMedian: percentile(cr, 50), perPhase,
     outputMean: n ? sum(runs.map((r) => sum(Object.values(r.phases).map((p) => p.out ?? 0)))) / n : null,
     perMergedIssue: merged ? sum(cr) / merged : null,
-    usdN: usd.length, usdMean: mean(usd), usdTotal: usd.length ? sum(usd) : null,
+    usdN: usd.length, usdMean: mean(usd), usdMedian: percentile(usd, 50), usdTotal: usd.length ? sum(usd) : null,
   };
 }
 

package/bin/stats/experiment.mjs

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * forge#4629: `forgedock stats --experiment <id>`. Finished runs tagged with the experiment id, grouped by arm; each metric
+ * is null below the declared min n (release.mjs rule), a target is null when a value it needs is null, never a pass.
+ */
+import { aggregate } from "./aggregate.mjs";
+import { releaseMetrics } from "./release.mjs";
+import { benchRecord } from "../experiment.mjs";
+
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+
+/** One target against the per-arm metrics: absolute limit, or limit x the other arm's value. */
+export function evalTarget(t, byArm) {
+  const value = byArm[t.arm]?.[t.metric], base = t.vs ? byArm[t.vs]?.[t.metric] : 1;
+  const limit = isNum(base) ? t.limit * base : null;
+  const pass = isNum(value) && isNum(limit) ? (t.op === "<=" ? value <= limit : value >= limit) : null;
+  return { pass, value: isNum(value) ? value : null, limit };
+}
+
+export function experimentReport(runs, exp) {
+  const mine = runs.filter((r) => r.finished && r.experiment === exp.id);
+  const arms = {}, byArm = {};
+  for (const arm of exp.arms) {
+    const rs = mine.filter((r) => r.arm === arm).map((r) => ({ ...r, version: "0.0.0" })); // one group per arm
+    const g = aggregate({ runs: rs }, { minN: exp.minN }).groups[0];
+    const m = g ? releaseMetrics([g], { minN: exp.minN }).versions[0] : null;
+    byArm[arm] = Object.fromEntries(exp.metrics.map((k) => [k, m?.[k] ?? null]));
+    arms[arm] = { n: rs.length, ...byArm[arm] };
+  }
+  const targets = Object.fromEntries(exp.targets.map((t) => [t.text, evalTarget(t, byArm)]));
+  const verdicts = Object.values(targets);
+  return { id: exp.id, minN: exp.minN, ...benchRecord({ source: "stats", arms, targets }),
+    targets_met: verdicts.length > 0 && verdicts.every((v) => v.pass === true) };
+}
+
+const fmt = (v) => (v == null ? "n/a" : Number.isInteger(v) ? String(v) : v.toFixed(2));
+export function renderExperiment(r) {
+  const keys = Object.keys(Object.values(r.arms)[0] ?? {}).filter((k) => k !== "n");
+  const line = (c) => `| ${c.join(" | ")} |`;
+  const rows = Object.entries(r.arms).map(([a, v]) => line([a, String(v.n), ...keys.map((k) => fmt(v[k]))]));
+  const tg = Object.entries(r.targets).map(([t, v]) => `${t}: ${v.pass == null ? "n/a" : v.pass ? "met" : "missed"} (${fmt(v.value)} vs ${fmt(v.limit)})`);
+  return [`Experiment ${r.id} (min n ${r.minN}; n/a below it)`, line(["arm", "n", ...keys]), line(["---", "---", ...keys.map(() => "---")]), ...rows, ...tg, `targets met: ${r.targets_met}`].join("\n");
+}

package/bin/stats/github-source.mjs

@@ -77,9 +77,10 @@ export function runsFromRecords(records) {
     const key = `${r.repo}#${r.issue}#${r.run ?? ""}`;
     const run = byRun.get(key) ?? { source: "github", repo: r.repo ?? null, version: r.forge_version ?? null, issue: r.issue, run: r.run ?? null,
       finished: false, hydrated: false, usageEvents: 0, reason: null, code: null, remediated: false, startTs: null, endTs: null, wallMs: null,
-      phases: {}, builderAttempts: [], _first: null };
+      phases: {}, builderAttempts: [], experiment: null, arm: null, _first: null };
     byRun.set(key, run);
     if (!run._first || r.ts < run._first) run._first = r.ts;
+    if (r.experiment != null && r.arm != null) { run.experiment = String(r.experiment); run.arm = String(r.arm); } // #4629
     if (r.event === "run_terminal") {
       run.finished = true; run.reason = r.reason ?? null; run.code = r.code ?? null; run.endTs = r.ts ?? null; run.wallMs = num(r.wall_ms);
       continue;

package/bin/stats/index.mjs

@@ -16,6 +16,9 @@ import { parseForgeYaml } from "../forge-utils.mjs";
 import { renderCompact, renderJson, renderMarkdown } from "./render.mjs";
 import { edgeQuery, parseEdge, verdictMix, contextPackAb, renderContextAb, builderTierReport, renderTiers, exploreAb, renderExploreAb, laneProfileReport, renderLaneProfiles } from "./query.mjs";
 import { batchEfficiency, windowEfficiency, compareEfficiency, formatEfficiencyLines, formatCompareLines, parseWindows } from "./efficiency.mjs";
+import { releaseMetrics, renderReleaseMetrics, metricsComment, postMetrics } from "./release.mjs";
+import { experimentReport, renderExperiment } from "./experiment.mjs";
+import { readExperiment } from "../experiment.mjs";
 import { poolUtilisation, formatPoolLines } from "./pool.mjs";
 import { batchRoot } from "../lean/batch.mjs";
 
@@ -23,6 +26,8 @@ const here = dirname(fileURLToPath(import.meta.url));
 
 export const HELP = `Usage: forgedock stats [--since <version|date>] [--repo <owner/repo>] [--json] [--refresh]
                       [--compact] [--no-local] [--no-cache] [--min-n N] [--release-times FILE]
+       forgedock stats --experiment <id> [--require-met] [--min-n N] [--repo <owner/repo>] [--json]
+       forgedock stats --by version [--post-metrics <pr>] [--min-n N] [--repo <owner/repo>] [--json]
        forgedock stats --pool [--window 7d,30d] [--repo <owner/repo>] [--json]
        forgedock stats --efficiency [--batch <id>] [--window 7d,30d] [--compare <date|version>] [--offline] [--repo <owner/repo>] [--json]
 
@@ -42,6 +47,14 @@ delta against the previous version and a low-n flag for samples under 10. Missin
   --release-times FILE  "<version> <epoch-seconds>" lines used to estimate versions of unversioned records
                         (default: git tags v* of the ForgeDock checkout)
 
+Per-release regression rule (forge#4628):
+  --require-met         with --experiment: exit 1 unless every target is met (#4634; for a named post-deploy check)
+  --experiment <id>     arms of the forge.yaml experiment: section (arms, ratio, metrics, targets, min_n) against its targets; n/a below min n
+  --emit-event          with --experiment: emit one experiment.verdict event (met or missed; docs/EVENTS.md)
+  --by version          median cost per run, wall p50, first-pass rate and needs-human rate of each release against the
+                        previous one; a metric under --min-n runs is null and its verdict null, never a pass
+  --post-metrics <pr>   post (or update) the <!-- FORGE:METRICS v1 --> snapshot comment on that release PR
+
 ALERTS (forge#4354; local run logs only, skipped with --no-local): fail-soft rates over the last 50 lanes of the repo
 (DEPS_PREPARED failed per ecosystem, deferred-to-ci, repeated TOOLCHAIN_WARNING, outcome codes whose share doubled);
 thresholds under forge.yaml stats.alerts.*. Nothing is printed when none fired or under 10 lanes.
@@ -81,8 +94,11 @@ Lane profile baseline (forge#4436; local run logs only, every run of each issue,
   --issues <list>       pin the report to these issues (comma list), for a before/after on the same issue set
 `;
 
+/** #4634: 0, or 1 when --require-met and the experiment's targets are not all met (a named post-deploy check gates on it). */
+export function experimentExit(opts, report) { return opts.requireMet && report.targets_met !== true ? 1 : 0; }
+
 export function parseArgs(argv) {
-  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
+  const o = { since: null, repo: null, edge: null, milestone: null, contextAb: false, exploreAb: false, tiers: false, laneProfile: false, issues: null, efficiency: false, pool: false, batch: null, windows: null, compare: null, by: null, experiment: null, emitEvent: false, requireMet: false, postMetrics: null, offline: false, json: false, refresh: false, compact: false, local: true, cache: true, minN: DEFAULT_MIN_N, releaseTimes: null, help: false };
   const val = (i, name) => {
     const v = argv[i + 1];
     if (v == null || v.startsWith("--")) throw new Error(`${name} needs a value`);
@@ -112,8 +128,13 @@ export function parseArgs(argv) {
       try { if (!v || !Object.keys(parseSince(v)).length) throw bad(); } catch { throw bad(); }
       o.compare = v;
     }
+    else if (name === "--by") { const v = take(); if (v !== "version") throw new Error("--by must be version"); o.by = v; }
+    else if (name === "--experiment") { const v = take(); if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v)) throw new Error("--experiment must be an experiment id"); o.experiment = v; }
+    else if (name === "--emit-event") o.emitEvent = true;
+    else if (name === "--post-metrics") { const v = take(); if (!/^[1-9]\d*$/.test(v)) throw new Error("--post-metrics must be a PR number"); o.postMetrics = Number(v); }
+    else if (name === "--require-met") o.requireMet = true;
     else if (name === "--offline") o.offline = true;
-    else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); }
+    else if (name === "--min-n") { const v = take(); if (!/^\d+$/.test(v)) throw new Error("--min-n must be a non-negative integer"); o.minN = Number(v); o.minNSet = true; }
     else if (name === "--json") o.json = true;
     else if (name === "--refresh") o.refresh = true;
     else if (name === "--compact") o.compact = true;
@@ -125,6 +146,7 @@ export function parseArgs(argv) {
   if (o.batch && !o.efficiency) throw new Error("--batch and --window need --efficiency");
   if (o.windows && !o.efficiency && !o.pool) throw new Error("--batch and --window need --efficiency");
   if ((o.compare || o.offline) && !o.efficiency) throw new Error("--compare and --offline need --efficiency");
+  if (o.postMetrics && !o.by) throw new Error("--post-metrics needs --by version");
   return o;
 }
 
@@ -287,6 +309,12 @@ export async function buildReport(opts, deps = {}) {
   }
   const report = aggregate({ runs: [...model.runs, ...local], events: model.events, issues: model.issues },
     { minN: opts.minN, releaseTimes, ...parseSince(opts.since) });
+  if (opts.experiment) { // forge#4629
+    const exp = deps.experiment ?? readExperiment();
+    if (!exp || exp.id !== opts.experiment) throw new Error(`experiment ${opts.experiment} is not declared in forge.yaml (experiment: id, arms, ...)`);
+    return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, query: { experiment: experimentReport([...model.runs, ...local], { ...exp, minN: opts.minNSet ? opts.minN : exp.minN }) } };
+  }
+  if (opts.by) return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, query: { release_metrics: releaseMetrics(report.groups, { minN: opts.minN, repo }) } };
   // forge#4354: fail-soft alerts from the local run logs only (no gh call)
   const alerts = opts.local ? computeAlerts(deps.localLanes ?? readLanes(runsDir(env)), { repo, ...(deps.alertConfig ?? readAlertConfig()) }) : [];
   return { repo, sources: { github: { items: gh.items.length, records: gh.records.length, cached: gh.fromCache }, local: { runs: local.length } }, ...report, alerts };
@@ -299,6 +327,10 @@ export async function runStats(argv, deps = {}) {
     return 2;
   }
   if (opts.help) { process.stdout.write(HELP); return 0; }
+  if (opts.requireMet && !opts.experiment) {
+    process.stderr.write(`forgedock stats: --require-met needs --experiment <id>\n`);
+    return 2;
+  }
   opts.repo = resolveRepo(opts.repo);
   if (!opts.repo || !isValidRepo(opts.repo)) {
     process.stderr.write(`forgedock stats: could not resolve a valid owner/repo (pass --repo owner/repo)\n`);
@@ -306,6 +338,14 @@ export async function runStats(argv, deps = {}) {
   }
   try {
     const report = await buildReport(opts, { onProgress: makeProgressPrinter(), ...deps });
+    if (report.query?.release_metrics) {
+      const rm = report.query.release_metrics;
+      if (opts.postMetrics) await postMetrics({ gh: deps.gh ?? defaultGh, repo: opts.repo, pr: opts.postMetrics, body: metricsComment(rm) });
+      process.stdout.write(`${opts.json ? renderJson(report) : renderReleaseMetrics(rm)}\n`);
+      return 0;
+    }
+    if (report.query?.experiment && opts.emitEvent) await (deps.emitExperiment ?? (async (r, x) => (await import("../engine-cli.mjs")).experimentEventSink(r, x)))(opts.repo, report.query.experiment);
+    if (report.query?.experiment) { process.stdout.write(`${opts.json ? renderJson(report) : renderExperiment(report.query.experiment)}\n`); return experimentExit(opts, report.query.experiment); }
     if (report.query?.lane_profile && !opts.json) { process.stdout.write(`${renderLaneProfiles(report.query.lane_profile)}\n`); return 0; }
     if (report.query?.tiers && !opts.json) { process.stdout.write(`${renderTiers(report.query.tiers)}\n`); return 0; }
     if (report.query?.pool && !opts.json) { process.stdout.write(`${formatPoolLines(report.query.pool).join("\n")}\n`); return 0; }

package/bin/stats/release.mjs

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * forge#4628: per-release regression rule over the `aggregate()` groups. Each metric is null when its own sample is
+ * below minN; a verdict is null when either side is null. There is no "pass" value, only regressed|improved|flat|null.
+ */
+import { DEFAULT_MIN_N, UNKNOWN, compareVersions } from "./aggregate.mjs";
+
+export const METRICS_MARKER = "<!-- FORGE:METRICS v1 -->";
+const isNum = (x) => typeof x === "number" && Number.isFinite(x);
+// lower is better for all but the first-pass rate
+const METRICS = [
+  { key: "usd_median", label: "median cost/run", lowerBetter: true, flat: 0.05 },
+  { key: "wall_p50_ms", label: "wall p50", lowerBetter: true, flat: 0.05 },
+  { key: "first_pass_rate", label: "first-pass rate", lowerBetter: false, flat: 0.02 },
+  { key: "needs_human_rate", label: "needs-human rate", lowerBetter: true, flat: 0.02 },
+];
+
+/** The metric values of one group, each null below minN (its own n). */
+function valuesOf(g, minN) {
+  const pick = (n, v) => (isNum(n) && n >= minN && isNum(v) ? v : null);
+  return {
+    n: g.n,
+    usd_median: pick(g.cost?.usdN, g.cost?.usdMedian),
+    wall_p50_ms: pick(g.time?.wall?.n, g.time?.wall?.p50),
+    first_pass_rate: pick(g.firstPass?.n, g.firstPass?.rate),
+    needs_human_rate: pick(g.flow?.n, g.flow?.needsHumanPct),
+  };
+}
+
+/** Relative change for cost and time, absolute for rates; flat inside the tolerance. */
+function verdict(m, cur, prev) {
+  if (!isNum(cur) || !isNum(prev)) return null;
+  const rate = m.key.endsWith("_rate");
+  const d = rate ? cur - prev : prev === 0 ? (cur === 0 ? 0 : Infinity) : (cur - prev) / prev;
+  if (Math.abs(d) <= m.flat) return "flat";
+  return (d < 0) === m.lowerBetter ? "improved" : "regressed";
+}
+
+/** Known versions of one repo, oldest first, each with its metrics and the verdicts against the previous one. */
+export function releaseMetrics(groups, { minN = DEFAULT_MIN_N, repo = null } = {}) {
+  const known = groups.filter((g) => g.version !== UNKNOWN && (repo == null || g.repo === repo))
+    .sort((a, b) => compareVersions(a.version, b.version));
+  const versions = known.map((g, i) => {
+    const metrics = valuesOf(g, minN), prev = i ? valuesOf(known[i - 1], minN) : null;
+    const verdicts = Object.fromEntries(METRICS.map((m) => [m.key, prev ? verdict(m, metrics[m.key], prev[m.key]) : null]));
+    return { version: g.version, against: i ? known[i - 1].version : null, ...metrics, verdicts };
+  });
+  return { repo, minN, versions };
+}
+
+const fmt = (key, v) => (v == null ? "n/a" : key === "usd_median" ? `$${v.toFixed(2)}` : key === "wall_p50_ms" ? `${(v / 60000).toFixed(1)}m` : `${(v * 100).toFixed(0)}%`);
+
+export function renderReleaseMetrics(rm) {
+  const rows = rm.versions.map((v) => [v.version, String(v.n), ...METRICS.map((m) => `${fmt(m.key, v[m.key])} (${v.verdicts[m.key] ?? "n/a"})`)]);
+  const head = ["version", "n", ...METRICS.map((m) => m.label)];
+  const line = (r) => `| ${r.join(" | ")} |`;
+  return [`Release metrics (min n ${rm.minN}; n/a below it)`, line(head), line(head.map(() => "---")), ...rows.map(line)].join("\n");
+}
+
+/** The FORGE:METRICS comment: marker, table and the snapshot as fenced JSON. */
+export function metricsComment(rm) {
+  return `${METRICS_MARKER}\n${renderReleaseMetrics(rm)}\n\n\`\`\`json\n${JSON.stringify(rm)}\n\`\`\`\n`;
+}
+
+/** Create the FORGE:METRICS comment on the PR, or update the existing one (argv-based gh). */
+export async function postMetrics({ gh, repo, pr, body }) {
+  const raw = await gh(["api", "--paginate", `repos/${repo}/issues/${pr}/comments`]);
+  const pages = String(raw ?? "").trim().replace(/\]\s*\[/g, ",");
+  const list = pages ? JSON.parse(pages.startsWith("[") ? pages : `[${pages}]`) : [];
+  const mine = list.filter((c) => typeof c?.body === "string" && c.body.startsWith(METRICS_MARKER)).pop();
+  if (mine) await gh(["api", "-X", "PATCH", `repos/${repo}/issues/comments/${mine.id}`, "-f", `body=${body}`]);
+  else await gh(["api", "-X", "POST", `repos/${repo}/issues/${pr}/comments`, "-f", `body=${body}`]);
+  return mine ? "updated" : "created";
+}

package/bin/stats/release.test.mjs

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { aggregate } from "./aggregate.mjs";
+import { parseArgs } from "./index.mjs";
+import { METRICS_MARKER, metricsComment, postMetrics, releaseMetrics, renderReleaseMetrics } from "./release.mjs";
+
+const trig = (fp) => [{ trigger: "build" }, ...(fp ? [] : [{ trigger: "review" }])];
+function runs(version, n, { usd, wall, fp, human }) {
+  return Array.from({ length: n }, (_, i) => ({
+    source: "github", repo: "o/r", version, issue: i + (version === "1.2.0" ? 100 : 0),
+    finished: true, usageEvents: 1, reason: i < human ? "needs-human" : "merged", wallMs: wall, endTs: "2026-10-01T00:00:00Z",
+    phases: { build: { usd, cr: 1, ms: 1 } }, builderAttempts: trig(i < fp),
+  }));
+}
+const groupsFor = (all, minN = 10) => aggregate({ runs: all }, { minN }).groups;
+
+describe("releaseMetrics (#4628)", () => {
+  const prev = runs("1.1.0", 12, { usd: 2, wall: 600000, fp: 6, human: 1 });
+  it("compares each metric with the previous release and never reports a pass", () => {
+    const cur = runs("1.2.0", 12, { usd: 3, wall: 300000, fp: 11, human: 1 });
+    const rm = releaseMetrics(groupsFor([...prev, ...cur]), { minN: 10 });
+    const v = rm.versions[1];
+    assert.equal(v.against, "1.1.0");
+    assert.deepEqual(v.verdicts, { usd_median: "regressed", wall_p50_ms: "improved", first_pass_rate: "improved", needs_human_rate: "flat" });
+    assert.equal(rm.versions[0].verdicts.usd_median, null);
+    assert.ok(!JSON.stringify(rm).includes("pass\""));
+  });
+  it("is null below min n, and the verdict is null when either side is", () => {
+    const cur = runs("1.2.0", 3, { usd: 9, wall: 1, fp: 0, human: 3 });
+    const rm = releaseMetrics(groupsFor([...prev, ...cur]), { minN: 10 });
+    const v = rm.versions[1];
+    assert.deepEqual([v.usd_median, v.wall_p50_ms, v.first_pass_rate, v.needs_human_rate], [null, null, null, null]);
+    assert.deepEqual(Object.values(v.verdicts), [null, null, null, null]);
+    assert.match(renderReleaseMetrics(rm), /n\/a/);
+  });
+  it("skips unknown versions", () => {
+    const rm = releaseMetrics(groupsFor(runs(null, 12, { usd: 1, wall: 1, fp: 1, human: 0 })), { minN: 1 });
+    assert.deepEqual(rm.versions, []);
+  });
+});
+
+describe("FORGE:METRICS comment (#4628)", () => {
+  const rm = releaseMetrics([], { minN: 10, repo: "o/r" });
+  it("carries the marker and the JSON snapshot", () => {
+    const b = metricsComment(rm);
+    assert.ok(b.startsWith(METRICS_MARKER));
+    assert.deepEqual(JSON.parse(/```json\n(.*)\n```/s.exec(b)[1]), rm);
+  });
+  it("creates the comment, then updates it", async () => {
+    const calls = [];
+    const mk = (list) => (args) => { calls.push(args.join(" ")); return args[0] === "api" && args.length === 3 ? JSON.stringify(list) : ""; };
+    assert.equal(await postMetrics({ gh: mk([{ id: 1, body: "x" }]), repo: "o/r", pr: 7, body: "b" }), "created");
+    assert.match(calls.at(-1), /-X POST repos\/o\/r\/issues\/7\/comments/);
+    assert.equal(await postMetrics({ gh: mk([{ id: 5, body: `${METRICS_MARKER}\nold` }]), repo: "o/r", pr: 7, body: "b" }), "updated");
+    assert.match(calls.at(-1), /-X PATCH repos\/o\/r\/issues\/comments\/5/);
+  });
+});
+
+describe("--by / --post-metrics parsing (#4628)", () => {
+  it("accepts version and a PR number; rejects the rest", () => {
+    assert.equal(parseArgs(["--by", "version", "--post-metrics", "12"]).postMetrics, 12);
+    assert.throws(() => parseArgs(["--by", "repo"]), /--by must be version/);
+    assert.throws(() => parseArgs(["--post-metrics", "12"]), /needs --by version/);
+    assert.throws(() => parseArgs(["--by", "version", "--post-metrics", "x"]), /PR number/);
+  });
+});

package/commands/orchestrate.md

@@ -102,7 +102,10 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
   `--resume`; a batch started without the daemon still needs `forgedock orchestrate --resume <id>`. Change K of a running batch with
   `forgedock orchestrate --set-parallel <id> K`. Issues filed while a batch runs join it with
   `forgedock orchestrate --add <id> N...` (or `--resume <id> N...`), under its K and DAG; never start a second batch
-  for them. Its pre-filter still closes already-fixed ones. The runner requeues `deferred` lanes by code (host busy and a full sandbox pool after their
+  for them. After a decision (a `FORGE:DECISION` posted and `needs-human` removed), a blocked issue that already ended in the batch
+  (`needs-human`, `invalid`, `skipped`, `dependency-*`, `ci-pending`) returns to it with
+  `forgedock orchestrate --requeue <id> N... --reason "..."` (refused while it is labelled `needs-human` or closed; skipped dependents
+  return to pending with it; a finished batch is reopened); `--add` skips such an issue as `already in batch`. Its pre-filter still closes already-fixed ones. The runner requeues `deferred` lanes by code (host busy and a full sandbox pool after their
   retry-in, uncapped, a usage limit holds admissions for that account (backend identity) until the reset, across every batch on
   the host; `forgedock limits show|clear <id>|--all` lists or drops holds; transient merge/push/idle failures back off; other
   deferrals stop after `orchestration.max_deferrals`), relaunches an `engine-error` once, and stops admitting after
@@ -137,7 +140,8 @@ When a lane finishes (agent lanes; the batch runner applies the same rules itsel
   ran (no lease, comment or worktree). Report the code and its detail (for example a missing `milestone/<slug>`
   branch), and mark the issue's dependents skipped. Do not retry it with a different `--lane`.
 - `dependency-blocked`: its dependency is open and labelled `needs-human`. Report it as blocked by that issue,
-  do not label it `needs-human`, and mark its dependents skipped.
+  do not label it `needs-human`, and mark its dependents skipped. Once the blocker is cleared, `forgedock orchestrate --requeue <id> N...`
+  returns the issue and its skipped dependents to the batch.
 - `decomposed`: the builder split the issue and the engine filed sub-issues (listed in its output, each with
   `**Parent**` and `**Depends on**` lines). Add them to the DAG, launch them, and let the parent's dependents
   wait on the sub-issues.
@@ -159,7 +163,8 @@ When a lane finishes (agent lanes; the batch runner applies the same rules itsel
 - `needs-human` with a **human code** (`builder.blocked`: product-decision, spend, credentials, promotion, customer): an
   external action agents cannot take, or a product decision too critical for agents. It is never auto-recovered. Read
   the issue's last comment for the reason, mark its dependents skipped, and add it to the escalation list (step 4). Only
-  explicit or logical edges skip a dependent; an issue that merely overlaps the same file keeps running.
+  explicit or logical edges skip a dependent (after you decide the block in scope, clear the label and run
+  `forgedock orchestrate --requeue <id> N... --reason "..."`, never a second batch); an issue that merely overlaps the same file keeps running.
 - `needs-human` with a **terminal code** that is neither recoverable nor human (for example a `conformance.*` code, or
   `contract.invalid` without a mid-run issue edit): the runner does not recover it and it is not an escalation. The
   batch report lists it as `unrecovered: #N (<code>: <detail>)`, like a recovery that did not merge. Mark its dependents