forgedock 1.43.0 → 1.43.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -45,6 +45,7 @@ export const OUTCOME_CODES = Object.freeze({
45
45
  RUN_INTERRUPTED: "run.interrupted", // forge#4179: SIGTERM/SIGINT on a core run; teardown ran from the ledger, lane requeued
46
46
  PATH_SWITCH_CONFLICT: "path.switch_conflict",
47
47
  ENGINE_ERROR: "engine.error",
48
+ ENGINE_DCO_REWRITE: "engine.dco_rewrite", // forge#4997: the DCO sign-off rewrite of unpushed commits failed (git filter-branch); mechanical engine-error (not needs-human), relaunched by the batch runner
48
49
  ENGINE_VALIDATE_RESULT_LOST: "engine.validate_result_lost", // forge#4708: a validate result recorded before a CI fix round is missing after it and could not be carried forward
49
50
  INFRA_DISK_QUOTA: "infra.disk_quota",
50
51
  INFRA_CLI_SPAWN_TRANSIENT: "infra.cli_spawn_transient",
@@ -1283,6 +1283,11 @@ async function addedEdges({ added, records, io, repo }) {
1283
1283
  * state`): skipped `closed` (`unreadable` when the read fails); the rest go to the batch's `add` file, through the daemon
1284
1284
  * when `viaDaemon(issues)` answers (non-null), else written here (batch.mjs `addIssues` skips issues already in the batch).
1285
1285
  */
1286
+ /** #5000: what happened to the batch after an add/requeue; never implies a run when no daemon hosts it. */
1287
+ const hostedLine = (id, res) => res.reopened ? `batch ${id}: finished batch reopened and hosted by the schedd`
1288
+ : res.hosted ? `batch ${id}: hosted by the schedd; the runner picks it up on its next pass`
1289
+ : `batch ${id}: queued but NOT hosted (no schedd hosts it); run: forgedock orchestrate --resume ${id}`;
1290
+
1286
1291
  async function addToBatch({ id, tokens, root, io, viaDaemon = null, print = (l) => console.log(l) }) {
1287
1292
  const batchMod = await import("./lean/batch.mjs");
1288
1293
  const b = batchMod.openBatch(root, id);
@@ -1308,11 +1313,13 @@ async function addToBatch({ id, tokens, root, io, viaDaemon = null, print = (l)
1308
1313
  let res = { added: [], skipped: [], requeued: [] };
1309
1314
  if (open.length) {
1310
1315
  res = (viaDaemon ? await viaDaemon(open) : null) ?? b.addIssues(open);
1311
- if (res.reopened === undefined && res.requeued?.length && b.readReport()) b.clearReport(); // #4687: no daemon: the next --resume reruns it
1316
+ if (res.reopened === undefined && (res.added?.length || res.requeued?.length) && b.readReport()) b.clearReport(); // #4687/#5000: no daemon: the next --resume reruns it
1312
1317
  }
1313
1318
  const out = { ok: true, batch: id, added: res.added ?? [], requeued: res.requeued ?? [], skipped: [...(res.skipped ?? []), ...skipped] };
1319
+ if (res.hosted !== undefined) { out.hosted = !!res.hosted; out.reopened = !!res.reopened; }
1314
1320
  out.edges = await addedEdges({ added: out.added, records, io, repo }); // #4773
1315
1321
  print(`batch ${id}: added ${out.added.map((n) => `#${n}`).join(" ") || (out.requeued.length ? "" : "(none)")}${out.requeued.length ? `${out.added.length ? " " : ""}${out.requeued.map((n) => `requeued #${n}`).join(" ")}` : ""}${out.skipped.length ? `; skipped ${out.skipped.map((x) => `${/^\d+$/.test(String(x.issue)) ? "#" : ""}${x.issue} (${x.reason})`).join(", ")}` : ""}`);
1322
+ if (out.added.length || out.requeued.length) print(hostedLine(id, res));
1316
1323
  if (out.edges.length) print(`batch ${id}: edges ${out.edges.map((e) => `#${e.from} blocked by #${e.to}`).join(", ")}`);
1317
1324
  return out;
1318
1325
  }
@@ -1371,7 +1378,9 @@ async function requeueBatch({ id, tokens, reason, root, io, viaDaemon = null, pr
1371
1378
  if (res.reopened === undefined && res.requeued?.length && b.readReport()) b.clearReport(); // no daemon: the next --resume reruns it
1372
1379
  }
1373
1380
  const out = { ok: true, batch: id, requeued: res.requeued ?? [], refused: [...(res.refused ?? []), ...refused] };
1381
+ if (res.hosted !== undefined) { out.hosted = !!res.hosted; out.reopened = !!res.reopened; }
1374
1382
  print(`batch ${id}: ${out.requeued.map((n) => `requeued #${n}`).join(" ") || "requeued (none)"}${out.refused.length ? `; refused ${out.refused.map((x) => `${/^\d+$/.test(String(x.issue)) ? "#" : ""}${x.issue} (${x.reason})`).join(", ")}` : ""}`);
1383
+ if (out.requeued.length) print(hostedLine(id, res));
1375
1384
  return out;
1376
1385
  }
1377
1386
 
@@ -1962,7 +1971,10 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
1962
1971
  } catch { /* alerts never fail a batch */ }
1963
1972
  for (const l of report) sink.line(l);
1964
1973
  const ok = rows.every((r) => ["merged", "invalid", "decomposed"].includes(r.outcome));
1965
- try { batch.writeReport({ rows, crossLane, ok, reclaimed, report: report.join("\n") }); } catch { /* the report is a cache of the rows */ }
1974
+ // #4917: a batch with any pending issue is not finished: no report, so a resume or the daemon picks it up
1975
+ let unfinished = false;
1976
+ try { unfinished = batch.pendingIssues().length > 0 && !batch.readCancel(); } catch { unfinished = false; } // a cancel ends the batch for good
1977
+ if (!unfinished) try { batch.writeReport({ rows, crossLane, ok, reclaimed, report: report.join("\n") }); } catch { /* the report is a cache of the rows */ }
1966
1978
  return { rows, crossLane, batch: batch.id, reclaimed, ok };
1967
1979
  }
1968
1980
 
@@ -2106,6 +2118,7 @@ export async function scheddFromCli(argv, deps = {}) {
2106
2118
  const pend = b.pendingAdds?.length ? `, pending adds: ${b.pendingAdds.map((n) => `#${n}`).join(" ")}` : "";
2107
2119
  console.log(` ${b.id}${b.running ? "" : " (not hosted)"}: ${age}${pend}`);
2108
2120
  }
2121
+ for (const u of st.unhosted ?? []) console.log(`schedd: batch ${u.id} is not hosted but has ${u.pending.length} pending: ${u.pending.map((n) => `#${n}`).join(" ")} (forgedock orchestrate --resume ${u.id})`);
2109
2122
  return { running: true, ...st };
2110
2123
  }
2111
2124
  if (argv[0] === "watchdog") { // #4780: run by the schedd watchdog timer; signals a live daemon whose heartbeat went stale
package/bin/journey.mjs CHANGED
@@ -642,9 +642,9 @@ export function openUrl(url) {
642
642
  if (process.platform === "win32") {
643
643
  spawnSync("cmd", ["/c", "start", "", url], { stdio: "ignore", windowsHide: true });
644
644
  } else if (process.platform === "darwin") {
645
- spawnSync("open", [url], { detached: true, stdio: "ignore" });
645
+ spawnSync("open", [url], { detached: true, windowsHide: true, stdio: "ignore" });
646
646
  } else {
647
- spawnSync("xdg-open", [url], { detached: true, stdio: "ignore" });
647
+ spawnSync("xdg-open", [url], { detached: true, windowsHide: true, stdio: "ignore" });
648
648
  }
649
649
  } catch {
650
650
  // Best-effort — if the browser can't be opened, the journey continues.
@@ -191,6 +191,8 @@ export function openBatch(root, id) {
191
191
  }
192
192
  return { requeued, refused };
193
193
  },
194
+ /** #4917: the issues still `pending` in state.json (deferred or requeued included); [] when none or unreadable. */
195
+ pendingIssues: () => { const p = readJson(f("state.json"))?.pending; return Array.isArray(p) ? p.filter((n) => Number.isSafeInteger(n)) : []; },
194
196
  /** Every `--requeue` entry, in order; [] when missing or invalid. */
195
197
  readRequeued: () => readRequeueFile(f("requeue")),
196
198
  /** Entries the runner has not applied yet (a finished or unhosted batch with a pending requeue). */
@@ -265,7 +267,7 @@ export function launchLane({ batch, issue, args = [], bin = process.execPath, sc
265
267
  const out = openSync(batch.logFile(issue), "a", 0o600);
266
268
  try {
267
269
  // FORGEDOCK_BATCH_ID names the lane's resource ledger dir (~/.forge/ledger/<batch>/, #4179).
268
- const child = spawnFn(bin, [script, "run-issue", String(issue), ...args, "--result-file", rf], { cwd, env: { ...env, FORGEDOCK_BATCH_ID: batch.id }, detached: true, stdio: ["ignore", out, out] });
270
+ const child = spawnFn(bin, [script, "run-issue", String(issue), ...args, "--result-file", rf], { cwd, env: { ...env, FORGEDOCK_BATCH_ID: batch.id }, detached: true, windowsHide: true, stdio: ["ignore", out, out] });
269
271
  child.on?.("error", () => { /* surfaces as a dead pid without a result */ });
270
272
  child.unref?.();
271
273
  return { pid: child.pid };
@@ -288,6 +288,8 @@ describe("lane launch env (#4179)", () => {
288
288
  assert.equal(pid, 99);
289
289
  assert.equal(spawned[0].opts.env.FORGEDOCK_BATCH_ID, batch.id);
290
290
  assert.equal(spawned[0].opts.env.KEEP, "1");
291
+ assert.equal(spawned[0].opts.windowsHide, true, "a detached lane must not open a console window on Windows");
292
+ assert.equal(spawned[0].opts.detached, true);
291
293
  assert.deepEqual(spawned[0].argv, ["/x/forgedock.mjs", "run-issue", "7", "--repo", "o/r", "--result-file", batch.resultFile(7)]);
292
294
  });
293
295
  });
@@ -888,6 +890,20 @@ describe("#4784: usage-limit deferrals", () => {
888
890
  b.store.save({ order: [1], results: { 1: { outcome: "deferred" } } });
889
891
  assert.deepEqual(b.requeue([1]).requeued, [1]);
890
892
  });
893
+ it("#4957: a blocker deferred on a usage limit past maxDeferrals keeps its dependents pending; the chain runs in order after the hold", async () => {
894
+ const { c, now, sleep } = fakeClock();
895
+ const bodies = { 1: "", 2: "Depends on #1", 3: "Depends on #2" };
896
+ const starts = [];
897
+ let limited = 0;
898
+ const rows = await orchestrate({ issues: [1, 2, 3], parallel: 3, readBody: async (n) => bodies[n], now, sleep, maxDeferrals: 3,
899
+ runOne: async (n) => {
900
+ starts.push(n);
901
+ if (n === 1 && limited++ < 5) return { terminalReason: "deferred", code: "rate_limited", resetAtEpochMs: c.t + 600_000, detail: "usage limit hit" };
902
+ return { terminalReason: "merged" };
903
+ } });
904
+ assert.deepEqual(starts, [1, 1, 1, 1, 1, 1, 2, 3], "#2 and #3 never start before #1 merges");
905
+ assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[1, "merged"], [2, "merged"], [3, "merged"]]);
906
+ });
891
907
  it("a requeuable deferred row: an issue that gave up after N deferrals (#4984)", async () => {
892
908
  const { now, sleep } = fakeClock();
893
909
  const rows = await orchestrate({ issues: [1], readBody: async () => "", now, sleep, maxDeferrals: 3,
@@ -967,6 +983,27 @@ describe("#4550: forgedock orchestrate --requeue", () => {
967
983
  assert.equal(b.readRequeued()[0].reason, "decided");
968
984
  assert.equal(b.readReport(), null, "the finished batch is reopened for --resume");
969
985
  });
986
+
987
+ it("#5000: --add/--requeue on a finished batch without a daemon clear the report and say it is not hosted", async () => {
988
+ const root = tmpDir("batch-add-finished-");
989
+ const b = openBatch(root, "b1");
990
+ b.writePlan({ argv: ["1"], repo: "o/r" });
991
+ b.store.save({ order: [5], results: { 5: { outcome: "needs-human" } } });
992
+ b.writeReport({ rows: [], report: "x" });
993
+ const io = { gh: async () => JSON.stringify({ state: "OPEN", labels: [], body: "", author: { login: "a" } }) };
994
+ const lines = [];
995
+ const run = async (argv) => { const l = console.log; console.log = (x) => lines.push(String(x)); try { return await orchestrateFromCli(argv, { batchRoot: root, io }); } finally { console.log = l; } };
996
+ const add = await run(["--add", "b1", "7"]);
997
+ assert.deepEqual(add.added, [7]);
998
+ assert.equal(b.readReport(), null, "a plain add reopens the finished batch for --resume");
999
+ assert.ok(lines.some((l) => /NOT hosted.*--resume b1/.test(l)), "the CLI says nothing is scheduled yet");
1000
+ b.writeReport({ rows: [], report: "x" });
1001
+ lines.length = 0;
1002
+ const rq = await run(["--requeue", "b1", "5"]);
1003
+ assert.deepEqual(rq.requeued, [5]);
1004
+ assert.equal(b.readReport(), null);
1005
+ assert.ok(lines.some((l) => /NOT hosted.*--resume b1/.test(l)));
1006
+ });
970
1007
  });
971
1008
 
972
1009
  describe("#4540: remote admissions are capped to the provider's free capacity", () => {
@@ -328,7 +328,7 @@ const shellGlobRe = (g) => new RegExp(`^${String(g).replace(/[.+^${}()|[\]\\]/g,
328
328
  const AMEND_KINDS = ["add-deliverable", "add-requirement", ...TRACKED, "amend-requirement", "supersedes", "decision", "substitute-validate"];
329
329
  const UNRELATED_CI_KIND = "unrelated-ci-failure";
330
330
  const GUARD_ID_RE = /^#?\d+$/;
331
- const AMEND_SLIPS = { "amend-requirement": "defer", limitation: "defer" };
331
+ const AMEND_SLIPS = { "amend-requirement": "defer", limitation: "defer", deferred: "defer", deferral: "defer", "scope-deferral": "defer" };
332
332
  // #4916: near-synonyms builders emit for the per-guard `supersedes` amendment
333
333
  const SUPERSEDES_SYNONYMS = ["supersede-guard", "supersede", "supersedes-guard"];
334
334
 
@@ -338,6 +338,11 @@ function normalizeAmendment(a) {
338
338
  const n = { ...a };
339
339
  if (blank(n.kind) && !blank(n.type)) n.kind = n.type;
340
340
  // #4594: amend-requirement with a revised `text` is a real amendment; without one it stays the slip for defer
341
+ // #4856: kinds compare case-insensitively with `_` as `-`; only a recognised kind or alias is rewritten, so unknown kinds keep their spelling in the error
342
+ if (typeof n.kind === "string") {
343
+ const k = n.kind.trim().toLowerCase().replace(/_/g, "-");
344
+ if (AMEND_KINDS.includes(k) || Object.hasOwn(AMEND_SLIPS, k)) n.kind = k;
345
+ }
341
346
  if (Object.hasOwn(AMEND_SLIPS, n.kind) && !(n.kind === "amend-requirement" && !blank(n.text))) n.kind = AMEND_SLIPS[n.kind];
342
347
  if (SUPERSEDES_SYNONYMS.includes(n.kind)) n.kind = "supersedes";
343
348
  if (TRACKED.includes(n.kind) && blank(n.requirement) && /^R\d+$/.test(String(n.id ?? ""))) n.requirement = n.id;
@@ -568,6 +573,18 @@ export function normalizeRunnerCommand(command) {
568
573
  export function cliUsageError({ rc, output } = {}) {
569
574
  return rc !== 0 && CLI_USAGE_RE.test(String(output ?? ""));
570
575
  }
576
+ const FORGEDOCK_ENTRY_RE = /^\/.*\/forgedock\/bin\/(?:lean\/checks|forgedock)\.mjs$/;
577
+ const FORGEDOCK_TOKEN_RE = /(?<![\w./-])\/[^\s'"`;&|()]*\/forgedock\/bin\/(lean\/checks|forgedock)\.mjs/g;
578
+ /**
579
+ * #5005: a frozen validate command names forgedock's own entry point by the absolute path of the install that wrote it; on
580
+ * the host that runs it that path may not exist. Rewrites each such token whose file is missing to this install's path.
581
+ */
582
+ export function resolveForgedockPaths(command, { exists = () => true, checksCli, forgedockCli } = {}) {
583
+ return String(command ?? "").replace(FORGEDOCK_TOKEN_RE, (tok, which) => {
584
+ const here = which === "forgedock" ? forgedockCli : checksCli;
585
+ return here && !exists(tok) ? here : tok;
586
+ });
587
+ }
571
588
  const KIND_NAME = { command: "command", "python-module": "Python module", "node-module": "Node package" };
572
589
  /**
573
590
  * forge#4323: the host tool or module a failed command could not find, from its {rc, output}: bash/sh "X: command not found"
@@ -581,6 +598,7 @@ export function missingTool({ rc, output } = {}, { repoHas = () => false, comman
581
598
  const py = PY_MISSING_RE.exec(out);
582
599
  if (py) { const tool = String(py[1] ?? py[2]).split(".")[0]; return local(tool) ? null : { tool, kind: "python-module" }; }
583
600
  const js = NODE_MISSING_RE.exec(out);
601
+ if (js && FORGEDOCK_ENTRY_RE.test(js[1])) return { tool: "forgedock", kind: "command" }; // #5005: the host's forgedock install is not at the frozen path
584
602
  if (js) {
585
603
  const spec = js[1], scoped = spec.startsWith("@");
586
604
  if (!scoped && spec.includes("/")) return null;
@@ -3,7 +3,7 @@
3
3
  import { readFileSync } from "node:fs";
4
4
  import { describe, it } from "node:test";
5
5
  import assert from "node:assert/strict";
6
- import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
6
+ import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables } from "./conformance.mjs";
7
7
 
8
8
  const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
9
9
  const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -69,6 +69,22 @@ describe("amendments", () => {
69
69
  });
70
70
  });
71
71
 
72
+ describe("amendment kind aliases (#4856)", () => {
73
+ it("deferral spellings apply as a tracked defer of R1", () => {
74
+ for (const kind of ["deferred", "deferral", "scope_deferral", "Scope-Deferral"]) {
75
+ const a = applyAmendments(C, [{ kind, requirement: "R1", why: "w" }]);
76
+ assert.deepEqual(a.problems, [], kind);
77
+ assert.equal(a.contract.requirements.find((r) => r.id === "R1").disposition, "defer", kind);
78
+ assert.deepEqual(a.toTrack.map((r) => r.id), ["R1"], kind);
79
+ }
80
+ });
81
+ it("scope-note and other unlisted kinds stay unknown and list the valid kinds", () => {
82
+ for (const kind of ["scope-note", "Scope_Note", "flaky-note"]) {
83
+ assert.match(applyAmendments(C, [{ kind, requirement: "R1", why: "w" }]).problems[0], new RegExp(`unknown kind '${kind}'; valid kinds are add-deliverable`));
84
+ }
85
+ });
86
+ });
87
+
72
88
  describe("#4655 multi-requirement amendment references", () => {
73
89
  it("R1/R2, 'R1, R2' and an en-dash range split into one applied row per id", () => {
74
90
  for (const requirement of ["R1/R2", "R1, R2", "R1\u2013R2", "R1 and R2"]) {
@@ -281,6 +297,22 @@ describe("precheck (deterministic, no reviewer)", () => {
281
297
  assert.equal(missingTool({ rc: 1, output: "ModuleNotFoundError: No module named 'app'" }, { repoHas: repoTracks(["app.py"]) }), null);
282
298
  assert.equal(missingTool({ rc: 1, output: "1 test failed: expected not found" }), null, "rc != 127 needs command not found");
283
299
  });
300
+ it("#5005 a missing forgedock install is env.tool_missing, not conformance; other absolute paths stay code defects", () => {
301
+ const out = "node:internal/modules/cjs/loader\nError: Cannot find module '/opt/node/lib/node_modules/forgedock/bin/lean/checks.mjs'\n code: 'MODULE_NOT_FOUND'";
302
+ assert.deepEqual(missingTool({ rc: 1, output: out }), { tool: "forgedock", kind: "command" });
303
+ assert.equal(missingTool({ rc: 1, output: "Error: Cannot find module '/repo/forgedock/src/x.mjs'" }), null);
304
+ const contract = { requirements: [], deliverables: [], validate: [{ id: "V2", command: "node /opt/node/lib/node_modules/forgedock/bin/lean/checks.mjs --base origin/staging" }] };
305
+ const result = { validation: [{ id: "V2", command: "", exit: 1 }] };
306
+ const pc = precheck({ contract, result, changedFiles: [], validateRuns: { V2: { rc: 1, output: out } } });
307
+ assert.deepEqual(pc.failures.map((f) => f.code), ["env.tool_missing"]);
308
+ });
309
+ it("#5005 resolveForgedockPaths rewrites a missing forgedock entry point to this install, leaves an existing one", () => {
310
+ const o = { checksCli: "/h/forgedock/bin/lean/checks.mjs", forgedockCli: "/h/forgedock/bin/forgedock.mjs" };
311
+ const cmd = "node /opt/node/lib/node_modules/forgedock/bin/lean/checks.mjs --base origin/staging && node /opt/node/lib/node_modules/forgedock/bin/forgedock.mjs check";
312
+ assert.equal(resolveForgedockPaths(cmd, { ...o, exists: () => false }), "node /h/forgedock/bin/lean/checks.mjs --base origin/staging && node /h/forgedock/bin/forgedock.mjs check");
313
+ assert.equal(resolveForgedockPaths(cmd, { ...o, exists: () => true }), cmd);
314
+ assert.equal(resolveForgedockPaths("node bin/x.mjs", { ...o, exists: () => false }), "node bin/x.mjs");
315
+ });
284
316
  it("a deliver requirement reported deferred without an amendment -> untracked_deferral; partial -> unimplemented", () => {
285
317
  const r = { ...res, requirements: [{ id: "R1", status: "deferred" }, { id: "R2", status: "partial" }] };
286
318
  assert.deepEqual(codes(precheck({ contract: C, result: r, changedFiles: files })), ["conformance.untracked_deferral", "conformance.unimplemented"]);
@@ -254,7 +254,7 @@ export function runCheckCommand({ command, cwd, timeoutMs = 300000, env = null }
254
254
  return new Promise((done) => {
255
255
  let out = "", timedOut = false, child;
256
256
  const add = (b) => { out += String(b); if (out.length > 2 * CHECK_OUTPUT_CAP) out = out.slice(-CHECK_OUTPUT_CAP); };
257
- try { child = spawn("bash", ["-c", String(command)], { cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], ...(env ? { env: { ...process.env, ...env } } : {}) }); }
257
+ try { child = spawn("bash", ["-c", String(command)], { cwd, detached: true, windowsHide: true, stdio: ["ignore", "pipe", "pipe"], ...(env ? { env: { ...process.env, ...env } } : {}) }); }
258
258
  catch (e) { done({ rc: 127, output: String(e?.message ?? e) }); return; }
259
259
  const kill = () => { try { process.kill(-child.pid, "SIGKILL"); } catch { try { child.kill("SIGKILL"); } catch { /* gone */ } } };
260
260
  const timer = setTimeout(() => { timedOut = true; kill(); }, timeoutMs);
@@ -528,15 +528,18 @@ describe("execution budget and sandbox cost (#4255)", () => {
528
528
  const prov = fakeProvider({ finish: (n) => ({ terminalReason: "merged", pr: 100 + n, cost_usd: 2 }), usage: async (h) => ({ minutes: 5 * h.issue, cost_usd: 0.1 * h.issue }) });
529
529
  const events = [];
530
530
  const file = join(work, "day.json");
531
- const res = await quiet(() => orchestrateFromCli(["1", "2", "3", "--repo", "o/r", "--no-prefilter"], base(work, { batchId: "bc1",
531
+ const printed = []; // #4917: #3 stays pending (budget hold), so the batch is not finalised: no report.json, the printed report carries the sandbox lines
532
+ const log0 = console.log; console.log = (l) => { printed.push(String(l)); };
533
+ const res = await (async () => { try { return await orchestrateFromCli(["1", "2", "3", "--repo", "o/r", "--no-prefilter"], base(work, { batchId: "bc1",
532
534
  execution: parseExecution({ mode: "remote", max_remote: "1", budget_minutes: "15" }), sandboxProvider: prov,
533
535
  sandboxLedger: (n) => makeLedger({ root: ledRoot, batch: "bc1", lane: `sandbox-${n}`, owner: "t", issue: n }),
534
- executionBudget: openExecutionBudget({ budget: parseExecution({ budget_minutes: "15" }).budget, file }), emitTelemetry: (rec) => { events.push(rec); } })));
536
+ executionBudget: openExecutionBudget({ budget: parseExecution({ budget_minutes: "15" }).budget, file }), emitTelemetry: (rec) => { events.push(rec); } })); } finally { console.log = log0; } })();
535
537
  assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.sandbox?.minutes ?? null]), [[1, "merged", 5], [2, "merged", 10], [3, "held", null]]);
536
- const rep = openBatch(join(work, "batches"), "bc1").readReport();
537
- assert.deepEqual(rep.rows[1].sandbox, { minutes: 10, cost_usd: 0.2, attempts: 1 });
538
- assert.match(rep.report, /sandbox: 2 lanes, 15\.0 min, \$0\.30 \(token cost \$4\.00\); budget: batch 15 min/);
539
- assert.match(rep.report, /sbx-min/);
538
+ assert.equal(openBatch(join(work, "batches"), "bc1").readReport(), null);
539
+ assert.deepEqual(res.rows[1].sandbox, { minutes: 10, cost_usd: 0.2, attempts: 1 });
540
+ const text = printed.join("\n");
541
+ assert.match(text, /sandbox: 2 lanes, 15\.0 min, \$0\.30 \(token cost \$4\.00\); budget: batch 15 min/);
542
+ assert.match(text, /sbx-min/);
540
543
  assert.deepEqual(events.filter((e) => e.event === "SANDBOX_USAGE").map((e) => [e.issue, e.minutes, e.cost_usd, e.batch]), [[1, 5, 0.1, "bc1"], [2, 10, 0.2, "bc1"]]);
541
544
  assert.equal(Object.keys(JSON.parse(readFileSync(file, "utf8")).days[new Date().toISOString().slice(0, 10)]).length, 2);
542
545
  });
@@ -366,7 +366,7 @@ function realIo(repo, env = process.env) {
366
366
  },
367
367
  launch: (n, lane) => {
368
368
  let res = null;
369
- const p = spawn(process.execPath, [CLI, "run-issue", String(n), "--lane", lane], { detached: true, stdio: "ignore", cwd: process.cwd(), env });
369
+ const p = spawn(process.execPath, [CLI, "run-issue", String(n), "--lane", lane], { detached: true, windowsHide: true, stdio: "ignore", cwd: process.cwd(), env });
370
370
  p.on("close", (code) => { res = { code: code ?? 1 }; }); p.on("error", () => { res = { code: 1 }; });
371
371
  return { poll: () => res };
372
372
  },
@@ -375,7 +375,7 @@ function realIo(repo, env = process.env) {
375
375
  reported: (id) => existsSync(join(root, id, "report.json")),
376
376
  start: async (issues, parallel, lane) => {
377
377
  const before = new Set(listBatches(root));
378
- spawn(process.execPath, [CLI, "orchestrate", ...issues.map(String), "--parallel", String(parallel), "--lane", lane, "--repo", repo], { detached: true, stdio: "ignore", cwd: process.cwd(), env }).unref();
378
+ spawn(process.execPath, [CLI, "orchestrate", ...issues.map(String), "--parallel", String(parallel), "--lane", lane, "--repo", repo], { detached: true, windowsHide: true, stdio: "ignore", cwd: process.cwd(), env }).unref();
379
379
  for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); }
380
380
  throw new Error("orchestrate did not create a batch within 60s");
381
381
  },
package/bin/lean/lean.mjs CHANGED
@@ -5,6 +5,7 @@
5
5
  * ForgeDock registers everything on GitHub (labels, one register comment, PR, merge). No investigation, synthesis,
6
6
  * severity taxonomy, risk tiers or local CI gates. All effects are injected through `io` so it is unit-testable.
7
7
  */
8
+ import { existsSync } from "node:fs";
8
9
  import { join, dirname, basename, delimiter } from "node:path";
9
10
  import { hostname as osHostname } from "node:os";
10
11
  import { appendEvent } from "../engine/runlog.mjs";
@@ -14,7 +15,7 @@ import { ciPollDelayMs, ENGINE_VERSION, DEFAULT_MAX_SESSION_LIMIT_PAUSES, DEFAUL
14
15
  import { openDependency, laneMergedPrs, dependencyOnLane, isTrustedComment, decomposedSubissues, isDecomposedRecord, reopenRevokedDecomposition } from "../engine/phases.mjs";
15
16
  import { createHash, randomBytes } from "node:crypto";
16
17
  import { otherLaneOwner } from "./lanelock.mjs";
17
- import { describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
18
+ import { resolveForgedockPaths, describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, precheck, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
18
19
  import { checkAdmission, hostParallelism, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
19
20
  import { namedPaths, isTrustedIssueComment } from "./prefilter.mjs";
20
21
  import { checksAt } from "./postdeploy-check.mjs";
@@ -37,7 +38,7 @@ import { hostCapabilityBlock, contractHostNeeds } from "./execution.mjs";
37
38
  const MISSING_TOOL_RE = /command not found|not found in PATH|no such file or directory.*\b(?:docker|promtool)\b|\b(?:docker|promtool)\b[^\n]{0,80}(?:not (?:installed|found|available)|no such file|cannot connect|daemon)|cannot connect to the docker daemon|no docker daemon|no module named|ModuleNotFoundError|\bENOENT\b[^\n]{0,60}spawn/i;
38
39
  /** #4685: true when a builder's block summary names a missing tool or module (environment), not a credential. */
39
40
  export function missingToolBlock(summary) { return MISSING_TOOL_RE.test(String(summary ?? "")); }
40
- import { REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, investigationVerdict, decisionNote, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, unrunnableValidateNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, sequenceSyncPrompt, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt, i18nCopiesPrompt } from "./prompts.mjs";
41
+ import { CHECKS_CLI, FORGEDOCK_CLI, REVIEWER_SYSTEM, LEAN_DIR, RESULT_FILE, INVESTIGATION_FILE, IMPLEMENTATION_FILE, CONTRACT_FILE, builderPlanPrompt, builderImplementPrompt, contractCorrectionPrompt, conformanceResumePrompt, reviewerPrompt, investigationVerdict, decisionNote, reviewerRepairPrompt, REVIEW_DIFF_MAX_BYTES, blockCorrectionPrompt, HUMAN_BLOCK_REASONS, structurePrompt, hotspotPlanNote, hotspotBuildRule, reproPlanNote, unrunnableValidateNote, contextPackNote, docsPrompt, ciFailurePrompt, ciUnavailablePrompt, precheckFailurePrompt, contextRestartPrompt, turnBudgetWrapupPrompt, syncChecks, GROUPING_SYSTEM, enumeratingPlanNote, enumeratingBuildRule, invariantsPlanNote, invariantsBuildRule, missingEntriesPrompt, sequencePrompt, sequenceSyncNote, sequenceSyncPrompt, fixSessionPrompt, postMergeFailurePrompt, duplicateExportsPrompt, migrationGeneratorPrompt, i18nCopiesPrompt } from "./prompts.mjs";
41
42
  import { postMergeCommands } from "./postmerge.mjs";
42
43
  import { guardAfterMerge, withHistoryRows } from "./guards.mjs";
43
44
  import { loadGuards, selectGuards, runGuards, supersededIds, supersessionComment } from "./guardgate.mjs";
@@ -356,7 +357,7 @@ export async function runCore(o) {
356
357
  }
357
358
  async function heavyStep(phase, fn) { const g = await heavyAcquire(phase); try { return await fn(); } finally { try { g?.release?.(); } catch { /* fail-open */ } } }
358
359
  // ---- lane resource ledger (#4179): every owned resource is registered before it is created, released after its teardown ----
359
- let ledger = null, deleteLocalBranch = false;
360
+ let ledger = null, deleteLocalBranch = false, laneHead = null; // #4958: laneHead = {branch, sha, pushedTo}
360
361
  const ledgerDo = (fn) => { if (!ledger) return; try { fn(ledger); } catch (e) { ev({ event: "LEDGER_ERROR", error: String(e?.message ?? e).slice(0, 300) }); } };
361
362
  const protectedBranch = (b) => isProtectedLaneBranch(b, { issue, lane, branches, defaults });
362
363
  /** Teardown from the ledger (#4179), on every exit path: builder pids, checkpoint of unpushed work, run tmp, worktree, branch, docker. */
@@ -366,6 +367,7 @@ export async function runCore(o) {
366
367
  }
367
368
  // Never discard work silently: an unmerged lane commits and pushes what is left before its worktree goes.
368
369
  const saved = !wtCreated || reason === "merged" || (await checkpoint(`exit (${reason})`));
370
+ await reportLaneHead(reason);
369
371
  try { ghCache?.dispose?.(); } catch { /* best effort */ }
370
372
  if (runTmp) {
371
373
  try { if ((await io.runTmp.remove(runTmp)) !== false) ledgerDo((l) => l.release("tmp", runTmp)); } catch { /* best effort */ }
@@ -449,7 +451,7 @@ export async function runCore(o) {
449
451
  // forge#4165: pr, milestone and the conformance verdict ride on the existing RUN_TERMINAL write (verdict-mix queries).
450
452
  const term = { event: "RUN_TERMINAL", reason, code, ...(extra.pr ? { pr: extra.pr } : {}), ...(milestone ? { milestone } : {}),
451
453
  conformance_mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", ...(shadowWould ? { shadow_would: shadowWould } : {}),
452
- context_pack: ctxCfg.mode, ...ctxAb, ...expTag, explore: exploreCfg.mode, wall_ms: now() - t0, ...(guardAddedMs ? { guard_added_ms: guardAddedMs } : {}), model_ms: modelMs, light, ...(ctxLane ? { lane_type: ctxLane } : {}), ...(pitMatched != null ? { pitfalls_matched: pitMatched } : {}), ...(tb.record() ? { turns: tb.record() } : {}),
454
+ ...(laneHead && reason !== "merged" ? { lane_branch: laneHead.pushedTo ?? "unpushed", lane_sha: laneHead.sha } : {}), context_pack: ctxCfg.mode, ...ctxAb, ...expTag, explore: exploreCfg.mode, wall_ms: now() - t0, ...(guardAddedMs ? { guard_added_ms: guardAddedMs } : {}), model_ms: modelMs, light, ...(ctxLane ? { lane_type: ctxLane } : {}), ...(pitMatched != null ? { pitfalls_matched: pitMatched } : {}), ...(tb.record() ? { turns: tb.record() } : {}),
453
455
  ...(route ? { builder_tier: route.tier, builder_class: route.class, builder_escalated: escalated, builder_model: route.model ?? "default" } : {}) }; // forge#4275: the on/off A/B groups runs by these; #4394: per-tier stats
454
456
  const phases = phaseLedger.snapshot(); // #4908: run log gets the nested ledger, FORGE:TELEMETRY the flattened scalars
455
457
  ev({ ...term, phases });
@@ -477,7 +479,7 @@ export async function runCore(o) {
477
479
  }
478
480
  await releaseLease(reason, code, extra.mergedTo ?? null);
479
481
  done = { terminalReason: reason, detail, branch, worktree: wt, cost_usd: usage.cost_usd, tokens: { input: usage.input, output: usage.output, cache: usage.cache }, wall_ms: now() - t0, model_ms: modelMs, phases: phaseLedger.snapshot(),
480
- light, ...(lightVerdict ? { light_eligibility: lightVerdict } : {}), ...(lightUpgrade ? { light_upgrade: lightUpgrade } : {}), // #4453
482
+ ...(laneHead && reason !== "merged" ? { laneHead: { ...laneHead } } : {}), light, ...(lightVerdict ? { light_eligibility: lightVerdict } : {}), ...(lightUpgrade ? { light_upgrade: lightUpgrade } : {}), // #4453
481
483
  ...(Object.keys(waits).length ? { waits: { ...waits } } : {}), ...(heavyQueueMax ? { heavyQueueMax } : {}), ...(precheckRan ? { precheckMs, precheckDeferred } : {}),
482
484
  ...(firstPassOf(builderTriggers) != null ? { firstPass: firstPassOf(builderTriggers) } : {}), ...(tb.record() ? { turns: tb.record() } : {}), // #4392
483
485
  ...(route ? { builderRoute: { tier: route.tier, class: route.class, escalated, model: route.model ?? "default" } } : {}),
@@ -641,7 +643,7 @@ export async function runCore(o) {
641
643
  try {
642
644
  const r = await ensureSignedOff({ git, lane, branch });
643
645
  if (r.rewritten) { T.decisions.push(`Rewrote ${r.rewritten} unpushed commit(s) with the lane identity and a sign-off (placeholder author or no Signed-off-by, #4651)`); ev({ event: "DCO_SIGNOFF_REWRITE", count: r.rewritten }); }
644
- } catch (e) { throw Object.assign(e, { laneCode: OUTCOME_CODES.CI_DCO }); }
646
+ } catch (e) { throw Object.assign(e, e?.noIdentity === true ? { laneCode: OUTCOME_CODES.CI_DCO } : { engineCode: OUTCOME_CODES.ENGINE_DCO_REWRITE }); } // #4997: a failed rewrite is mechanical (engine-error, recoverable), only a missing identity is a human block
645
647
  };
646
648
  const gitPush = async (args) => {
647
649
  if (!args.includes("--delete")) { await untrackToolState({ git, base, ev, keep: await keepPaths() }); await signOffUnpushed(); } // #4590: never push .forge-lean/ or tool caches this branch added
@@ -659,10 +661,33 @@ export async function runCore(o) {
659
661
  const dirty = String(await git(["status", "--porcelain"])).trim();
660
662
  if (dirty) { await git(["add", "-A"]); if (await unstageToolState(git)) await git(["commit", "--no-verify", "-m", `wip(forge): checkpoint after ${reason}`]); } // #4590: tool state is never staged
661
663
  const ahead = String(await git(["diff", "--name-only", `${base}...HEAD`])).trim();
662
- if (ahead) await gitPush(["push", "-u", "origin", branch]);
664
+ if (ahead) { laneHead = { branch, sha: String(await git(["rev-parse", "HEAD"])).trim(), pushedTo: null }; await gitPush(["push", "-u", "origin", branch]); laneHead.pushedTo = branch; }
663
665
  ev({ event: "CHECKPOINT", reason, dirty: Boolean(dirty), pushed: Boolean(ahead) });
664
666
  return true;
665
- } catch (e) { ev({ event: "CHECKPOINT_FAILED", reason, error: String(e?.message ?? e).slice(0, 300) }); return false; }
667
+ } catch (e) { ev({ event: "CHECKPOINT_FAILED", reason, error: String(e?.message ?? e).slice(0, 300) }); return laneHead ? await rescuePush(reason, e?.pushRejected) : false; }
668
+ }
669
+ /** #4958: the normal push failed (sign-off rewrite, rejection): push HEAD as is to the lane branch, else to a salvage branch, so a torn-down remote sandbox never takes the commits with it. */
670
+ async function rescuePush(reason, rejected) {
671
+ // #5014: gitPush untracks tool state and rewrites unpushed commits for DCO before it pushes, so HEAD may have moved since the capture
672
+ try { const now = String(await git(["rev-parse", "HEAD"])).trim(); if (now) laneHead.sha = now; } catch { /* keep the captured sha */ }
673
+ // a rejected lane-branch push (protected branch, permissions) would be rejected again: go straight to the salvage branch
674
+ for (const ref of [...(rejected ? [] : [branch]), `forge-salvage/${issue}-${laneHead.sha.slice(0, 10)}`]) {
675
+ try {
676
+ await git(["push", "origin", `${laneHead.sha}:refs/heads/${ref}`]);
677
+ laneHead.pushedTo = ref;
678
+ ev({ event: "CHECKPOINT_RESCUED", reason, ref, sha: laneHead.sha });
679
+ return !String(await git(["status", "--porcelain"])).trim(); // an uncommitted remainder keeps the worktree
680
+ } catch (e) { ev({ event: "CHECKPOINT_RESCUE_FAILED", reason, ref, error: String(e?.message ?? e).slice(0, 300) }); }
681
+ }
682
+ return false;
683
+ }
684
+ /** #4958: where the lane's commits ended up, for the terminal comment, RUN_TERMINAL and the returned result. */
685
+ async function reportLaneHead(reason) {
686
+ if (!laneHead || reason === "merged") return;
687
+ const { branch: b, sha, pushedTo } = laneHead;
688
+ const line = pushedTo ? `Lane commits pushed: branch \`${pushedTo}\` at ${sha.slice(0, 10)}; a rerun or --requeue resumes from it` : `Lane commits NOT pushed: ${sha.slice(0, 10)} on \`${b}\` exists only in this lane's worktree and is lost if its sandbox is torn down`;
689
+ T.anomalies.push(line);
690
+ if (!["invalid", "needs-human", "budget-exceeded"].includes(reason)) { try { await post("FORGE:LANE_BRANCH", `## Lane branch\n${line}`); } catch { /* best effort */ } }
666
691
  }
667
692
  const CONTINUE_NOTE = "\n\nThe branch already holds work from an earlier session that was interrupted (see the latest `wip(forge): checkpoint` commit). Continue from it: read `git log` and `git diff origin/BASE...HEAD`, do not start over.";
668
693
  const recoverNote = () => (recover ? `\n\nRecovery pass (${recover.code}): ${(RECOVER_HOW[recover.code] ?? "an earlier run ended needs-human for a mechanical reason; resolve it").replaceAll("BASE", lane)}.` : "");
@@ -1054,6 +1079,10 @@ export async function runCore(o) {
1054
1079
  const prs = JSON.parse(String(await ghc(["pr", "list", "-R", repo, "--head", branch, "--state", "open", "--json", "number,headRefOid,baseRefName"])) || "[]");
1055
1080
  const remote = String(await io.git(["ls-remote", "--heads", "origin", branch], { cwd })).trim();
1056
1081
  if (prs.length || remote) resumed = { pr: prs[0]?.number ?? null, base: prs[0]?.baseRefName ?? null };
1082
+ if (!prs.length && !remote) { // #4958: commits the exit rescue could only push to a salvage branch resume from there
1083
+ const salv = String(await io.git(["ls-remote", "--heads", "origin", `refs/heads/forge-salvage/${issue}-*`], { cwd })).split("\n").map((l) => l.trim().split(/\s+/)[1]).filter(Boolean).sort().at(-1);
1084
+ if (salv) { await io.git(["fetch", "origin", `${salv}:refs/remotes/origin/${branch}`], { cwd }); resumed = { pr: null, base: null }; ev({ event: "RESUME_FROM_SALVAGE", ref: salv }); }
1085
+ }
1057
1086
  } catch { /* no resume info: start fresh */ }
1058
1087
  // An open PR on another base (a hand-retargeted PR, or one opened before lane resolution) is never merged (#4125).
1059
1088
  // #4593: a shipped milestone lane (fully merged into staging) cannot take new work; follow the open PR's live base.
@@ -2181,6 +2210,8 @@ export async function runCore(o) {
2181
2210
  }
2182
2211
  const verifierSection = () => { const n = renderVerifierNote(verifierRun); return n ? `\n\n${n}` : ""; };
2183
2212
  /** #4323: rerun once each validate command the builder recorded as failed, with the lane env; {[id]: {rc, output}} or null. */
2213
+ /** #5005: a frozen forgedock entry-point path missing on this host is replaced by this install's own. */
2214
+ const forgedockResolved = (c) => resolveForgedockPaths(String(c), { exists: existsSync, checksCli: CHECKS_CLI, forgedockCli: FORGEDOCK_CLI });
2184
2215
  async function rerunFailedValidatesRaw(contract) {
2185
2216
  if (!io.runCommand) return null;
2186
2217
  const runs = Array.isArray(result?.validation) ? result.validation : [], out = {};
@@ -2189,11 +2220,11 @@ export async function runCore(o) {
2189
2220
  const r = runs.find((x) => x?.id === v.id);
2190
2221
  if (!r || r.exit === undefined || r.exit === 0) continue;
2191
2222
  let x;
2192
- try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: String(v.command), cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2223
+ try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: forgedockResolved(v.command), cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2193
2224
  // #4761: a runner usage error (vitest -q) is the frozen command's defect: try the normalised command once
2194
2225
  const fixed = cliUsageError(x ?? {}) ? normalizeRunnerCommand(v.command) : null;
2195
2226
  if (fixed && fixed !== String(v.command)) {
2196
- try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: fixed, cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2227
+ try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: forgedockResolved(fixed), cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2197
2228
  const note = `Validate ${v.id} hit a runner usage error; reran normalised (\`${fixed}\`): exit ${x?.rc}`;
2198
2229
  if (!T.decisions.includes(note)) T.decisions.push(note);
2199
2230
  }
@@ -2210,7 +2241,7 @@ export async function runCore(o) {
2210
2241
  for (const v of contract.validate ?? []) {
2211
2242
  if (!v.substituted?.from) continue;
2212
2243
  let x;
2213
- try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: String(v.substituted.from), cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2244
+ try { x = await heavyStep("precheck", async () => (await runWithTimeoutRetry((timeoutMs) => io.runCommand({ command: forgedockResolved(v.substituted.from), cwd: wt, timeoutMs, env }), vcfg.precheckTimeoutSeconds * 1000)).r); } catch (e) { x = { rc: 127, output: String(e?.message ?? e) }; }
2214
2245
  out[v.id] = { rc: Number.isInteger(x?.rc) ? x.rc : 1, output: String(x?.output ?? "") };
2215
2246
  }
2216
2247
  if (Object.keys(out).length) ev({ event: "SUBSTITUTE_RERUN", runs: Object.entries(out).map(([id, x]) => ({ id, rc: x.rc })) });
@@ -2937,6 +2968,7 @@ export async function runCore(o) {
2937
2968
  if (owner) return finish("deferred", `git push was rejected (non-fast-forward): ${owner.detail}; another lane owns ${branch}`, { code: OUTCOME_CODES.LEASE_HELD, branch }, OUTCOME_CODES.LEASE_HELD);
2938
2969
  }
2939
2970
  if (e?.pushRejected) return await needsHuman(`git push was rejected (no retry): ${String(e?.stderr ?? e?.message ?? e).slice(0, 400)}`);
2971
+ if (e?.engineCode) return finish("engine-error", e?.message ?? String(e), { code: e.engineCode }, e.engineCode); // #4997
2940
2972
  return finish("engine-error", e?.message ?? String(e));
2941
2973
  }
2942
2974
  }
@@ -3819,7 +3819,7 @@ describe("batch robustness (#4112-#4115)", () => {
3819
3819
  const w = world({ pushErrors: Array(10).fill("fatal: the remote end hung up unexpectedly\nerror: RPC failed; HTTP 503") });
3820
3820
  const r = await run(w);
3821
3821
  assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "push.transient");
3822
- assert.equal(w.pushes, 8, "4 for the PR push, then 4 by the exit checkpoint that never discards unpushed work (#4179)");
3822
+ assert.equal(w.pushes, 10, "4 for the PR push, 4 by the exit checkpoint that never discards unpushed work (#4179), 1 lane-branch rescue, 1 salvage rescue (#4958)");
3823
3823
  assert.equal(w.prs.length, 0);
3824
3824
  });
3825
3825
 
@@ -3827,12 +3827,63 @@ describe("batch robustness (#4112-#4115)", () => {
3827
3827
  for (const msg of ["remote: error: GH006: Protected branch update failed for refs/heads/x", "fatal: unable to access: The requested URL returned error: 403", "! [remote rejected] x -> x (pre-receive hook declined)"]) {
3828
3828
  const w = world({ pushErrors: [msg, msg, msg] });
3829
3829
  const r = await run(w);
3830
- assert.equal(w.pushes, 2, `${msg}: one rejected PR push, one rejected exit checkpoint (#4179), no retries`);
3830
+ assert.equal(w.pushes, 3, `${msg}: one rejected PR push, one rejected exit checkpoint (#4179), no retries; the third error rejects the salvage rescue (#4958)`);
3831
3831
  assert.equal(r.terminalReason, "needs-human");
3832
3832
  assert.equal(w.calls.filter((c) => c[1] === "worktree" && c[2] === "remove").length, 1, "the unpushed worktree is kept: only the setup remove ran");
3833
3833
  }
3834
3834
  });
3835
3835
 
3836
+ it("#4958 a failed exit push is rescued to the lane branch by sha and the branch and sha are reported", async () => {
3837
+ const odd = "fatal: unexpected sandbox git failure";
3838
+ const w = world({ pushErrors: [odd, odd] });
3839
+ const r = await run(w);
3840
+ assert.equal(w.pushes, 3, "PR push, failed exit checkpoint, rescue push");
3841
+ const rescue = w.calls.filter((c) => c[0] === "git" && c[1] === "push").at(-1);
3842
+ assert.match(rescue.join(" "), /origin [0-9a-z]+:refs\/heads\/feat\/issue-7-lean$/);
3843
+ assert.equal(r.laneHead.pushedTo, "feat/issue-7-lean"); assert.ok(r.laneHead.sha);
3844
+ assert.ok(w.telemetry.some((t) => t.lane_branch === "feat/issue-7-lean" && t.lane_sha === r.laneHead.sha));
3845
+ });
3846
+
3847
+ it("#5014 the rescue pushes and reports the head after the pre-push sign-off rewrite, not the sha captured before it", async () => {
3848
+ const odd = "fatal: unexpected sandbox git failure";
3849
+ const w = world({ pushErrors: [odd, odd] });
3850
+ const g = w.io.git;
3851
+ let pushes = 0;
3852
+ w.io.git = async (a, o2) => { if (a[0] === "push" && ++pushes === 2) w.head = "rewritten0000000000000000000000000000000"; return g(a, o2); }; // the exit push rewrote HEAD (DCO) then failed
3853
+ const r = await run(w);
3854
+ const rescue = w.calls.filter((c) => c[0] === "git" && c[1] === "push").at(-1);
3855
+ assert.match(rescue.join(" "), /origin rewritten0+:refs\/heads\/feat\/issue-7-lean$/);
3856
+ assert.equal(r.laneHead.sha, "rewritten0000000000000000000000000000000");
3857
+ assert.ok(w.telemetry.some((t) => t.lane_sha === "rewritten0000000000000000000000000000000"));
3858
+ });
3859
+
3860
+ it("#4958 a rejected lane-branch push is rescued to a forge-salvage branch and reported as pushed there", async () => {
3861
+ const msg = "remote: error: GH006: Protected branch update failed for refs/heads/x";
3862
+ const w = world({ pushErrors: [msg, msg] });
3863
+ const r = await run(w);
3864
+ assert.equal(w.pushes, 3, "PR push, rejected exit checkpoint, one salvage push (no second lane-branch attempt)");
3865
+ assert.match(w.calls.filter((c) => c[0] === "git" && c[1] === "push").at(-1).join(" "), /:refs\/heads\/forge-salvage\/7-/);
3866
+ assert.match(r.laneHead.pushedTo, /^forge-salvage\/7-/);
3867
+ assert.ok(traj(w).some((c) => /Lane commits pushed: branch `forge-salvage\/7-/.test(c)));
3868
+ });
3869
+
3870
+ it("#4958 when every rescue push is rejected the head is reported as NOT pushed", async () => {
3871
+ const msg = "remote: error: GH006: Protected branch update failed for refs/heads/x";
3872
+ const w = world({ pushErrors: [msg, msg, msg, msg] });
3873
+ const r = await run(w);
3874
+ assert.equal(r.laneHead.pushedTo, null);
3875
+ assert.ok(traj(w).some((c) => /Lane commits NOT pushed/.test(c)));
3876
+ });
3877
+
3878
+ it("#4958 a rerun with only a forge-salvage branch on origin resumes from it", async () => {
3879
+ const w = world({ prList: [] });
3880
+ const g = w.io.git;
3881
+ w.io.git = async (a, o2) => (a[0] === "ls-remote" && a.at(-1) === "refs/heads/forge-salvage/7-*" ? "abc\trefs/heads/forge-salvage/7-1234567890\n" : g(a, o2));
3882
+ await run(w);
3883
+ assert.ok(w.calls.some((c) => c[0] === "git" && c[1] === "fetch" && c.includes("refs/heads/forge-salvage/7-1234567890:refs/remotes/origin/feat/issue-7-lean")));
3884
+ assert.ok(w.calls.some((c) => c[0] === "git" && c[1] === "worktree" && c[2] === "add" && c.includes("origin/feat/issue-7-lean")), "the worktree is built from the salvaged head");
3885
+ });
3886
+
3836
3887
  it("#4118 GitHub's transient '[remote rejected] ... (failed)' and pack write errors are retried, then merge", async () => {
3837
3888
  for (const msg of ["To github.com:o/r.git\n ! [remote rejected] feat/issue-7-lean -> feat/issue-7-lean (failed)",
3838
3889
  "remote: error: unable to write file ./objects/pack/pack-1.pack: No such file or directory\n ! [remote rejected] x -> x (failed)"]) {
@@ -69,3 +69,19 @@ describe("#4773: Blocked by edges and the blocked label", () => {
69
69
  assert.deepEqual(started, [1]);
70
70
  });
71
71
  });
72
+
73
+ describe("#4917: a deferred issue still waiting when the last running lane ends", () => {
74
+ it("stays pending in the saved state (future notBefore) while the batch stops on the breaker", async () => {
75
+ let failLast; const gate = new Promise((r) => { failLast = r; });
76
+ let saved = null;
77
+ const store = { load: () => null, save: (st) => { saved = st; } };
78
+ await orchestrate({ issues: [1, 2], parallel: 2, pollMs: 5, readBody: async () => "", breaker: { errors: 1, window_min: 30 }, maxDeferrals: 3, store,
79
+ runOne: async (n) => {
80
+ if (n === 1) { setTimeout(failLast, 20); return { terminalReason: "deferred", code: "run.interrupted", retryInSeconds: 3600, cost_usd: 0 }; }
81
+ await gate; return { terminalReason: "engine-error", detail: "boom" };
82
+ } });
83
+ assert.ok(saved.pending.includes(1), JSON.stringify(saved.pending));
84
+ assert.ok(Number(saved.notBefore?.[1]) > Date.now(), JSON.stringify(saved.notBefore));
85
+ assert.deepEqual(saved.running, {});
86
+ });
87
+ });
@@ -179,7 +179,7 @@ export function probeEnv(allow = [], env = process.env) {
179
179
  export function spawnProbe({ run, cwd, env, timeoutMs }) {
180
180
  return new Promise((resolve) => {
181
181
  let stdout = "", stderr = "", timedOut = false;
182
- const child = nodeSpawn("sh", ["-c", run], { cwd, env, detached: true, stdio: ["ignore", "pipe", "pipe"] });
182
+ const child = nodeSpawn("sh", ["-c", run], { cwd, env, detached: true, windowsHide: true, stdio: ["ignore", "pipe", "pipe"] });
183
183
  const cap = (s, d) => (s + d).slice(-OUTPUT_TAIL * 4);
184
184
  child.stdout.on("data", (d) => { stdout = cap(stdout, d); });
185
185
  child.stderr.on("data", (d) => { stderr = cap(stderr, d); });
@@ -172,6 +172,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
172
172
  const run = Promise.resolve().then(() => runBatch({ argv, cwd, env, sink, keepBreaker, signal: ac.signal })).then((result) => {
173
173
  const report = ent.dir ? openBatch(root, ent.id).readReport()?.report ?? null : null;
174
174
  emit({ type: "done", result, report });
175
+ if (ent.id) rehostLater(ent.id); // #4917
175
176
  }, (e) => {
176
177
  const message = e?.message ?? String(e);
177
178
  if (e?.code === "batch.stopped") { log(`schedd: batch ${ent.id ?? argv.join(" ")} stopped (lanes keep running; resumed by the next daemon)`); if (!started) reject(new Error(message)); return; }
@@ -180,6 +181,28 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
180
181
  runs.set(run, ac);
181
182
  });
182
183
 
184
+ // #4917: a batch that ended with a deferred issue still waiting (no report) is re-hosted once its retry-at is near, without a restart
185
+ const rehostTimers = new Set();
186
+ const rehostLater = (id) => {
187
+ if (closing) return;
188
+ let b, st;
189
+ try { b = openBatch(root, id); if (b.readReport() || b.readCancel() || !b.pendingIssues().length) return; st = b.store.load(); } catch { return; }
190
+ if (st?.breaker && !st.breaker.transport) return; // a tripped breaker holds the batch for an operator --resume (a re-host would stop again at once)
191
+ const at = Math.min(...b.pendingIssues().map((n) => Number(st?.notBefore?.[n]) || Infinity));
192
+ if (!Number.isFinite(at)) return; // nothing waits on the clock: a deliberate hold stays for --resume
193
+ if (at <= Date.now()) return; // the run that just ended already saw this retry-at pass and did not admit: re-hosting would loop every second
194
+ const t = setTimeout(() => {
195
+ rehostTimers.delete(t);
196
+ if (closing || hosted.has(id) || starting.has(id)) return;
197
+ log(`schedd: re-hosting batch ${id} (pending issues waiting on a retry)`);
198
+ const started = host({ argv: ["--resume", id], cwd: process.cwd(), env: {}, keepBreaker: true });
199
+ starting.set(id, started);
200
+ started.catch((e) => log(`schedd: re-host ${id} failed: ${e?.message ?? e}`)).finally(() => starting.delete(id));
201
+ }, Math.min(60_000, Math.max(1000, at - Date.now())));
202
+ t.unref?.();
203
+ rehostTimers.add(t);
204
+ };
205
+
183
206
  /** #4775: wait for a batch start at most `admitMs`; the start itself goes on (a second host() would duplicate the batch). */
184
207
  const bound = (p, what) => {
185
208
  let t;
@@ -204,7 +227,15 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
204
227
  };
205
228
 
206
229
  const handle = async (req, send) => {
207
- if (req.op === "status") return send({ type: "status", pid, lastTick, batches: statusBatches() }, true);
230
+ if (req.op === "status") {
231
+ // #4917: a batch dir that still holds pending issues but is not hosted (finalised early, or never resumed)
232
+ const unhosted = [];
233
+ for (const id of listBatches(root)) {
234
+ if (hosted.has(id) || starting.has(id)) continue;
235
+ try { const pending = openBatch(root, id).pendingIssues(); if (pending.length) unhosted.push({ id, pending }); } catch { /* advisory */ }
236
+ }
237
+ return send({ type: "status", pid, lastTick, batches: statusBatches(), unhosted }, true);
238
+ }
208
239
  if (req.op === "set-parallel") {
209
240
  const b = openBatch(root, req.id);
210
241
  if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -215,8 +246,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
215
246
  const b = openBatch(root, req.id);
216
247
  if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
217
248
  const { added, skipped, requeued = [] } = b.addIssues(req.issues ?? []);
218
- // #4687: a re-queued skipped issue reopens a finished, unhosted batch like --requeue does
219
- const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
249
+ // #4687/#5000/#4917: a re-queued skipped issue or a newly added one reopens a finished, unhosted batch like --requeue does
250
+ const reopen = (added.length > 0 || requeued.length > 0 || b.unrunAdds().length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
220
251
  if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
221
252
  return send({ type: "ok", added, skipped, requeued, hosted: hosted.has(req.id), reopened: reopen }, true);
222
253
  }
@@ -225,7 +256,8 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
225
256
  const b = openBatch(root, req.id);
226
257
  if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
227
258
  const { requeued, refused } = b.requeue(req.issues ?? [], { reason: req.reason, by: req.by });
228
- const reopen = requeued.length > 0 && !hosted.has(req.id) && !starting.has(req.id);
259
+ // #4917: a refused repeat requeue still reopens when an earlier entry was never applied
260
+ const reopen = (requeued.length > 0 || b.unrunRequeues().length > 0) && !hosted.has(req.id) && !starting.has(req.id);
229
261
  if (reopen) await hostBounded({ argv: ["--resume", req.id], cwd: process.cwd(), env: {} }, null, `batch ${req.id}`);
230
262
  return send({ type: "ok", requeued, refused, hosted: hosted.has(req.id), reopened: reopen }, true);
231
263
  }
@@ -306,6 +338,7 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
306
338
  clearInterval(lockTimer);
307
339
  clearInterval(lagTimer);
308
340
  clearInterval(beatTimer);
341
+ for (const t of rehostTimers) clearTimeout(t);
309
342
  if (dueTimer) clearInterval(dueTimer);
310
343
  for (const c of conns) c.destroy();
311
344
  const stopping = [...runs.keys()];
@@ -448,7 +481,7 @@ export function defaultSpawn({ dir = schedDir(), script = SCHEDD_SCRIPT, env = p
448
481
  mkdirSync(dir, { recursive: true });
449
482
  const out = openSync(scheddPaths(dir).log, "a");
450
483
  try {
451
- const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, stdio: ["ignore", out, out] });
484
+ const child = spawn(process.execPath, [script, "schedd"], { env, detached: true, windowsHide: true, stdio: ["ignore", out, out] });
452
485
  child.on?.("error", () => { /* the connect retry reports it */ });
453
486
  child.unref?.();
454
487
  } finally { closeSync(out); }
@@ -249,6 +249,55 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
249
249
  } finally { await h.cleanup(); }
250
250
  });
251
251
 
252
+ it("#5000: add to a finished unhosted batch reopens it through the daemon and runs the new issue", async () => {
253
+ const h = host();
254
+ try {
255
+ await h.daemon();
256
+ const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
257
+ await withRoot(h, () => capture(async () => {
258
+ writeFileSync(h.release, "");
259
+ await orchestrateFromCli(["1", "--parallel", "2", "--repo", "o/r", "--no-prefilter"], h.client);
260
+ const [id] = listBatches(h.root);
261
+ const b = openBatch(h.root, id);
262
+ const res = await orchestrateFromCli(["--add", id, "6"], client);
263
+ assert.deepEqual(res.added, [6]);
264
+ assert.equal(res.reopened, true, "the finished batch is hosted again");
265
+ await until(() => h.launched().includes(6));
266
+ await until(() => b.store.load()?.results?.[6]?.outcome === "merged");
267
+ assert.deepEqual(listBatches(h.root), [id], "no second batch");
268
+ }));
269
+ } finally { await h.cleanup(); }
270
+ });
271
+
272
+ it("#4917: a held (pending) issue is flagged unhosted by status once its report is gone; --requeue re-hosts it and says so", async () => {
273
+ const h = host();
274
+ try {
275
+ await h.daemon();
276
+ const client = { ...h.client, io: { gh: async (a) => (a[0] === "issue" ? JSON.stringify({ state: "OPEN", labels: [] }) : "o/r\n") } };
277
+ await withRoot(h, () => capture(async () => {
278
+ const run = orchestrateFromCli(["1", "2", "--parallel", "1", "--repo", "o/r", "--no-prefilter"], h.client);
279
+ await until(() => h.launched().length === 1 && listBatches(h.root).length === 1);
280
+ const [id] = listBatches(h.root);
281
+ const b = openBatch(h.root, id);
282
+ b.holdIssues([2]);
283
+ writeFileSync(h.release, "");
284
+ const res = await run;
285
+ assert.equal(res.rows.find((r) => r.issue === 2).outcome, "held");
286
+ assert.deepEqual(b.pendingIssues(), [2]);
287
+ b.clearReport(); // a batch finalised before the fix, or one never resumed
288
+ const st = await request(await connectDaemon({ dir: h.sched, spawnDaemon: null }), { op: "status" });
289
+ assert.deepEqual(st.unhosted, [{ id, pending: [2] }]);
290
+ // finalised by an older version: a report plus unapplied requeue entries still reopens, and the CLI says so
291
+ b.releaseIssues([2]);
292
+ b.store.save({ ...b.store.load(), results: { ...b.store.load().results, 1: { issue: 1, outcome: "needs-human", pr: null, cost_usd: 0, minutes: 0 } } });
293
+ b.writeReport({ rows: [{ issue: 1, outcome: "needs-human" }] });
294
+ const rq = await capture(() => orchestrateFromCli(["--requeue", id, "1"], client));
295
+ assert.deepEqual(rq.res.requeued, [1]);
296
+ assert.ok(rq.lines.some((l) => l.includes("reopened and hosted by the schedd")), rq.lines.join("\n"));
297
+ }));
298
+ } finally { await h.cleanup(); }
299
+ });
300
+
252
301
  it("#4376: --cancel stops one batch (its lane torn down, the rest held, report written); the other batch and the daemon run on", async () => {
253
302
  const h = host();
254
303
  try {
@@ -350,6 +399,66 @@ await orchestrateFromCli(["1", "2", "--parallel", "2", "--repo", "o/r", "--no-pr
350
399
  } finally { await h.cleanup(); }
351
400
  });
352
401
 
402
+ it("#4917: a batch that ends unreported with a deferred issue waiting is re-hosted by the running daemon once its retry-at is near", async () => {
403
+ const dir = tmpDir("schedd-rehost-"), root = join(dir, "batches"), id = "b20261031-000000-4917";
404
+ const calls = [];
405
+ const runBatch = async ({ argv, sink }) => {
406
+ calls.push(argv);
407
+ const b = openBatch(root, id);
408
+ if (argv[0] !== "--resume") {
409
+ b.writePlan({ daemon: true, repo: "o/r" });
410
+ b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 1200 } });
411
+ } else b.store.save({ ...b.store.load(), pending: [], results: { 1: { issue: 1, outcome: "merged" } } });
412
+ sink.batch({ id, dir: b.dir });
413
+ return {};
414
+ };
415
+ const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
416
+ try {
417
+ await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
418
+ await until(() => calls.length === 2, 8000);
419
+ assert.deepEqual(calls[1], ["--resume", id]);
420
+ } finally { await d.close(); }
421
+ });
422
+
423
+ it("#4917: a resume that leaves the waiting issue unadmitted (breaker tripped, retry-at passed) is not re-hosted again", async () => {
424
+ const dir = tmpDir("schedd-rehost-loop-"), root = join(dir, "batches"), id = "b20261031-000000-4918";
425
+ const calls = [];
426
+ const runBatch = async ({ argv, sink }) => {
427
+ calls.push(argv);
428
+ const b = openBatch(root, id);
429
+ // first run: deferred issue 1 waits 300 ms; every resume ends at once (a tripped breaker), leaving it pending with a past retry-at
430
+ if (argv[0] !== "--resume") { b.writePlan({ daemon: true, repo: "o/r" }); b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 } }); }
431
+ sink.batch({ id, dir: b.dir });
432
+ return {};
433
+ };
434
+ const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
435
+ try {
436
+ await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
437
+ await until(() => calls.length >= 2, 8000);
438
+ await new Promise((r) => setTimeout(r, 3500));
439
+ assert.equal(calls.length, 2, `re-hosted ${calls.length - 1} times; a past retry-at the resume did not act on must not loop`);
440
+ } finally { await d.close(); }
441
+ });
442
+
443
+ it("#4917: a batch whose breaker tripped (not transport) is not re-hosted by the timer", async () => {
444
+ const dir = tmpDir("schedd-rehost-brk-"), root = join(dir, "batches"), id = "b20261031-000000-4919";
445
+ const calls = [];
446
+ const runBatch = async ({ argv, sink }) => {
447
+ calls.push(argv);
448
+ const b = openBatch(root, id);
449
+ b.writePlan({ daemon: true, repo: "o/r" });
450
+ b.store.save({ v: 1, order: [1], results: {}, running: {}, pending: [1], notBefore: { 1: Date.now() + 300 }, breaker: { at: Date.now(), errors: 3 } });
451
+ sink.batch({ id, dir: b.dir });
452
+ return {};
453
+ };
454
+ const d = await startDaemon({ dir, root, runBatch, log: () => {}, resumeOnStart: false });
455
+ try {
456
+ await request(await connectDaemon({ dir, spawnDaemon: null }), { op: "submit", argv: ["1"], cwd: dir, env: {} });
457
+ await new Promise((r) => setTimeout(r, 2000));
458
+ assert.equal(calls.length, 1, "a tripped breaker holds the batch for an operator --resume");
459
+ } finally { await d.close(); }
460
+ });
461
+
353
462
  it("#4454: close() stops every hosted batch loop before releasing the lock; no batch admits a lane after close()", async () => {
354
463
  const dir = tmpDir("schedd-stop-"), root = join(dir, "batches"), lock = scheddPaths(dir).lock;
355
464
  const started = { a: [], b: [] }, stoppedWithLock = {};
@@ -158,7 +158,7 @@ const CLI = join(dirname(fileURLToPath(import.meta.url)), "..", "forgedock.mjs")
158
158
  export function realIo(env = process.env, { pool = false } = {}) {
159
159
  let self;
160
160
  const root = batchRoot(env);
161
- const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
161
+ const detached = (args) => { const p = spawn(process.execPath, [CLI, ...args], { detached: true, windowsHide: true, stdio: "ignore", env }); p.unref(); return { kill: async () => { try { process.kill(-p.pid, "SIGTERM"); } catch { /* gone */ } } }; };
162
162
  const waitBatch = async (before) => { for (let i = 0; i < 60; i++) { const id = listBatches(root).find((b) => !before.has(b)); if (id) return id; await new Promise((r) => setTimeout(r, 1000)); } throw new Error("orchestrate did not create a batch within 60s"); };
163
163
  return (self = {
164
164
  now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
@@ -37,7 +37,7 @@ function writeLast(dir, repo, ts) {
37
37
  */
38
38
  export function launchNightly({ cli, repo, cwd = process.cwd(), spawnFn = spawn }) {
39
39
  try {
40
- const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, stdio: "ignore" });
40
+ const c = spawnFn(process.execPath, [cli, "selftest", "--live", "--nightly-report", repo], { cwd, detached: true, windowsHide: true, stdio: "ignore" });
41
41
  c.on("error", () => {});
42
42
  c.unref();
43
43
  return c;
@@ -8,6 +8,7 @@
8
8
  * never in the range, so they are never rewritten. Deterministic, 0 LLM calls.
9
9
  */
10
10
  const SEP = "\x1f", END = "\x1e";
11
+ const STASH_REF = "refs/forgedock/dco-stash";
11
12
  const PLACEHOLDER_RE = /@localhost$/i;
12
13
 
13
14
  /** True when the commit must be rewritten: placeholder author or committer, or no sign-off naming its author. */
@@ -59,8 +60,17 @@ export async function ensureSignedOff({ git, lane, branch, env = process.env })
59
60
  throw Object.assign(new Error(`${bad.length} unpushed commit(s) fail DCO (placeholder author or no sign-off: ${bad.slice(0, 5).map((c) => c.sha.slice(0, 8)).join(", ")}) and the worktree has no usable git identity to rewrite them`), { noIdentity: true });
60
61
  }
61
62
  const fenv = { FD_BAD: bad.map((c) => c.sha).join(" "), FD_NAME: name, FD_EMAIL: email, FILTER_BRANCH_SQUELCH_WARNING: "1" };
63
+ // filter-branch refuses a dirty tree (lockfile churn, build output): park tracked changes on a private ref, never the shared stash stack
64
+ const saved = await ok(["status", "--porcelain", "--untracked-files=no"]) ? await ok(["stash", "create"]) : "";
65
+ if (saved) { await git(["update-ref", STASH_REF, saved]); await git(["reset", "-q", "--hard", "HEAD"]); }
66
+ let failure = null;
62
67
  try { await git(["filter-branch", "-f", "--env-filter", ENV_FILTER, "--msg-filter", MSG_FILTER, "--", "HEAD", ...exclude], { env: fenv }); }
63
- catch (e) { throw Object.assign(new Error(`could not rewrite ${bad.length} unpushed commit(s) for DCO: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`), { noIdentity: false }); }
68
+ catch (e) { failure = `could not rewrite ${bad.length} unpushed commit(s) for DCO: ${String(e?.stderr ?? e?.message ?? e).slice(0, 300)}`; }
69
+ if (saved) {
70
+ try { await git(["stash", "apply", saved]); await ok(["update-ref", "-d", STASH_REF]); }
71
+ catch { failure = `${failure ? `${failure}; ` : ""}could not restore the lane's uncommitted changes after the DCO rewrite; they are kept as ${saved} (${STASH_REF})`; }
72
+ }
73
+ if (failure) throw Object.assign(new Error(failure), { noIdentity: false, engineCode: "engine.dco_rewrite" });
64
74
  await ok(["for-each-ref", "--format=delete %(refname)", "refs/original/"]).then(async (l) => { for (const r of l.split("\n").map((x) => x.replace(/^delete /, "")).filter(Boolean)) await ok(["update-ref", "-d", r]); });
65
75
  return { rewritten: bad.length };
66
76
  }
@@ -4,7 +4,7 @@
4
4
  import { describe, it } from "node:test";
5
5
  import assert from "node:assert/strict";
6
6
  import { execFileSync } from "node:child_process";
7
- import { writeFileSync } from "node:fs";
7
+ import { readFileSync, writeFileSync } from "node:fs";
8
8
  import { join } from "node:path";
9
9
  import { tmpDir } from "../tests/_tmp.mjs";
10
10
  import { ensureSignedOff, needsSignoff, offendersOf } from "./signoff.mjs";
@@ -99,4 +99,27 @@ describe("ensureSignedOff", () => {
99
99
  await assert.rejects(ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} }), (e) => e.noIdentity === true);
100
100
  assert.equal(f.sh("rev-parse", "HEAD"), bad);
101
101
  });
102
+
103
+ it("rewrites on a dirty tree (#4997): unstaged and staged tracked changes survive, no stash entry is left", async () => {
104
+ const f = fixture();
105
+ f.commit("a.txt", "unpushed placeholder", "placeholder");
106
+ writeFileSync(join(f.wt, "base.txt"), "churn\n");
107
+ writeFileSync(join(f.wt, "a.txt"), "staged\n"); f.sh("add", "a.txt");
108
+ const r = await ensureSignedOff({ git: f.git, lane: "staging", branch: "feat/x", env: {} });
109
+ assert.equal(r.rewritten, 1);
110
+ assert.equal(readFileSync(join(f.wt, "base.txt"), "utf8"), "churn\n");
111
+ assert.equal(readFileSync(join(f.wt, "a.txt"), "utf8"), "staged\n");
112
+ assert.equal(f.sh("stash", "list"), "");
113
+ assert.equal(f.sh("for-each-ref", "refs/forgedock/"), "");
114
+ assert.match(f.sh("log", "-1", "--format=%B"), /Signed-off-by: Lane Bot/);
115
+ });
116
+
117
+ it("a failed rewrite carries engineCode engine.dco_rewrite, restores the tree, and is not noIdentity (#4997)", async () => {
118
+ const f = fixture();
119
+ f.commit("a.txt", "unpushed placeholder", "placeholder");
120
+ writeFileSync(join(f.wt, "base.txt"), "churn\n");
121
+ const git = async (a, o) => { if (a[0] === "filter-branch") throw new Error("boom"); return f.git(a, o); };
122
+ await assert.rejects(ensureSignedOff({ git, lane: "staging", branch: "feat/x", env: {} }), (e) => e.engineCode === "engine.dco_rewrite" && e.noIdentity === false && /boom/.test(e.message));
123
+ assert.equal(readFileSync(join(f.wt, "base.txt"), "utf8"), "churn\n");
124
+ });
102
125
  });
@@ -364,7 +364,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
364
364
  // systemd services lack WSL_INTEROP; point it at a live interop socket so wsl.exe can run.
365
365
  const sock = (() => { try { return readdirSync("/run/WSL").filter((f) => f.endsWith("_interop")).map((f) => join("/run/WSL", f)).sort().pop(); } catch { return null; } })();
366
366
  setTimeout(() => {
367
- const child = spawn("/mnt/c/Windows/System32/wsl.exe", ["--shutdown"], { detached: true, stdio: "ignore", env: { ...process.env, ...(sock ? { WSL_INTEROP: sock } : {}) } });
367
+ const child = spawn("/mnt/c/Windows/System32/wsl.exe", ["--shutdown"], { detached: true, windowsHide: true, stdio: "ignore", env: { ...process.env, ...(sock ? { WSL_INTEROP: sock } : {}) } });
368
368
  child.unref();
369
369
  }, 1500);
370
370
  }
package/bin/runner.mjs CHANGED
@@ -2092,7 +2092,7 @@ export async function runCliBackendAsync(opts) {
2092
2092
  const finish = (r) => { if (settled) return; settled = true; clearInterval(timer); resolveP(r); };
2093
2093
  let child;
2094
2094
  const input = typeof o.input === "string" ? o.input : null; // forge#4129: large prompt over stdin
2095
- try { child = spawn(bin, args, { cwd: o.cwd, env: o.env, stdio: [input === null ? "ignore" : "pipe", "pipe", "pipe"], detached: true }); }
2095
+ try { child = spawn(bin, args, { cwd: o.cwd, env: o.env, stdio: [input === null ? "ignore" : "pipe", "pipe", "pipe"], detached: true, windowsHide: true }); }
2096
2096
  catch (error) { return finish({ status: null, signal: null, pid: 0, error, stdout: "", stderr: "" }); }
2097
2097
  if (input !== null) { child.stdin.on("error", () => { /* child exited before reading: surfaced by close */ }); child.stdin.end(input); }
2098
2098
  const signalGroup = (sig) => { try { process.kill(-child.pid, sig); } catch { try { child.kill(sig); } catch { /* gone */ } } };
@@ -45,6 +45,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
45
45
  | `gh.auth` | A `gh` call in a core run answered HTTP 401 (expired or revoked bot token) and one run of `scripts/refresh-bot-token.sh` plus a retry did not fix it; the run is `deferred` (retry in 120s), never `engine-error`. See `docs/CONFIG.md` "Bot token refresh" | lean | `bin/lean/retry.mjs` |
46
46
  | `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
47
47
  | `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
48
+ | `engine.dco_rewrite` | Core work-on (forge#4997): the DCO sign-off rewrite (`git filter-branch`) of the lane's unpushed commits failed, or the lane's uncommitted changes could not be restored after it (kept on `refs/forgedock/dco-stash`). A mechanical engine step, not a human block: the lane ends `engine-error` (never `needs-human`), which `forgedock orchestrate` relaunches like any engine-error | lean | `bin/lean/signoff.mjs` `ensureSignedOff` |
48
49
  | `engine.validate_result_lost` | Core work-on (forge#4708): a validate result recorded before a merge-time CI fix round is missing from the fix session's result file and could not be carried forward; an engine bookkeeping error, not an unimplemented requirement | lean | `bin/lean/lean.mjs` `mergeFixRound` |
49
50
  | `infra.disk_quota` | Disk full or quota exhausted (EDQUOT/ENOSPC, errno -28/-122); retryable infra failure, never `needs-human` | engine | Runner error text |
50
51
  | `infra.cli_spawn_transient` | CLI spawn failed transiently before the child ran (ETIMEDOUT, EAGAIN, ENOMEM, EMFILE); retried with backoff on a separate bounded counter that does not consume a phase attempt | engine | `err.code` `CLI_SPAWN_TRANSIENT` |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "forgedock",
3
- "version": "1.43.0",
3
+ "version": "1.43.1",
4
4
  "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
5
5
  "bin": {
6
6
  "forgedock": "bin/forgedock.mjs"
@@ -86,7 +86,7 @@ export async function acquireSlot(owner, deadline) {
86
86
  function exec(argv, { cwd, env, timeoutMs, onData }) {
87
87
  return new Promise((resolve) => {
88
88
  let child; let killed = false; let timer;
89
- try { child = spawn(argv[0], argv.slice(1), { cwd, env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); } catch (e) { resolve({ code: 127, signal: null, error: e.message, timedOut: false }); return; }
89
+ try { child = spawn(argv[0], argv.slice(1), { cwd, env, detached: true, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] }); } catch (e) { resolve({ code: 127, signal: null, error: e.message, timedOut: false }); return; }
90
90
  const kill = () => { try { process.kill(-child.pid, 'SIGTERM'); } catch { /* gone */ } setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } }, 3000).unref(); };
91
91
  const unreg = () => { cleanups.delete(hard); };
92
92
  const hard = () => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* gone */ } };