forgedock 1.35.0 → 1.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +78 -324
  2. package/bin/bot-token-status.mjs +26 -0
  3. package/bin/engine/events.mjs +161 -0
  4. package/bin/engine/events.test.mjs +142 -0
  5. package/bin/engine/outcome-codes.mjs +39 -1
  6. package/bin/engine/phases.mjs +65 -5
  7. package/bin/engine/telemetry-seam.mjs +5 -4
  8. package/bin/engine/telemetry-seam.test.mjs +4 -0
  9. package/bin/engine-cli.mjs +372 -46
  10. package/bin/engine.mjs +3 -1
  11. package/bin/experiment.mjs +55 -0
  12. package/bin/forgedock.mjs +108 -5
  13. package/bin/hooks/reread-nudge.mjs +15 -4
  14. package/bin/labels.json +50 -0
  15. package/bin/lean/batch.mjs +52 -6
  16. package/bin/lean/batch.test.mjs +176 -2
  17. package/bin/lean/conformance.e2e.test.mjs +1 -1
  18. package/bin/lean/conformance.mjs +136 -11
  19. package/bin/lean/conformance.test.mjs +148 -1
  20. package/bin/lean/decisions.mjs +181 -0
  21. package/bin/lean/decisions.test.mjs +71 -0
  22. package/bin/lean/effects.mjs +32 -14
  23. package/bin/lean/execution.mjs +62 -16
  24. package/bin/lean/execution.test.mjs +59 -2
  25. package/bin/lean/gatewatch.mjs +31 -0
  26. package/bin/lean/gatewatch.test.mjs +59 -0
  27. package/bin/lean/guardgate.mjs +170 -0
  28. package/bin/lean/guardgate.test.mjs +153 -0
  29. package/bin/lean/guards.mjs +243 -0
  30. package/bin/lean/guards.test.mjs +177 -0
  31. package/bin/lean/heavy.mjs +54 -8
  32. package/bin/lean/heavy.test.mjs +74 -1
  33. package/bin/lean/hostkill.mjs +1 -1
  34. package/bin/lean/i18n.mjs +65 -0
  35. package/bin/lean/i18n.test.mjs +47 -0
  36. package/bin/lean/integration.mjs +276 -0
  37. package/bin/lean/integration.test.mjs +189 -0
  38. package/bin/lean/lane.mjs +5 -2
  39. package/bin/lean/lean.mjs +169 -30
  40. package/bin/lean/lean.test.mjs +360 -24
  41. package/bin/lean/lifecycle.mjs +129 -0
  42. package/bin/lean/lifecycle.test.mjs +118 -0
  43. package/bin/lean/light.mjs +1 -1
  44. package/bin/lean/milestonesync.mjs +130 -0
  45. package/bin/lean/milestonesync.test.mjs +133 -0
  46. package/bin/lean/orchestrate.mjs +132 -20
  47. package/bin/lean/parked-sweep.mjs +45 -0
  48. package/bin/lean/parked-sweep.test.mjs +71 -0
  49. package/bin/lean/postdeploy-check.mjs +181 -0
  50. package/bin/lean/postdeploy-check.test.mjs +206 -0
  51. package/bin/lean/postdeploy-monitor.mjs +97 -0
  52. package/bin/lean/postdeploy-monitor.test.mjs +147 -0
  53. package/bin/lean/precheck.mjs +6 -4
  54. package/bin/lean/precheck.test.mjs +17 -2
  55. package/bin/lean/prefilter.mjs +106 -21
  56. package/bin/lean/prefilter.test.mjs +86 -6
  57. package/bin/lean/probe.mjs +4 -3
  58. package/bin/lean/probe.test.mjs +1 -1
  59. package/bin/lean/prompts.mjs +23 -9
  60. package/bin/lean/resume.test.mjs +1 -1
  61. package/bin/lean/retry.mjs +28 -0
  62. package/bin/lean/sandbox-provider.contract.test.mjs +36 -3
  63. package/bin/lean/sandbox-remote.mjs +11 -4
  64. package/bin/lean/schedd.mjs +77 -13
  65. package/bin/lean/schedd.test.mjs +131 -5
  66. package/bin/lean/sequence.mjs +24 -0
  67. package/bin/lean/sequence.test.mjs +31 -1
  68. package/bin/lean/signoff.mjs +66 -0
  69. package/bin/lean/signoff.test.mjs +102 -0
  70. package/bin/lean/train.mjs +177 -0
  71. package/bin/lean/train.test.mjs +113 -0
  72. package/bin/pool/cli.mjs +2 -0
  73. package/bin/pool/health.mjs +30 -0
  74. package/bin/review-context.mjs +298 -0
  75. package/bin/runner.mjs +26 -3
  76. package/bin/stats/aggregate.mjs +1 -1
  77. package/bin/stats/compounding.mjs +157 -0
  78. package/bin/stats/compounding.test.mjs +74 -0
  79. package/bin/stats/experiment.mjs +44 -0
  80. package/bin/stats/github-source.mjs +2 -1
  81. package/bin/stats/index.mjs +62 -3
  82. package/bin/stats/release.mjs +75 -0
  83. package/bin/stats/release.test.mjs +68 -0
  84. package/commands/bootstrap.md +10 -8
  85. package/commands/deploy-info.md +3 -0
  86. package/commands/issue.md +2 -0
  87. package/commands/milestone.md +43 -54
  88. package/commands/orchestrate/phase-4-execution.md +2 -0
  89. package/commands/orchestrate.md +15 -3
  90. package/commands/pipeline-status.md +20 -0
  91. package/commands/review-pr.md +1 -0
  92. package/commands/work-on/build/implement.md +1 -1
  93. package/commands/work-on/build.md +4 -2
  94. package/commands/work-on/investigate.md +3 -1
  95. package/commands/work-on/remediate.md +1 -1
  96. package/docs/CONFIG.md +120 -2
  97. package/docs/EVENTS.md +65 -0
  98. package/docs/OUTCOME-CODES.md +9 -1
  99. package/docs/STATS.md +51 -0
  100. package/docs/TELEMETRY.md +3 -1
  101. package/docs/site/install.md +89 -0
  102. package/docs/site/start-a-new-project.md +1 -1
  103. package/docs/spec/label-state-machine.md +6 -0
  104. package/package.json +1 -1
  105. package/pool/install/install-agent.sh +4 -2
  106. package/pool/install/install-router.sh +4 -2
  107. package/pool/install/install-schedd.sh +28 -0
  108. package/pool/install/unit-hardening.sh +34 -0
  109. package/scripts/acceptance-red-green.sh +19 -2
  110. package/scripts/acceptance-red-green.test.sh +2 -0
  111. package/scripts/bench-lightlane.mjs +2 -1
  112. package/scripts/bootstrap-create.mjs +3 -2
  113. package/scripts/bootstrap-validate.mjs +31 -4
  114. package/scripts/eval-gate-scorecard.mjs +7 -0
  115. package/scripts/file-deferred-followups.sh +286 -77
  116. package/scripts/file-deferred-followups.test.sh +117 -2
  117. package/scripts/hotspots.sh +23 -2
  118. package/scripts/hotspots.test.sh +9 -0
  119. package/scripts/plan-validate-live.mjs +111 -0
  120. package/scripts/readiness.mjs +141 -0
  121. package/scripts/review-prepare.sh +17 -1
  122. package/scripts/review-prepare.test.sh +17 -0
  123. package/scripts/settle-scope.mjs +74 -0
  124. package/scripts/systemd/forgedock-bot-token-refresh.service +7 -0
  125. package/scripts/systemd/forgedock-bot-token-refresh.timer +11 -0
  126. package/scripts/wait-ci-green.sh +14 -3
  127. package/scripts/wait-ci-green.test.sh +3 -1
  128. package/templates/bootstrap/needs-human-issue.md +4 -1
  129. package/templates/github-actions/migration-journal-guard.test.mjs +26 -0
@@ -62,6 +62,16 @@ const safeRegExp = (src) => { try { return new RegExp(src, "i"); } catch { retur
62
62
  const ENV_RE = /^[A-Z_][A-Z0-9_]*$/;
63
63
  const IMAGE_RE = /^[A-Za-z0-9][A-Za-z0-9._\/:@-]*$/;
64
64
  const readJson = (file) => { try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; } };
65
+ const TRANSPORT_CODES = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EAI_AGAIN", "EPIPE", "ENOTFOUND", "EHOSTUNREACH", "ENETUNREACH", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
66
+ /** #4672: a failure of the link to the provider (connection refused/reset, `fetch failed`, timeout, 502/503/504), not of the lane. */
67
+ export function isTransportError(e) {
68
+ if (!e || e.unreachable) return false;
69
+ if (e.transient === true || [502, 503, 504].includes(e.status)) return true;
70
+ const code = e.code ?? e.cause?.code;
71
+ if (TRANSPORT_CODES.has(code)) return true;
72
+ if (e.name === "AbortError" || e.name === "TimeoutError") return true;
73
+ return /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|socket hang up|network error|timed? ?out/i.test(String(e.message ?? ""));
74
+ }
65
75
  const errText = (e) => String(e?.stderr || e?.message || e).split("\n")[0].slice(0, 300);
66
76
 
67
77
  /**
@@ -98,7 +108,9 @@ export function parseExecution(sec = {}) {
98
108
  const patRaw = Array.isArray(s.host_capability_patterns) ? s.host_capability_patterns : str(s.host_capability_patterns) ? str(s.host_capability_patterns).split(",") : [];
99
109
  const hostCapabilityPatterns = patRaw.map((x) => { const t = str(x), i = t.indexOf(":"); return { name: t.slice(0, i).trim().toLowerCase(), pattern: t.slice(i + 1).trim() }; })
100
110
  .filter((x) => CAPABILITY_RE.test(x.name) && x.pattern && safeRegExp(x.pattern));
101
- return { mode, maxRemote, provider, image, secrets, budget, ...(hostCapabilities.length ? { hostCapabilities } : {}), ...(hostCapabilityPatterns.length ? { hostCapabilityPatterns } : {}), ...(endpoint ? { endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(cacheStore ? { cacheStore } : {}), ...(allowActions ? { allowActions } : {}), ...(fallbackLocal ? { fallbackLocal } : {}), ...(heavy !== null ? { heavy } : {}) };
111
+ const uw = Number(str(s.unreachable_window_min));
112
+ const unreachableWindowMin = str(s.unreachable_window_min) !== "" && Number.isFinite(uw) && uw >= 0 ? uw : null; // unset: providerLanes defaults to 5
113
+ return { mode, maxRemote, provider, image, secrets, budget, ...(hostCapabilities.length ? { hostCapabilities } : {}), ...(hostCapabilityPatterns.length ? { hostCapabilityPatterns } : {}), ...(endpoint ? { endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(cacheStore ? { cacheStore } : {}), ...(allowActions ? { allowActions } : {}), ...(fallbackLocal ? { fallbackLocal } : {}), ...(heavy !== null ? { heavy } : {}), ...(unreachableWindowMin !== null ? { unreachableWindowMin } : {}) };
102
114
  }
103
115
 
104
116
  const round1 = (x) => Math.round(x * 10) / 10;
@@ -429,7 +441,8 @@ export function providerByName(name, opts) {
429
441
  try { forgeYaml = opts?.cwd ? readFileSync(join(opts.cwd, "forge.yaml"), "utf8") : null; } catch { forgeYaml = null; }
430
442
  // The router's token is resolved under a fixed name so remoteProvider redacts it like any `token_env` credential.
431
443
  const tokenEnv = exec.tokenEnv ?? (router?.token ? "FORGEDOCK_POOL_TOKEN" : undefined);
432
- const credentials = (n) => (n === "FORGEDOCK_POOL_TOKEN" && !exec.tokenEnv ? router?.token ?? null : resolveCredential(n, { env }));
444
+ // #4685: an empty env/keychain lookup of a configured token_env falls back to the router's token file.
445
+ const credentials = (n) => (n === "FORGEDOCK_POOL_TOKEN" && !exec.tokenEnv ? router?.token ?? null : resolveCredential(n, { env }) ?? router?.token ?? null);
433
446
  const settings = { ...exec, ...(exec.endpoint ? {} : router ? { endpoint: router.endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(forgeYaml ? { forgeYaml } : {}) };
434
447
  return remoteProvider({ batch: opts.batch, repo: opts.repo, settings, name: "pool", env, credentials, transport: remoteTransportFor(settings, { transport: opts?.transport, env, credentials }), ...(opts?.now ? { now: opts.now } : {}) });
435
448
  }
@@ -443,7 +456,7 @@ export function providerByName(name, opts) {
443
456
  * returns that sandbox's ledger (ledger.mjs `makeLedger`).
444
457
  */
445
458
  export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env, secrets = [], pollMs = 5000, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
446
- costPerMinute = 0, now = () => Date.now() }) {
459
+ costPerMinute = 0, now = () => Date.now(), unreachableWindowMs = 300000 }) {
447
460
  const cpm = nonNeg(Number(costPerMinute)) ?? 0;
448
461
  // #4255: the sandbox's usage: the provider's `usage()`, else the billable minutes it put on the handle (#4254), else wall
449
462
  // clock since the handle's startedAt; cost is the provider's, else minutes x cost_per_minute.
@@ -463,29 +476,53 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
463
476
  if (ok) { try { led?.release("sandbox", h.id); led?.remove(); } catch { /* the reaper retries */ } }
464
477
  return ok;
465
478
  };
466
- const wait = async (h) => {
479
+ // #4672: a transport failure (router restart, refused/reset connection, 502/503/504) is retried with backoff until the lane has
480
+ // waited `unreachableWindowMs` in a row without one successful poll; then it ends sandbox.unreachable with the sandbox kept.
481
+ const guard = (st) => async (fn) => {
482
+ for (;;) {
483
+ try { const v = await fn(); st.waited = 0; st.tries = 0; return v; } catch (e) {
484
+ if (!isTransportError(e)) throw e;
485
+ if (st.waited >= unreachableWindowMs) throw Object.assign(new Error(`unreachable for ${Math.round(st.waited / 1000)}s: ${errText(e)}`), { unreachable: true });
486
+ const d = Math.min(pollMs * 2 ** Math.min(st.tries++, 10), 30000, Math.max(1, unreachableWindowMs - st.waited));
487
+ st.waited += d;
488
+ await sleep(d);
489
+ }
490
+ }
491
+ };
492
+ const unreachable = (h, e) => ({ terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_UNREACHABLE, retryInSeconds: 60,
493
+ detail: `sandbox ${h.id} ${errText(e)}; kept for adopt on resume` });
494
+ const wait = async (h, g = guard({ waited: 0, tries: 0 })) => {
467
495
  for (;;) {
468
- const r = await provider.fetchResult(h);
496
+ const r = await g(() => provider.fetchResult(h));
469
497
  if (r) return r;
470
- const st = await provider.status(h);
498
+ const st = await g(() => provider.status(h));
471
499
  if (st !== "running") {
472
- const last = await provider.fetchResult(h); // it may have written the result just before exiting
500
+ const last = await g(() => provider.fetchResult(h)); // it may have written the result just before exiting
473
501
  return last ?? { terminalReason: "engine-error", detail: `sandbox ${h.id} ${st} without a result` };
474
502
  }
475
503
  await sleep(pollMs);
476
504
  }
477
505
  };
478
506
  // A start failure repeats on a relaunch: typed, never engine-error (#4254). Retryable (capacity) defers with backoff.
479
- const startFailed = (id, e) => (e?.retryable === true
507
+ // #4685: a refused or missing credential defers with its own code (a fixed token is picked up on the retry), never needs-human.
508
+ const startFailed = (id, e) => ([401, 403].includes(e?.status)
509
+ ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_AUTH, retryInSeconds: 300, detail: `sandbox ${id} could not start (auth): ${errText(e)}; check the pool token (COMPUTESHARE_TOKEN or ~/.config/computeshare/forgedock.token)` }
510
+ : e?.retryable === true
480
511
  ? { terminalReason: "deferred", code: OUTCOME_CODES.SANDBOX_CAPACITY, detail: `sandbox ${id} could not start (capacity): ${errText(e)}`, ...(Number(e.retryInSeconds) > 0 ? { retryInSeconds: Number(e.retryInSeconds) } : {}) }
481
512
  : { terminalReason: "needs-human", code: OUTCOME_CODES.SANDBOX_START_FAILED, detail: `sandbox ${id} failed to start: ${errText(e)}` });
482
513
  return {
514
+ // #4612: a graceful stop of a live sandbox (the provider's checkpoint stop); false when it has none.
515
+ stopLane: async (n, rec) => (typeof provider.stop === "function" && rec?.handle ? (await provider.stop(rec.handle)) === true : false),
483
516
  runOne: async (n, lane, ctx = {}) => {
484
517
  const led = ledgerFor(n);
485
518
  const base = { provider: provider.name, id: provider.idFor(n), issue: n, startedAt: now() };
486
519
  // A sandbox an earlier attempt left behind (same id) goes first, or the new one could not start.
487
- if ((led?.entries("sandbox") ?? []).some((e) => e.value === base.id) && !(await tear(led, base))) {
488
- return { terminalReason: "engine-error", detail: `sandbox ${base.id} from an earlier attempt could not be torn down` };
520
+ const left = (led?.entries("sandbox") ?? []).find((e) => e.value === base.id);
521
+ if (left) {
522
+ // #4672: a sandbox kept as unreachable is adopted when the provider answers again; it is torn down only when it is gone.
523
+ const kept = await adoptLeft(n, left, ctx);
524
+ if (kept) return kept;
525
+ if (!(await tear(led, base))) return { terminalReason: "engine-error", detail: `sandbox ${base.id} from an earlier attempt could not be torn down` };
489
526
  }
490
527
  led?.register("sandbox", base.id, { provider: provider.name, handle: base }); // before it exists
491
528
  let h = base, r, started = false;
@@ -496,12 +533,15 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
496
533
  ctx.onStart?.({ ...(Number.isInteger(h.pid) ? { pid: h.pid } : {}), handle: h, remote: true });
497
534
  r = await wait(h);
498
535
  } catch (e) {
536
+ if (started && e?.unreachable) return withUsage(unreachable(h, e), h); // #4672: no tear, the ledger keeps it
499
537
  r = started ? { terminalReason: "engine-error", detail: `sandbox ${base.id}: ${errText(e)}` } : startFailed(base.id, e);
500
538
  }
501
539
  await tear(led, h); // every path, after the result is in (tear never throws)
502
540
  return started ? withUsage(r, h) : r; // a sandbox that never started used nothing
503
541
  },
504
- adopt: async (n, rec, ctx = {}) => {
542
+ adopt: (n, rec, ctx = {}) => adoptSaved(n, rec, ctx),
543
+ };
544
+ async function adoptSaved(n, rec, ctx = {}) {
505
545
  const saved = rec?.handle;
506
546
  if (!saved?.id || saved.provider !== provider.name) return null;
507
547
  // Usage runs from the sandbox's real start (persisted on the handle), not from this adopt.
@@ -509,15 +549,21 @@ export function providerLanes({ provider, laneArgs, ledgerFor, env = process.env
509
549
  const led = ledgerFor(n);
510
550
  led?.register("sandbox", h.id, { provider: provider.name, handle: { provider: h.provider, id: h.id, issue: h.issue, ...(Number.isFinite(h.startedAt) ? { startedAt: h.startedAt } : {}) } }); // now owned by this dispatcher
511
551
  let r;
552
+ const g = guard({ waited: 0, tries: 0 });
512
553
  try {
513
- const ended = await provider.fetchResult(h);
514
- if (!ended && (await provider.status(h)) !== "running") r = null; // relaunched
515
- else { if (!ended) ctx.onLive?.(); r = await wait(h); } // #4504: a live sandbox is awaited in the background
554
+ const ended = await g(() => provider.fetchResult(h));
555
+ if (!ended && (await g(() => provider.status(h))) !== "running") r = null; // relaunched
556
+ else { if (!ended) ctx.onLive?.(); r = await wait(h, g); } // #4504: a live sandbox is awaited in the background
516
557
  } catch (e) {
558
+ if (e?.unreachable) return withUsage(unreachable(h, e), h); // #4672: same tolerance; no tear, left for the next adopt
517
559
  r = { terminalReason: "engine-error", detail: `sandbox ${h.id}: ${errText(e)}` };
518
560
  }
519
561
  await tear(led, h);
520
562
  return withUsage(r, h);
521
- },
522
- };
563
+ }
564
+ // A leftover ledger entry of this lane's sandbox: its result if it can still be had, else null (torn down by the caller).
565
+ async function adoptLeft(n, left, ctx) {
566
+ const r = await adoptSaved(n, { handle: left.handle }, ctx).catch(() => null);
567
+ return r ?? null;
568
+ }
523
569
  }
@@ -6,7 +6,7 @@ import assert from "node:assert/strict";
6
6
  import { mkdirSync, writeFileSync, existsSync, readFileSync } from "node:fs";
7
7
  import { join } from "node:path";
8
8
  import { tmpDir } from "../tests/_tmp.mjs";
9
- import { parseExecution, SANDBOX_PROVIDERS, detectNeeds, hostCapabilityBlock, placement, placeLane, validationWeight, hostHeadroom, scopedEnv, providerLanes, containerProvider, providerByName, openExecutionBudget } from "./execution.mjs";
9
+ import { parseExecution, SANDBOX_PROVIDERS, detectNeeds, hostCapabilityBlock, placement, placeLane, validationWeight, hostHeadroom, scopedEnv, providerLanes, isTransportError, containerProvider, providerByName, openExecutionBudget } from "./execution.mjs";
10
10
  import { makeLedger, countLiveLanes } from "./ledger.mjs";
11
11
  import { openBatch } from "./batch.mjs";
12
12
  import { orchestrate, formatTable, formatSandboxSummary } from "./orchestrate.mjs";
@@ -24,7 +24,7 @@ function fakeProvider(opts = {}) {
24
24
  async start({ issue, env, secrets }) {
25
25
  p.calls.push(["start", issue]);
26
26
  p.ledgerAtStart = opts.ledger?.entries("sandbox").map((e) => e.value) ?? null;
27
- if (opts.startThrows) throw new Error("boom: no capacity");
27
+ if (opts.startThrows) throw Object.assign(new Error("boom: no capacity"), opts.startThrows === 401 ? { status: 401 } : {});
28
28
  p.live.add(issue);
29
29
  p.env = env; p.secrets = secrets;
30
30
  if (opts.finish) { const r = opts.finish(issue); if (r !== undefined) { p.results[issue] = r; p.live.delete(issue); } }
@@ -81,6 +81,56 @@ describe("execution config (#4253)", () => {
81
81
  });
82
82
  });
83
83
 
84
+ describe("providerLanes: an unreachable provider is retried, then deferred without teardown (#4672)", () => {
85
+ const flaky = (failPolls) => {
86
+ const prov = fakeProvider(); let polls = 0;
87
+ prov.live.add(7);
88
+ const gate = () => { if (polls++ < failPolls) throw Object.assign(new TypeError("fetch failed"), { cause: { code: "ECONNREFUSED" } }); };
89
+ prov.fetchResult = async (h) => { gate(); return prov.results[h.issue] ?? null; };
90
+ prov.status = async (h) => { gate(); prov.calls.push(["status", h.issue]); return prov.live.has(h.issue) ? "running" : "exited"; };
91
+ return prov;
92
+ };
93
+ const lanes = (prov, extra = {}) => { const root = tmpDir("exec-unreach-"); const ledgerFor = ledgerIn(root); const slept = [];
94
+ return { slept, ledgerFor, l: providerLanes({ provider: prov, laneArgs: () => [], ledgerFor, pollMs: 1000, now: () => 1000, sleep: async (ms) => { slept.push(ms); if (slept.length === 6) { prov.results[7] = { terminalReason: "merged", pr: 9 }; prov.live.delete(7); } }, ...extra }) }; };
95
+
96
+ it("unreachable: N failed polls then recovery yields the real result", async () => {
97
+ const prov = flaky(3); prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
98
+ const { l, slept } = lanes(prov);
99
+ const r = await l.runOne(7, "staging");
100
+ assert.equal(r.terminalReason, "merged");
101
+ assert.ok(slept.length >= 3 && slept[1] > slept[0]); // backoff
102
+ assert.equal(prov.calls.filter((c) => c[0] === "teardown").length, 1); // only after the result
103
+ });
104
+
105
+ it("unreachable: down past the window yields sandbox.unreachable, no teardown, ledger kept", async () => {
106
+ const prov = flaky(1e9); prov.start = async () => ({ provider: "fake", id: "fake-7", issue: 7 });
107
+ const { l, ledgerFor } = lanes(prov, { unreachableWindowMs: 5000 });
108
+ const r = await l.runOne(7, "staging");
109
+ assert.deepEqual([r.terminalReason, r.code], ["deferred", "sandbox.unreachable"]);
110
+ assert.equal(prov.calls.some((c) => c[0] === "teardown"), false);
111
+ assert.deepEqual(ledgerFor(7).entries("sandbox").map((e) => e.value), ["fake-7"]);
112
+ });
113
+
114
+ it("unreachable: adopt applies the same tolerance, and a non-transport error is still engine-error", async () => {
115
+ const down = flaky(1e9);
116
+ const { l } = lanes(down, { unreachableWindowMs: 3000 });
117
+ const r = await l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } });
118
+ assert.equal(r.code, "sandbox.unreachable");
119
+ assert.equal(down.calls.some((c) => c[0] === "teardown"), false);
120
+ const ok = flaky(2); ok.results[7] = { terminalReason: "merged", pr: 1 };
121
+ assert.equal((await lanes(ok).l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } })).terminalReason, "merged");
122
+ const bad = fakeProvider(); bad.fetchResult = async () => { throw new Error("boom"); };
123
+ assert.equal((await lanes(bad).l.adopt(7, { handle: { provider: "fake", id: "fake-7", issue: 7 } })).terminalReason, "engine-error");
124
+ });
125
+
126
+ it("unreachable: classifies transport errors and parses the window key", () => {
127
+ for (const e of [new TypeError("fetch failed"), Object.assign(new Error("x"), { code: "ECONNRESET" }), Object.assign(new Error("x"), { status: 502 }), Object.assign(new Error("x"), { name: "AbortError" })]) assert.equal(isTransportError(e), true);
128
+ for (const e of [new Error("boom"), Object.assign(new Error("HTTP 404"), { status: 404 })]) assert.equal(isTransportError(e), false);
129
+ assert.equal(parseExecution({}).unreachableWindowMin, undefined); // providerLanes defaults it to 5 min
130
+ assert.equal(parseExecution({ unreachable_window_min: "2" }).unreachableWindowMin, 2);
131
+ });
132
+ });
133
+
84
134
  describe("providerLanes: sandbox ledger and teardown on every exit path (#4253)", () => {
85
135
  const run = async ({ finish, startThrows, teardownOk, pollMs = 1 } = {}) => {
86
136
  const root = tmpDir("exec-ledger-");
@@ -118,6 +168,13 @@ describe("providerLanes: sandbox ledger and teardown on every exit path (#4253)"
118
168
  assert.equal(led.read(), null);
119
169
  });
120
170
 
171
+ it("#4685 start refused with HTTP 401: deferred sandbox.auth with a long retry, never needs-human", async () => {
172
+ const { r } = await run({ startThrows: 401 });
173
+ assert.equal(r.terminalReason, "deferred");
174
+ assert.equal(r.code, "sandbox.auth");
175
+ assert.equal(r.retryInSeconds, 300);
176
+ });
177
+
121
178
  it("start throws: typed needs-human sandbox.start_failed row (#4254), teardown still runs", async () => {
122
179
  const { r, prov, led } = await run({ startThrows: true });
123
180
  assert.equal(r.terminalReason, "needs-human");
@@ -0,0 +1,31 @@
1
+ /**
2
+ * forge#4656: ship-gate watchdog. A gate (a settle pass, a hold list, `/milestone ship` pre-flight) that waits on an issue the
3
+ * scheduler will not admit never resolves; this pure module names those waits so the batch reports `GATE_DEADLOCK` within one pass
4
+ * instead of polling silently. No I/O.
5
+ */
6
+
7
+ /**
8
+ * `gates`: [{ name, waitsOn: [issue...], since?: ms }]; `admissible(n)`: true when the scheduler could still run or finish `n`;
9
+ * `now`: ms. Returns one { issue, gate, ageMs } per (gate, issue) wait that cannot resolve, in gate order.
10
+ */
11
+ export function detectGateDeadlock({ gates, admissible, now = Date.now(), firstSeen = null }) {
12
+ const out = [];
13
+ for (const g of Array.isArray(gates) ? gates : []) {
14
+ for (const n of Array.isArray(g?.waitsOn) ? g.waitsOn : []) {
15
+ let ok = true;
16
+ try { ok = admissible(n) !== false; } catch { ok = true; }
17
+ if (ok) continue;
18
+ const key = `${g.name}#${n}`;
19
+ if (firstSeen && !firstSeen.has(key)) firstSeen.set(key, Number.isFinite(g.since) ? g.since : now);
20
+ const since = firstSeen?.get(key) ?? (Number.isFinite(g.since) ? g.since : now);
21
+ out.push({ issue: n, gate: String(g.name), ageMs: Math.max(0, now - since) });
22
+ }
23
+ }
24
+ return out;
25
+ }
26
+
27
+ /** One `pipeline-status` line: `GATE_DEADLOCK issue=#N gate=G age=12m`. */
28
+ export function formatGateDeadlock(d) {
29
+ const m = Math.round((Number(d.ageMs) || 0) / 60000);
30
+ return `GATE_DEADLOCK issue=#${d.issue} gate=${d.gate} age=${m}m`;
31
+ }
@@ -0,0 +1,59 @@
1
+ import { describe, it } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { detectGateDeadlock, formatGateDeadlock } from "./gatewatch.mjs";
4
+ import { orchestrate } from "./orchestrate.mjs";
5
+ import { classifyNeedsHuman, classifyLegacyNeedsHuman, parkedLabel } from "../engine/outcome-codes.mjs";
6
+
7
+ describe("#4656 gate watchdog", () => {
8
+ it("a milestone ship gate waiting on a parked issue yields GATE_DEADLOCK with issue, gate and age", () => {
9
+ const firstSeen = new Map([["ship#7", 0]]);
10
+ const d = detectGateDeadlock({ gates: [{ name: "ship", waitsOn: [7, 8] }], admissible: (n) => n !== 7, now: 120000, firstSeen });
11
+ assert.deepEqual(d, [{ issue: 7, gate: "ship", ageMs: 120000 }]);
12
+ assert.equal(formatGateDeadlock(d[0]), "GATE_DEADLOCK issue=#7 gate=ship age=2m");
13
+ });
14
+ it("an admissible wait is not a deadlock", () => {
15
+ assert.deepEqual(detectGateDeadlock({ gates: [{ name: "ship", waitsOn: [8] }], admissible: () => true }), []);
16
+ });
17
+ it("the batch runner reports GATE_DEADLOCK once per (gate, issue) for a parked issue on the gate, instead of waiting", async () => {
18
+ const rows = [], decisions = [];
19
+ const runOne = async (n) => n === 1
20
+ ? { terminalReason: "needs-human", code: "conformance.unimplemented", detail: "unknown requirement", pr: 9 }
21
+ : { terminalReason: "merged", pr: 10 };
22
+ const _unused = async (n) => n === 1
23
+ ? { terminalReason: "needs-human", code: "ci.red", detail: "lint failed", pr: 9 }
24
+ : { terminalReason: "merged", pr: 10 };
25
+ const relabels = [];
26
+ const res = await orchestrate({ issues: [1, 2], parallel: 1, runOne, readBody: async () => "", onUpdate: (r) => rows.push(r), decide: (d) => decisions.push(d),
27
+ getGates: () => [{ name: "ship", waitsOn: [1] }], recovery: { maxAttempts: 1 }, relabelNeedsHuman: async (a) => { relabels.push(a); return true; }, pollMs: 1, sleep: async () => {} });
28
+ const dl = rows.filter((r) => r.outcome === "GATE_DEADLOCK");
29
+ assert.equal(dl.length, 1);
30
+ assert.match(dl[0].detail, /GATE_DEADLOCK issue=#1 gate=ship/);
31
+ assert.ok(Array.isArray(res));
32
+ assert.deepEqual(relabels.map((r) => r.parked?.code), ["conformance.unimplemented"]);
33
+ });
34
+ });
35
+
36
+ describe("#4656 parked class and migration", () => {
37
+ it("parked is opt-in and leaves the default classes untouched", () => {
38
+ const row = { outcome: "needs-human", code: "conformance.unimplemented" };
39
+ assert.equal(classifyNeedsHuman(row), "terminal");
40
+ assert.equal(classifyNeedsHuman(row, { parked: true }), "parked");
41
+ assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: "builder.blocked" }, { parked: true }), "human");
42
+ assert.equal(classifyNeedsHuman({ outcome: "needs-human", code: "ci.red" }, { parked: true }), "recoverable");
43
+ assert.equal(classifyNeedsHuman({ outcome: "needs-human" }, { parked: true }), "terminal");
44
+ });
45
+ it("legacy needs-human classifies from the terminal annotation and defaults to external", () => {
46
+ assert.deepEqual(classifyLegacyNeedsHuman("terminal: ci.red after final round"), { kind: "parked", code: "ci.red" });
47
+ assert.deepEqual(classifyLegacyNeedsHuman("waiting for the AWS account"), { kind: "external" });
48
+ assert.deepEqual(classifyLegacyNeedsHuman(null), { kind: "external" });
49
+ assert.equal(parkedLabel("ci.red"), "parked:ci.red");
50
+ });
51
+ });
52
+
53
+ describe("#4656 report routing", () => {
54
+ it("a parked row past recovery is listed on the remediation queue line, not unrecovered or escalations", async () => {
55
+ const { formatOutcomeLists } = await import("./orchestrate.mjs");
56
+ const lines = formatOutcomeLists([{ issue: 1, outcome: "needs-human", code: "conformance.unimplemented", pr: 9, parkedQueued: true }]);
57
+ assert.deepEqual(lines, ["remediation queue (/work-on:remediate): #1 (parked:conformance.unimplemented PR #9)"]);
58
+ });
59
+ });
@@ -0,0 +1,170 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ /**
4
+ * Guard gate (#4693): execution-based, no pattern matching on diff text. A lane whose diff changes lines a merged fix
5
+ * protected (git blame on the lane's merge base names the guard's merge commit, or a later commit that kept the lines), or
6
+ * edits/deletes a guard's test file, reruns that guard's test files *as they exist on base* against the lane's tree. A
7
+ * failure is `conformance.guard_broken` (builder resumed once). The only override is a `supersedes` amendment (conformance.mjs).
8
+ * Deterministic, 0 LLM calls; a lane touching nothing guarded does one local index read and no git work.
9
+ */
10
+ import * as nodeFs from "node:fs";
11
+ import { dirname, extname, join } from "node:path";
12
+ import { OUTCOME_CODES } from "../engine/outcome-codes.mjs";
13
+ import { collectGuards, guardsIndexPath, readIndex, writeIndex } from "./guards.mjs";
14
+ import { isTestPath } from "./enumerating.mjs";
15
+
16
+ export const GUARD_CAP_MS = 120000;
17
+ export const INDEX_MAX_AGE_MS = 6 * 3600000;
18
+ const TIMEOUT_RC = 124;
19
+ const SLACK = 10; // lines a guarded hunk may have drifted since its merge when only a later commit kept it
20
+ const SHA_RE = /^[0-9a-f]{40}$/;
21
+ const ok = async (fn) => { try { await fn(); return true; } catch { return false; } };
22
+ const norm = (p) => String(p ?? "").replace(/^\.\//, "");
23
+
24
+ /** Guards from the local index; rebuilt from the GitHub register when absent or older than INDEX_MAX_AGE_MS (a stale index is still used when GitHub is unreachable). */
25
+ export async function loadGuards({ repo, gh, root, fs = nodeFs, now = Date.now() }) {
26
+ const cached = readIndex(repo, { root, fs });
27
+ let age = Infinity;
28
+ try { age = now - fs.statSync(guardsIndexPath(repo, root)).mtimeMs; } catch { /* no index yet */ }
29
+ if (cached && age < INDEX_MAX_AGE_MS) return cached;
30
+ try { const g = await collectGuards({ gh, repo }); writeIndex(repo, g, { root, fs }); return g; } catch { return cached ?? []; }
31
+ }
32
+
33
+ /** Old-side line ranges of a `git diff -U0` text for non-test files: {file: [[start, end]]}. Pure insertions touch no old line. */
34
+ export function oldRanges(diff) {
35
+ const out = {};
36
+ let file = null;
37
+ for (const line of String(diff ?? "").split("\n")) {
38
+ if (line.startsWith("--- ")) { file = line === "--- /dev/null" ? null : line.replace(/^--- a\//, ""); continue; }
39
+ if (line.startsWith("+++ ")) { if (line === "+++ /dev/null") continue; continue; }
40
+ const m = /^@@ -(\d+)(?:,(\d+))? \+/.exec(line);
41
+ if (!m || !file || isTestPath(file)) continue;
42
+ const start = Number(m[1]), len = m[2] === undefined ? 1 : Number(m[2]);
43
+ if (len > 0) (out[file] ??= []).push([start, start + len - 1]);
44
+ }
45
+ return out;
46
+ }
47
+
48
+ /** Commit shas that last touched lines a..b of `file` at `rev` (git blame --porcelain). */
49
+ export function blameShas(text) {
50
+ const shas = new Set();
51
+ for (const l of String(text ?? "").split("\n")) { const s = l.split(" ")[0]; if (SHA_RE.test(s)) shas.add(s); }
52
+ return shas;
53
+ }
54
+
55
+ /**
56
+ * The guards a lane's diff intersects, each with why it was selected: {guard, reason}. `files` are the lane's changed paths.
57
+ * Guards whose hunk files and test files the lane does not touch are never looked at (no git calls).
58
+ */
59
+ export async function selectGuards({ guards, files, git, base }) {
60
+ const changed = new Set((files ?? []).map(norm));
61
+ const cands = (guards ?? []).filter((g) => (g.hunks ?? []).some((h) => changed.has(norm(h.file))) || (g.tests ?? []).some((t) => changed.has(norm(t.path))));
62
+ if (!cands.length) return { selected: [], mergeBase: null };
63
+ const mergeBase = String(await git(["merge-base", base, "HEAD"])).trim();
64
+ const hunkFiles = [...new Set(cands.flatMap((g) => (g.hunks ?? []).map((h) => norm(h.file))).filter((f) => changed.has(f)))];
65
+ const ranges = hunkFiles.length ? oldRanges(String(await git(["diff", "-U0", "--no-renames", `${base}...HEAD`, "--", ...hunkFiles]))) : {};
66
+ const blamed = new Map(); // `${file}:${a}-${b}` -> Set(sha)
67
+ const shasOf = async (file, [a, b]) => {
68
+ const k = `${file}:${a}-${b}`;
69
+ if (!blamed.has(k)) blamed.set(k, blameShas(await git(["blame", "--porcelain", "-L", `${a},${b}`, mergeBase, "--", file]).catch(() => "")));
70
+ return blamed.get(k);
71
+ };
72
+ const selected = [];
73
+ for (const g of cands) {
74
+ const sha = String(g.merge_sha ?? "");
75
+ const known = SHA_RE.test(sha) || sha.length >= 7 ? await ok(() => git(["cat-file", "-e", `${sha}^{commit}`])) : false;
76
+ let reason = null;
77
+ for (const h of g.hunks ?? []) {
78
+ const f = norm(h.file);
79
+ for (const r of ranges[f] ?? []) {
80
+ const near = r[0] <= h.end + SLACK && r[1] >= h.start - SLACK;
81
+ if (!known) { if (near) reason = `lines ${r[0]}-${r[1]} of ${f} overlap the protected hunk (merge commit not local)`; }
82
+ else if (near) {
83
+ for (const s of await shasOf(f, r)) {
84
+ if (s === sha || (s.startsWith(sha) && sha.length >= 7) || sha.startsWith(s) || await ok(() => git(["merge-base", "--is-ancestor", sha, s]))) { reason = `lines ${r[0]}-${r[1]} of ${f} last changed by ${s.slice(0, 10)}, which carries the fix`; break; }
85
+ }
86
+ }
87
+ if (reason) break;
88
+ }
89
+ if (reason) break;
90
+ }
91
+ if (!reason) { const t = (g.tests ?? []).find((x) => changed.has(norm(x.path))); if (t) reason = `the lane edits or deletes its test ${t.path}`; }
92
+ if (reason) selected.push({ guard: g, reason });
93
+ }
94
+ return { selected, mergeBase };
95
+ }
96
+
97
+ /** The command that runs one test file: the contract's own validate command with that path substituted, else a per-extension default; null when unknown. */
98
+ export function commandFor(path, contract) {
99
+ const ext = extname(path);
100
+ for (const v of contract?.validate ?? []) {
101
+ const cmd = String(v?.command ?? "");
102
+ if (/(?:^|\s)(?:-k|-t|--grep|--test-name-pattern|--testNamePattern)(?:\s|=)|::/.test(cmd)) continue;
103
+ const toks = cmd.match(/[^\s"']+/g) ?? [];
104
+ const tests = toks.filter((t) => extname(t) === ext && isTestPath(t));
105
+ if (tests.length === 1 && /\b(?:node|pytest|python3?|vitest|jest|npx|pnpm|npm|yarn|bun)\b/.test(cmd)) return cmd.replace(tests[0], path);
106
+ }
107
+ if ([".mjs", ".cjs", ".js"].includes(ext)) return `node --test ${path}`;
108
+ if (ext === ".py") return `python -m pytest -q ${path}`;
109
+ return null;
110
+ }
111
+
112
+ /** Sibling path for the base copy of a test file, so its relative imports resolve against the lane's tree. */
113
+ export const basePathOf = (path) => { const e = extname(path); return `${path.slice(0, path.length - e.length)}__guardbase${e}`; };
114
+
115
+ const tailOf = (out) => String(out ?? "").split("\n").filter((l) => l.trim()).slice(-40).join("\n").slice(-1500);
116
+
117
+ /**
118
+ * Rerun the selected guards' base test files against the lane tree, in a small pool, under one wall-time cap.
119
+ * Returns {failures, runs, skipped[], ms}. A test that cannot be run (no base copy, unknown runner, over the cap) is skipped, never a failure.
120
+ */
121
+ export async function runGuards({ selected, mergeBase, contract, git, runCommand, cwd, env, capMs = GUARD_CAP_MS, parallelism = 2, fs = nodeFs, now = Date.now }) {
122
+ const t0 = now(), skipped = [], runs = [], jobs = [];
123
+ for (const { guard } of selected) {
124
+ for (const t of guard.tests ?? []) {
125
+ const path = norm(t.path), copy = basePathOf(path), command = commandFor(copy, contract);
126
+ if (!command) { skipped.push(`#${guard.issue} ${path}: no runner known for ${extname(path) || "this file"}`); continue; }
127
+ jobs.push({ guard, path, copy, command });
128
+ }
129
+ }
130
+ const made = [];
131
+ try {
132
+ const queue = jobs.slice();
133
+ const work = async () => {
134
+ for (let j = queue.shift(); j; j = queue.shift()) {
135
+ const left = capMs - (now() - t0);
136
+ if (left <= 0) { skipped.push(`#${j.guard.issue} ${j.path}: over the ${Math.round(capMs / 1000)}s guard cap`); continue; }
137
+ let body;
138
+ try { body = await git(["show", `${mergeBase}:${j.path}`]); } catch { skipped.push(`#${j.guard.issue} ${j.path}: not on base`); continue; }
139
+ const abs = join(cwd, j.copy);
140
+ try { fs.mkdirSync(dirname(abs), { recursive: true }); fs.writeFileSync(abs, body); made.push(abs); } catch (e) { skipped.push(`#${j.guard.issue} ${j.path}: ${String(e?.message ?? e).slice(0, 80)}`); continue; }
141
+ let r;
142
+ try { r = await runCommand({ command: j.command, cwd, timeoutMs: left, env }); } catch (e) { r = { rc: 127, output: String(e?.message ?? e) }; }
143
+ if (r?.rc === TIMEOUT_RC) { skipped.push(`#${j.guard.issue} ${j.path}: over the ${Math.round(capMs / 1000)}s guard cap`); continue; }
144
+ if (r?.rc === 127) { skipped.push(`#${j.guard.issue} ${j.path}: runner unavailable`); continue; }
145
+ runs.push({ issue: j.guard.issue, why: j.guard.why, test: j.path, command: j.command, rc: Number.isInteger(r?.rc) ? r.rc : 1, output: String(r?.output ?? "") });
146
+ }
147
+ };
148
+ await Promise.all(Array.from({ length: Math.max(1, Math.min(parallelism, jobs.length)) }, work));
149
+ } finally {
150
+ for (const p of made) { try { fs.rmSync(p, { force: true }); } catch { /* best effort */ } }
151
+ }
152
+ return { failures: guardFailures(runs), runs, skipped, ms: now() - t0 };
153
+ }
154
+
155
+ /** conformance.guard_broken rows (one per failing guard test): the guard's issue, its one-line why, the failing test and its output tail. */
156
+ export function guardFailures(runs) {
157
+ return (runs ?? []).filter((r) => r.rc !== 0).map((r) => ({
158
+ code: OUTCOME_CODES.CONFORMANCE_GUARD_BROKEN, kind: "guard_broken", guard: `#${r.issue}`, test: r.test,
159
+ detail: `guard #${r.issue} broken (${String(r.why ?? "").slice(0, 200)}): base test ${r.test} fails against your tree (\`${r.command}\`, exit ${r.rc}):\n${tailOf(r.output)}`,
160
+ }));
161
+ }
162
+
163
+ /** The comment the engine posts on a guarded issue once its guard is superseded. */
164
+ export const supersessionComment = ({ guard, pr, why }) => `The fix guard from ${guard} was superseded by PR #${pr}: ${String(why ?? "").trim()}\n\nThe lane's contract carried a \`supersedes\` amendment the reviewer accepted; this guard's test no longer describes current behaviour.`;
165
+
166
+ /** Guard ids the effective contract's accepted supersedes amendments retire, limited to guards the gate selected. */
167
+ export function supersededIds(contract, selected) {
168
+ const sel = new Set((selected ?? []).map((s) => `#${s.guard.issue}`));
169
+ return (contract?.supersedes ?? []).filter((s) => sel.has(s.guard));
170
+ }