forgedock 1.36.0 → 1.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/bin/bot-token-status.mjs +26 -0
  2. package/bin/engine/events.mjs +10 -1
  3. package/bin/engine/events.test.mjs +15 -1
  4. package/bin/engine/outcome-codes.mjs +39 -1
  5. package/bin/engine/phases.mjs +65 -5
  6. package/bin/engine/telemetry-seam.mjs +1 -1
  7. package/bin/engine/telemetry-seam.test.mjs +4 -0
  8. package/bin/engine-cli.mjs +268 -37
  9. package/bin/engine.mjs +3 -1
  10. package/bin/forgedock.mjs +108 -5
  11. package/bin/labels.json +50 -0
  12. package/bin/lean/batch.mjs +10 -4
  13. package/bin/lean/batch.test.mjs +75 -1
  14. package/bin/lean/conformance.e2e.test.mjs +1 -1
  15. package/bin/lean/conformance.mjs +125 -11
  16. package/bin/lean/conformance.test.mjs +122 -1
  17. package/bin/lean/decisions.mjs +181 -0
  18. package/bin/lean/decisions.test.mjs +71 -0
  19. package/bin/lean/effects.mjs +29 -14
  20. package/bin/lean/execution.mjs +60 -16
  21. package/bin/lean/execution.test.mjs +59 -2
  22. package/bin/lean/gatewatch.mjs +31 -0
  23. package/bin/lean/gatewatch.test.mjs +59 -0
  24. package/bin/lean/guardgate.mjs +170 -0
  25. package/bin/lean/guardgate.test.mjs +153 -0
  26. package/bin/lean/guards.mjs +243 -0
  27. package/bin/lean/guards.test.mjs +177 -0
  28. package/bin/lean/heavy.mjs +54 -8
  29. package/bin/lean/heavy.test.mjs +74 -1
  30. package/bin/lean/hostkill.mjs +1 -1
  31. package/bin/lean/integration.mjs +276 -0
  32. package/bin/lean/integration.test.mjs +189 -0
  33. package/bin/lean/lean.mjs +134 -28
  34. package/bin/lean/lean.test.mjs +239 -22
  35. package/bin/lean/lifecycle.mjs +129 -0
  36. package/bin/lean/lifecycle.test.mjs +118 -0
  37. package/bin/lean/light.mjs +1 -1
  38. package/bin/lean/milestonesync.mjs +130 -0
  39. package/bin/lean/milestonesync.test.mjs +133 -0
  40. package/bin/lean/orchestrate.mjs +83 -15
  41. package/bin/lean/parked-sweep.mjs +45 -0
  42. package/bin/lean/parked-sweep.test.mjs +71 -0
  43. package/bin/lean/postdeploy-check.mjs +38 -6
  44. package/bin/lean/postdeploy-check.test.mjs +62 -2
  45. package/bin/lean/postdeploy-monitor.mjs +12 -2
  46. package/bin/lean/postdeploy-monitor.test.mjs +27 -1
  47. package/bin/lean/prefilter.mjs +83 -23
  48. package/bin/lean/prefilter.test.mjs +73 -1
  49. package/bin/lean/probe.mjs +2 -2
  50. package/bin/lean/probe.test.mjs +1 -1
  51. package/bin/lean/prompts.mjs +15 -9
  52. package/bin/lean/resume.test.mjs +1 -1
  53. package/bin/lean/retry.mjs +28 -0
  54. package/bin/lean/sandbox-provider.contract.test.mjs +8 -3
  55. package/bin/lean/sandbox-remote.mjs +4 -4
  56. package/bin/lean/schedd.mjs +53 -9
  57. package/bin/lean/schedd.test.mjs +77 -5
  58. package/bin/lean/sequence.mjs +24 -0
  59. package/bin/lean/sequence.test.mjs +31 -1
  60. package/bin/lean/signoff.mjs +66 -0
  61. package/bin/lean/signoff.test.mjs +102 -0
  62. package/bin/lean/train.mjs +177 -0
  63. package/bin/lean/train.test.mjs +113 -0
  64. package/bin/pool/cli.mjs +2 -0
  65. package/bin/pool/health.mjs +30 -0
  66. package/bin/review-context.mjs +298 -0
  67. package/bin/runner.mjs +26 -3
  68. package/bin/stats/compounding.mjs +157 -0
  69. package/bin/stats/compounding.test.mjs +74 -0
  70. package/bin/stats/index.mjs +21 -2
  71. package/commands/bootstrap.md +10 -8
  72. package/commands/deploy-info.md +3 -0
  73. package/commands/issue.md +2 -0
  74. package/commands/milestone.md +43 -54
  75. package/commands/orchestrate/phase-4-execution.md +2 -0
  76. package/commands/orchestrate.md +8 -1
  77. package/commands/pipeline-status.md +20 -0
  78. package/commands/review-pr.md +1 -0
  79. package/commands/work-on/build/implement.md +1 -1
  80. package/commands/work-on/build.md +4 -2
  81. package/commands/work-on/investigate.md +3 -1
  82. package/commands/work-on/remediate.md +1 -1
  83. package/docs/CONFIG.md +71 -2
  84. package/docs/EVENTS.md +1 -4
  85. package/docs/OUTCOME-CODES.md +9 -1
  86. package/docs/STATS.md +20 -0
  87. package/docs/site/start-a-new-project.md +1 -1
  88. package/docs/spec/label-state-machine.md +6 -0
  89. package/package.json +1 -1
  90. package/pool/install/install-agent.sh +4 -2
  91. package/pool/install/install-router.sh +4 -2
  92. package/pool/install/install-schedd.sh +28 -0
  93. package/pool/install/unit-hardening.sh +34 -0
  94. package/scripts/acceptance-red-green.sh +19 -2
  95. package/scripts/acceptance-red-green.test.sh +2 -0
  96. package/scripts/bootstrap-create.mjs +3 -2
  97. package/scripts/bootstrap-validate.mjs +31 -4
  98. package/scripts/file-deferred-followups.sh +285 -77
  99. package/scripts/file-deferred-followups.test.sh +111 -4
  100. package/scripts/hotspots.sh +23 -2
  101. package/scripts/hotspots.test.sh +9 -0
  102. package/scripts/plan-validate-live.mjs +111 -0
  103. package/scripts/readiness.mjs +141 -0
  104. package/scripts/review-prepare.sh +17 -1
  105. package/scripts/review-prepare.test.sh +17 -0
  106. package/scripts/settle-scope.mjs +74 -0
  107. package/scripts/systemd/forgedock-bot-token-refresh.service +7 -0
  108. package/scripts/systemd/forgedock-bot-token-refresh.timer +11 -0
  109. package/scripts/wait-ci-green.sh +14 -3
  110. package/scripts/wait-ci-green.test.sh +3 -1
  111. package/templates/bootstrap/needs-human-issue.md +4 -1
  112. package/templates/github-actions/migration-journal-guard.test.mjs +26 -0
@@ -37,6 +37,8 @@
37
37
  * issue while maintenance starts stay within `share` of all lane starts (`dispatch: {started, maint}`, persisted), then
38
38
  * P2+ features, then (`whenIdle`) maintenance once no feature is pending. Maintenance the budget never admits ends `held`.
39
39
  *
40
+ * Milestone lifecycle (#4657), off unless `lifecycle` is passed (`bin/lean/lifecycle.mjs` makeLifecycle): at batch end
41
+ * `lifecycle.onBatchEnd(titles)` advances/closes each milestone a `merged` row's result named. Errors never change an outcome.
40
42
  * Milestone drift (#4161), off unless `drift` is passed (`bin/lean/drift.mjs` makeDrift): a `merged` milestone-lane run
41
43
  * whose result carries `structure` awaits `drift.onMerged(issue, row, result)` before its slot frees. A drift error is
42
44
  * reported as a `drift` row and never changes the outcome. With `drift.holds(issue, {body, lane})` (block_dispatch), a
@@ -100,6 +102,7 @@
100
102
  */
101
103
  import { parseDependsOn } from "../engine/phases.mjs";
102
104
  import { classifyNeedsHuman, recoveryCodeOf, OUTCOME_CODES } from "../engine/outcome-codes.mjs";
105
+ import { detectGateDeadlock, formatGateDeadlock } from "./gatewatch.mjs";
103
106
  import { recoveryEvidence } from "./render.mjs";
104
107
  import { renderAlerts } from "../stats/render.mjs";
105
108
  import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
@@ -236,9 +239,9 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
236
239
  */
237
240
  export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
238
241
  store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
239
- classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
240
- recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, stopLane = null,
241
- estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
242
+ classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
243
+ recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, getGates = null, stopLane = null,
244
+ estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null }) {
242
245
  const saved = store?.load?.() ?? null;
243
246
  const order = saved ? [...saved.order] : [];
244
247
  const set = new Set(order);
@@ -273,7 +276,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
273
276
  const parkedBase = new Map(); // #4525: issue -> base sha its parked PR was last synced (or reported conflicting) against
274
277
  let lastParkedSync = null;
275
278
  const bodies = new Map(); // issue -> body, kept in memory for drift.holds only
279
+ const lifecycleTitles = new Set(); // #4657: milestones this batch merged into, re-evaluated at batch end
276
280
  const driftHeld = new Map(); // issue -> flagged package dir holding it this pass
281
+ const integHeld = new Map(); // issue -> reason (red integration streak on its milestone lane, #4667) holding it this pass
277
282
  const reg = sched?.registry ?? null;
278
283
  const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
279
284
  const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
@@ -304,6 +309,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
304
309
  const recovering = toMap(saved?.recovering); // issue -> {code, attempt, detail, outcome, ...} of the recovery lane queued or running
305
310
  const requeueApplied = new Set(saved?.requeueApplied ?? []); // #4550: `--requeue` entries already applied (batch `requeue` file)
306
311
  const requeuedInfo = toMap(saved?.requeued); // #4550: issue -> {reason, by, at, prior: [earlier rows]}
312
+ const everRelabelled = new Set(saved?.relabelled ?? []); // #4656: issues the runner already relabelled; never parked on top
307
313
  const relabelled = new Set(saved?.relabelled ?? []); // #4315: queued recoveries relabelled needs-human when a batch stopped
308
314
  // #4267: decisions are recorded, never awaited: a failing or slow recorder cannot block the batch.
309
315
  const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
@@ -493,14 +499,23 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
493
499
  return true;
494
500
  };
495
501
 
496
- const skip = (n, detail) => { results.set(n, { issue: n, outcome: "skipped", pr: null, cost_usd: 0, minutes: 0, detail }); update(results.get(n)); };
502
+ // #4687: a dependency skip is a waiting state while a blocker re-check can still re-admit it (`waiting`, cleared when the batch ends);
503
+ // the log row names the triggering issue.
504
+ const skip = (n, detail, trigger = null) => {
505
+ results.set(n, { issue: n, outcome: "skipped", pr: null, cost_usd: 0, minutes: 0, detail, ...(trigger !== null && dispatchCheck ? { waiting: true } : {}) });
506
+ update({ ...results.get(n), ...(trigger !== null ? { trigger } : {}) });
507
+ };
497
508
  const requeue = (n, delayMs) => { pending.push(n); enqueued.set(n, now()); if (delayMs > 0) notBefore.set(n, now() + delayMs); else notBefore.delete(n); };
498
- const noteEngineError = () => {
509
+ const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]/i;
510
+ const TRANSPORT_COOLDOWN_MS = 5 * 60000;
511
+ let transportErrors = 0; // #4685: engine-errors in the current window that were link failures
512
+ const noteEngineError = (detail = "") => {
499
513
  if (!breaker) return;
500
514
  const t = now(), windowMs = breaker.window_min * 60000;
501
515
  errorTimes = [...errorTimes.filter((x) => t - x < windowMs), t];
516
+ transportErrors = TRANSPORT_TEXT.test(String(detail)) ? Math.min(transportErrors + 1, errorTimes.length) : 0;
502
517
  if (!tripped && errorTimes.length >= breaker.errors) {
503
- tripped = { at: t, detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
518
+ tripped = { at: t, ...(transportErrors >= errorTimes.length ? { transport: true } : {}), detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
504
519
  update({ issue: null, outcome: "breaker", detail: tripped.detail });
505
520
  }
506
521
  };
@@ -544,11 +559,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
544
559
  }
545
560
  if (parked !== undefined) {
546
561
  done();
547
- const final = { ...row, outcome: "skipped", detail: `dependency #${parked} needs-human` };
562
+ const final = { ...row, outcome: "skipped", detail: `dependency #${parked} needs-human`, ...(dispatchCheck ? { waiting: true } : {}) };
548
563
  results.set(n, final); save();
549
564
  decided({ issue: n, what: `skipped: dependency #${parked} needs-human (not pulled into the batch)`, why: firstLine(row.detail) || "the lane reported an open dependency",
550
565
  alternative: `pull #${parked} in and run it again` });
551
- update(final);
566
+ update({ ...final, trigger: parked });
552
567
  return;
553
568
  }
554
569
  }
@@ -588,7 +603,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
588
603
  row.detail = `gave up after ${deferrals.get(n) ?? 0} deferrals: ${r?.detail ?? r?.code ?? ""}`;
589
604
  }
590
605
  if (row.outcome === "engine-error") {
591
- noteEngineError();
606
+ noteEngineError(r?.detail);
592
607
  if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
593
608
  engineErrors.set(n, (engineErrors.get(n) ?? 0) + 1);
594
609
  requeue(n, 0); done(); save();
@@ -648,7 +663,16 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
648
663
  if (!prior && row.outcome === "budget-exceeded" && typeof relabelNeedsHuman === "function") {
649
664
  try { await relabelNeedsHuman({ issue: n, evidence: `Implement turn budget overrun (\`budget.exceeded\`) not recovered in this batch:\n${String(row.detail ?? "")}` }); } catch { /* best effort */ }
650
665
  }
666
+ // #4656: a mechanical terminal park (recovery spent or none applies) moves to the remediation queue, not the human queue.
667
+ if (row.outcome === "needs-human" && !row.parkedQueued && !everRelabelled.has(n) && typeof relabelNeedsHuman === "function" && (cls => cls === "parked" || (cls === "recoverable" && prior))(classifyNeedsHuman({ ...row, issueEdited: r?.issueEdited === true }, { parked: true }))) { // a recoverable row with a finished recovery has used its attempts
668
+ try { const res = await relabelNeedsHuman({ issue: n, parked: { code: row.code, pr: Number.isInteger(row.pr) ? row.pr : null }, evidence: `Mechanical park \`${row.code}\`${Number.isInteger(row.pr) ? ` (PR #${row.pr})` : ""}:\n${String(row.detail ?? "")}` }); if (res !== false && res !== "failed" && res !== "skipped") row.parkedQueued = true; } catch { /* best effort */ }
669
+ }
651
670
  results.set(n, row);
671
+ if (row.outcome === "merged" && lifecycle && r?.milestone) lifecycleTitles.add(String(r.milestone)); // #4657
672
+ if (row.outcome === "merged" && integration?.onMerged && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
673
+ // #4667: counts the merge toward the milestone's integration run; an error is a row, never the outcome.
674
+ try { await integration.onMerged(n, { ...row, lane: rec?.lane ?? r?.lane }, r); } catch (e) { update({ issue: n, outcome: "integration", detail: `integration check failed: ${String(e?.message ?? e).slice(0, 300)}` }); }
675
+ }
652
676
  if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
653
677
  save(); update(row);
654
678
  // Before the slot frees: the batch never ends with a fold pending, and block_dispatch sees the new flags.
@@ -746,7 +770,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
746
770
  }
747
771
  save();
748
772
 
749
- const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run");
773
+ const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
750
774
  // #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
751
775
  // lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
752
776
  // #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
@@ -759,7 +783,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
759
783
  queued: q.attempt, cause: why, recoveryMax });
760
784
  let res;
761
785
  try { res = await relabelNeedsHuman({ issue: n, evidence }); } catch { res = "failed"; }
762
- if (res !== false && res !== "failed") relabelled.add(n); // a failed label add is retried by a resume
786
+ if (res !== false && res !== "failed") { relabelled.add(n); everRelabelled.add(n); } // a failed label add is retried by a resume
763
787
  }
764
788
  save();
765
789
  })());
@@ -817,6 +841,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
817
841
  decided({ issue: e.issue, what: `requeued by ${e.by ?? "operator"} (forgedock orchestrate --requeue); prior outcome ${row.outcome}${list.length ? `; dependents back to pending: ${list.join(", ")}` : ""}`,
818
842
  why: e.reason || "operator decision", alternative: "start a second batch" });
819
843
  update({ issue: e.issue, outcome: "requeued", detail: `${e.reason || "operator decision"} (was ${row.outcome})` });
844
+ for (const m of moved) if (m !== e.issue) update({ issue: m, outcome: "requeued", detail: `dependency #${e.issue} requeued`, trigger: e.issue }); // #4687
820
845
  n++;
821
846
  }
822
847
  if (fresh.length) save();
@@ -847,6 +872,22 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
847
872
  update({ issue: null, outcome: "release", detail: fmt(freed) });
848
873
  }
849
874
  };
875
+ // #4656: a gate (hold list, settle, ship pre-flight) waiting on an issue the scheduler will not admit: one decision and row per (gate, issue).
876
+ const gateSeen = new Map(), gateReported = new Set();
877
+ const checkGates = () => {
878
+ const gates = [];
879
+ if (opHeld.size) gates.push({ name: "hold", waitsOn: [...opHeld] });
880
+ if (getGates) { try { gates.push(...(getGates() ?? [])); } catch { /* best effort */ } }
881
+ if (!gates.length) return;
882
+ const unadmittable = (n) => { const x = results.get(n); return !(x && (x.outcome === "needs-human" || x.parkedQueued)); };
883
+ for (const d of detectGateDeadlock({ gates, admissible: unadmittable, now: now(), firstSeen: gateSeen })) {
884
+ const key = `${d.gate}#${d.issue}`;
885
+ if (gateReported.has(key)) continue;
886
+ gateReported.add(key);
887
+ decided({ issue: d.issue, what: formatGateDeadlock(d), why: "the gate waits on a parked issue the scheduler will not admit", alternative: "/work-on:remediate the parked issue or release the gate" });
888
+ update({ issue: d.issue, outcome: "GATE_DEADLOCK", detail: formatGateDeadlock(d) });
889
+ }
890
+ };
850
891
  const checkCancel = () => {
851
892
  if (!getCancelled) return;
852
893
  let c = null;
@@ -878,6 +919,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
878
919
  // #4595: a skipped row whose blocker (`dependency #B ...`) was finished outside the lane is re-admitted: B is re-checked
879
920
  // through `dispatchCheck` (at most every WATCH_EVERY_MS, and once before the batch would end); fail-open.
880
921
  let lastWatch = null;
922
+ const settled = new Set(); // #4687: blockers whose close is final (not planned): their dependents stay skipped, no more re-checks
881
923
  const readmitted = new Set(); // "n:b": re-admit n on blocker b once; a lane that re-skips on b stays skipped
882
924
  const blockerOf = (n) => { const r = results.get(n); const m = r?.outcome === "skipped" ? /^dependency #(\d+) /.exec(r.detail ?? "") : null; return m ? Number(m[1]) : null; };
883
925
  const watchable = () => !!dispatchCheck && [...results.keys()].some((n) => blockerOf(n) !== null);
@@ -887,10 +929,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
887
929
  if (!final && lastWatch !== null && t0 - lastWatch < WATCH_EVERY_MS) return;
888
930
  lastWatch = t0;
889
931
  const resolved = new Set();
890
- const roots = new Set([...results.keys()].filter((n) => !readmitted.has(`${n}:${blockerOf(n)}`)).map(blockerOf).filter((b) => b !== null && blockerOf(b) === null));
932
+ const roots = new Set([...results.keys()].filter((n) => !readmitted.has(`${n}:${blockerOf(n)}`)).map(blockerOf).filter((b) => b !== null && blockerOf(b) === null && !settled.has(b)));
891
933
  for (const b of roots) {
892
934
  let g = null;
893
935
  try { g = (await dispatchCheck(b)) ?? null; } catch { g = null; }
936
+ if (g?.final) { // #4687: closed as not planned: terminal for its dependents
937
+ settled.add(b);
938
+ for (const n of results.keys()) {
939
+ const r = results.get(n);
940
+ if (blockerOf(n) === b && r.waiting) { const { waiting, ...rest } = r; results.set(n, rest); update({ ...rest, trigger: b }); }
941
+ }
942
+ continue;
943
+ }
894
944
  if (!g || !SATISFIED.has(g.outcome)) continue;
895
945
  resolved.add(b);
896
946
  const row = results.get(b);
@@ -909,6 +959,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
909
959
  readmitted.add(`${n}:${b}`);
910
960
  results.delete(n);
911
961
  pending.push(n);
962
+ update({ issue: n, outcome: "readmitted", detail: `dependency #${b} merged or closed`, trigger: b }); // #4687
912
963
  decided({ issue: n, what: `re-admitted: dependency #${b} merged or closed after this issue was skipped`, why: "the blocker was finished outside this batch lane",
913
964
  alternative: "leave it skipped until the next batch" });
914
965
  }
@@ -930,18 +981,25 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
930
981
  const n = pending[i];
931
982
  if (results.has(n)) { pending.splice(i, 1); continue; }
932
983
  const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
933
- if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`); again = true; }
984
+ if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`, bad); again = true; }
934
985
  }
935
986
  }
936
987
  const t = now();
937
988
  const holdUntil = heldUntil();
938
989
  let started = 0;
939
990
  driftHeld.clear();
991
+ integHeld.clear();
940
992
  opHeldHit.clear();
941
993
  refreshHeld();
994
+ checkGates();
942
995
  const isReady = (n) => {
943
996
  if (opHeld.has(n)) { opHeldHit.add(n); return false; }
944
997
  if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
998
+ if (integration?.holds) {
999
+ let why = null;
1000
+ try { why = integration.holds(n, { lane: lanes.get(n)?.lane ?? null }); } catch { why = null; }
1001
+ if (why) { integHeld.set(n, why); return false; }
1002
+ }
945
1003
  if (!drift?.holds) return true;
946
1004
  let pkg = null;
947
1005
  try { pkg = drift.holds(n, { body: bodies.get(n) ?? "", lane: lanes.get(n)?.lane ?? null }); } catch { pkg = null; }
@@ -987,6 +1045,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
987
1045
  }
988
1046
  }
989
1047
  if (reg) withdrawExcept(wanted);
1048
+ // #4685: a breaker tripped only by link failures (a router restart) closes itself after a cooldown
1049
+ if (tripped?.transport && now() - tripped.at >= TRANSPORT_COOLDOWN_MS) { tripped = null; errorTimes = []; transportErrors = 0; save(); }
990
1050
  if (tripped && !running.size) { await drainAdds(); break; } // breaker: let running lanes finish, keep the rest pending for a resume
991
1051
  if (cancelled && !running.size) { await drainAdds(); break; } // #4376: cancel: the rest end held for a resume
992
1052
  if (budgetStop && !running.size) { await drainAdds(); break; } // execution budget (stop): the same, the rest end held
@@ -1025,7 +1085,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
1025
1085
  else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
1026
1086
  else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
1027
1087
  else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume
1028
- else if (pending.length && driftHeld.size) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch): pending for a later batch
1088
+ else if (pending.length && (driftHeld.size || integHeld.size)) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch) or a red integration streak (#4667): pending for a later batch
1029
1089
  else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
1030
1090
  for (const n of pending.splice(0)) skip(n, "dependency cycle");
1031
1091
  save();
@@ -1038,6 +1098,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
1038
1098
  }
1039
1099
  if (reg) withdrawExcept(new Set());
1040
1100
  await relabelQueued();
1101
+ // #4657: a milestone whose `close_on` state is reached closes here; a failure is a row, never an outcome change.
1102
+ if (lifecycle?.onBatchEnd && lifecycleTitles.size && !halted) {
1103
+ try { await lifecycle.onBatchEnd([...lifecycleTitles]); } catch (e) { onUpdate({ issue: 0, outcome: "lifecycle", detail: `lifecycle check failed: ${String(e?.message ?? e).slice(0, 300)}` }); }
1104
+ }
1105
+ for (const [n, r] of results) if (r?.waiting) { const { waiting, ...rest } = r; results.set(n, rest); } // #4687: the batch ended: skipped for good
1041
1106
  save();
1042
1107
  const withRequeue = (r) => (r && requeuedInfo.has(r.issue) ? { ...r, requeued: { reason: requeuedInfo.get(r.issue).reason, by: requeuedInfo.get(r.issue).by, prior: requeuedInfo.get(r.issue).prior.map((p) => ({ outcome: p.outcome, detail: p.detail ?? null, pr: p.pr ?? null })) } } : r);
1043
1108
  return order.map((n) => withRequeue(results.get(n)) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: heldDetail(n),
@@ -1076,9 +1141,12 @@ export function formatOutcomeLists(rows) {
1076
1141
  for (const r of rows ?? []) {
1077
1142
  if (!r || r.recovered) continue;
1078
1143
  const human = r.outcome === "needs-human" && classifyNeedsHuman(r) === "human";
1144
+ if (r.parkedQueued && r.outcome === "needs-human") continue; // #4656: listed on the remediation line below, not as unrecovered
1079
1145
  if (!human && (r.outcome === "needs-human" || r.outcome === "budget-exceeded" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
1080
1146
  }
1081
1147
  const esc = (rows ?? []).filter((r) => r?.outcome === "needs-human" && classifyNeedsHuman(r) === "human");
1148
+ const rq = (rows ?? []).filter((r) => r?.outcome === "needs-human" && r.parkedQueued); // #4656: parked past recovery: the single remediation queue, not escalation
1149
+ if (rq.length) out.push(`remediation queue (/work-on:remediate): ${rq.map((r) => `#${r.issue} (parked:${r.code}${r.pr ? ` PR #${r.pr}` : ""})`).join(" ")}`);
1082
1150
  const parked = formatParkedLine(rows); // #4389: one line for every parked post-deploy follow-up, never an escalation
1083
1151
  if (parked) out.push(parked);
1084
1152
  if (esc.length) out.push(["escalations (human decisions, asked once):", ...esc.map((r) => `- #${r.issue} (${r.code}): ${one(r.detail)}`)].join("\n"));
@@ -1089,7 +1157,7 @@ export function formatTable(rows) {
1089
1157
  // #4255: sandbox minutes and cost next to the token cost, only when some lane ran in a sandbox.
1090
1158
  const sbx = rows.some((r) => r?.sandbox);
1091
1159
  const head = ["issue", "outcome", "PR", "cost", ...(sbx ? ["sbx-min", "sbx-$"] : []), "min"];
1092
- const body = rows.map((r) => [`#${r.issue}`, r.outcome + (["skipped", "held"].includes(r.outcome) && r.detail ? ` (${r.detail})` : r.prefilter?.closed || r.prefilter?.parked ? " (pre-filter)" : ""), r.pr ? `#${r.pr}` : "-", `$${(r.cost_usd ?? 0).toFixed(2)}`,
1160
+ const body = rows.map((r) => [`#${r.issue}`, (r.waiting && /^dependency #(\d+) /.test(r.detail ?? "") ? `waiting on #${/^dependency #(\d+) /.exec(r.detail)[1]}` : r.outcome) + (["skipped", "held"].includes(r.outcome) && r.detail ? ` (${r.detail})` : r.prefilter?.closed || r.prefilter?.parked ? " (pre-filter)" : ""), r.pr ? `#${r.pr}` : "-", `$${(r.cost_usd ?? 0).toFixed(2)}`,
1093
1161
  ...(sbx ? (r.sandbox ? [(Number(r.sandbox.minutes) || 0).toFixed(1), `$${(Number(r.sandbox.cost_usd) || 0).toFixed(2)}`] : ["-", "-"]) : []), String(r.minutes ?? 0)]);
1094
1162
  const w = head.map((h, i) => Math.max(h.length, ...body.map((b) => b[i].length)));
1095
1163
  const line = (c) => c.map((x, i) => x.padEnd(w[i])).join(" ").trimEnd();
@@ -0,0 +1,45 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ /**
4
+ * Parked post-deploy follow-up sweep (#4689). A follow-up parked by the #4389 gate (`awaiting-promotion`, `awaiting-due`,
5
+ * `awaiting-measurement`) is re-evaluated on the daemon timer, next to `sweepDue`, instead of waiting for the next batch:
6
+ * `prefilterIssue` with `gateOnly` re-runs the same deterministic gate (git ancestry, due date, soak window, precondition,
7
+ * named checks; no model call, no lane). A released issue loses its wait labels and is picked up by the next scheduler pass;
8
+ * one that is still waiting is left as it is.
9
+ */
10
+ import { prefilterIssue, lazyDefaultBranch, AWAITING_PROMOTION_LABEL, AWAITING_DUE_LABEL } from "./prefilter.mjs";
11
+ import { AWAITING_MEASUREMENT_LABEL } from "./postdeploy-check.mjs";
12
+
13
+ export const PARKED_LABELS = [AWAITING_PROMOTION_LABEL, AWAITING_DUE_LABEL, AWAITING_MEASUREMENT_LABEL];
14
+
15
+ /**
16
+ * Re-evaluate every parked follow-up of the repo. Sequential; an error is an `error` row, never a throw.
17
+ * Outcomes: released | closed | still-parked | skipped | error.
18
+ */
19
+ export async function sweepParked({ repo, cwd = process.cwd(), io, defaultBranch = null, preconditionCmd = null, minAge = null, dryRun = false, onResult = null } = {}) {
20
+ const found = new Set();
21
+ for (const label of PARKED_LABELS) {
22
+ try {
23
+ const list = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--label", label, "--limit", "100", "--json", "number"])));
24
+ for (const i of Array.isArray(list) ? list : []) if (Number.isInteger(i?.number)) found.add(i.number);
25
+ } catch { /* one unreadable label list: the others still run */ }
26
+ }
27
+ const resolve = defaultBranch ?? lazyDefaultBranch({ io, repo, cwd });
28
+ const rows = [];
29
+ for (const issue of [...found].sort((a, b) => a - b)) {
30
+ let row;
31
+ try {
32
+ const r = await prefilterIssue({ issue, repo, cwd, io, dryRun, defaultBranch: resolve, preconditionCmd, minAge, gateOnly: true });
33
+ row = { issue, outcome: /^pre-filter error/.test(r.detail) ? "error" : r.closed ? "closed" : r.parked ? "still-parked" : /^released/.test(r.detail) ? "released" : "skipped", detail: r.detail, kind: r.kind ?? null };
34
+ } catch (e) { row = { issue, outcome: "error", detail: String(e?.message ?? e).slice(0, 160) }; }
35
+ rows.push(row);
36
+ try { if (!dryRun) await onResult?.(row); } catch { /* an event sink never fails the sweep */ }
37
+ }
38
+ return rows;
39
+ }
40
+
41
+ /** One line, e.g. `parked: released #4 #5; closed #6`; "" when nothing changed. */
42
+ export function formatParkedSweepSummary(rows) {
43
+ const parts = [["released", "released"], ["closed", "closed"], ["error", "error"]].map(([k, l]) => [l, (rows ?? []).filter((r) => r.outcome === k).map((r) => `#${r.issue}`)]).filter(([, a]) => a.length).map(([l, a]) => `${l} ${a.join(" ")}`);
44
+ return parts.length ? `parked: ${parts.join("; ")}` : "";
45
+ }
@@ -0,0 +1,71 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ import { describe, it } from "node:test";
4
+ import assert from "node:assert/strict";
5
+ import { sweepParked, formatParkedSweepSummary } from "./parked-sweep.mjs";
6
+
7
+ const MERGE = "a".repeat(40);
8
+ const item = (extra = "") => `- [ ] \`R2: the window shows a row\` — post-deploy: [check:smoke] ${extra} after promotion`;
9
+ const bodyOf = (extra) => `## AC\n\n${item(extra)}\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n`;
10
+ const CHECKS = JSON.stringify({ smoke: "echo ok" });
11
+
12
+ /** Fake repo: `parked` maps issue -> {labels, body}; `promoted` says whether the parent's merge SHA is on origin/main. */
13
+ function world({ parked, promoted, now = Date.parse("2026-11-04T00:00:00Z") }) {
14
+ const w = { edits: [], closed: [], comments: [], commits: [] };
15
+ w.io = {
16
+ now: () => now,
17
+ gh: async (a) => {
18
+ const k = a.slice(0, 2).join(" ");
19
+ if (k === "issue list") { const l = a[a.indexOf("--label") + 1]; return JSON.stringify(Object.entries(parked).filter(([, v]) => v.labels.includes(l)).map(([n]) => ({ number: Number(n) }))); }
20
+ if (k === "issue view") { const n = Number(a[2]); return JSON.stringify({ number: n, state: "OPEN", title: "", comments: [], ...parked[n], labels: parked[n].labels.map((name) => ({ name })) }); }
21
+ if (k === "pr list") return JSON.stringify([{ number: 90, title: "fix (#40)", body: "", mergeCommit: { oid: MERGE } }]);
22
+ if (k === "issue edit") { w.edits.push(a); return ""; }
23
+ if (k === "issue comment") { w.comments.push(a); return ""; }
24
+ if (k === "issue close") { w.closed.push(Number(a[2])); return ""; }
25
+ throw new Error(`unexpected gh ${a.join(" ")}`);
26
+ },
27
+ git: async (a) => {
28
+ if (a[0] === "fetch") return "";
29
+ if (a[0] === "rev-parse") return "deadbeef\n";
30
+ if (a[0] === "show") return CHECKS;
31
+ if (a[0] === "merge-base") { if (promoted) return ""; throw Object.assign(new Error("exit 1"), { code: 1 }); }
32
+ throw new Error(`unexpected git ${a.join(" ")}`);
33
+ },
34
+ spawnCheck: async () => ({ code: 0, stdout: "ok", stderr: "" }),
35
+ };
36
+ return w;
37
+ }
38
+ const sweep = (w, extra = {}) => sweepParked({ repo: "o/r", cwd: "/repo", io: w.io, defaultBranch: "main", ...extra });
39
+
40
+ describe("parked follow-up sweep (#4689)", () => {
41
+ it("one pass after the parent's commit reaches the default branch removes awaiting-promotion and measures; before it nothing changes", async () => {
42
+ const parked = { 41: { labels: ["awaiting-promotion"], body: bodyOf() } };
43
+ const before = world({ parked, promoted: false });
44
+ const rows0 = await sweep(before);
45
+ assert.deepEqual(rows0.map((r) => [r.issue, r.outcome, r.kind]), [[41, "still-parked", "awaiting-promotion"]]);
46
+ assert.deepEqual(before.closed, []);
47
+ const after = world({ parked, promoted: true });
48
+ const rows = await sweep(after, { dryRun: false });
49
+ assert.ok(after.edits.some((a) => a.includes("--remove-label") && a.includes("awaiting-promotion")), "label removed");
50
+ assert.equal(rows[0].issue, 41); assert.notEqual(rows[0].outcome, "still-parked");
51
+ assert.equal(formatParkedSweepSummary([{ issue: 41, outcome: "released" }, { issue: 5, outcome: "error" }]), "parked: released #41; error #5");
52
+ });
53
+
54
+ it("a due date in the past is released, a future one stays parked", async () => {
55
+ const parked = { 50: { labels: ["awaiting-due", "due:2026-11-03"], body: bodyOf("due:2026-11-03") }, 51: { labels: ["awaiting-due"], body: bodyOf("due:2026-12-01") } };
56
+ const w = world({ parked, promoted: true });
57
+ const rows = await sweep(w);
58
+ assert.equal(rows.find((r) => r.issue === 51).outcome, "still-parked");
59
+ assert.notEqual(rows.find((r) => r.issue === 50).outcome, "still-parked");
60
+ assert.ok(w.edits.some((a) => a[2] === "50" && a.includes("--remove-label") && /awaiting-due,due:2026-11-03/.test(a.join(" "))));
61
+ });
62
+
63
+ it("an unreadable issue is an error row, never a throw; a dry run changes nothing", async () => {
64
+ const w = world({ parked: { 60: { labels: ["awaiting-promotion"], body: bodyOf() } }, promoted: true });
65
+ const g = w.io.gh; w.io.gh = async (a) => { if (a[1] === "view") throw new Error("boom"); return g(a); };
66
+ assert.equal((await sweep(w))[0].outcome, "error");
67
+ const w2 = world({ parked: { 61: { labels: ["awaiting-promotion"], body: bodyOf() } }, promoted: true });
68
+ await sweep(w2, { dryRun: true });
69
+ assert.deepEqual(w2.edits, []); assert.deepEqual(w2.closed, []);
70
+ });
71
+ });
@@ -17,6 +17,7 @@ export const CHECKS_FILE = ".forge/post-deploy-checks.json";
17
17
  export const CHECK_ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
18
18
  export const AWAITING_MEASUREMENT_LABEL = "awaiting-measurement";
19
19
  export const MONITORING_LABEL = "post-deploy-monitoring";
20
+ export const OWNER_ASK_MARKER = "FORGE:OWNER_ASK"; // #4689
20
21
  export const DEFAULT_EVERY_MS = 6 * 3600e3;
21
22
  const ITEM_RE = /^\s*[-*]\s+\[[ xX]\]\s+(.*)$/;
22
23
  const CHECK_TOKEN_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:\s*\[check:\s*([^\]\s]*)\s*\]/;
@@ -97,6 +98,26 @@ export async function addMonitoringLabel({ io, repo, issue }) {
97
98
  }
98
99
  }
99
100
 
101
+ /** #4689: tick every unticked post-deploy checklist item of a follow-up body (the evidence is in the FORGE:PROBE comment). */
102
+ export const tickItems = (body) => String(body ?? "").replace(/^(\s*[-*]\s+)\[ \](\s+`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:)/gm, "$1[x]$2");
103
+
104
+ /** #4689: one typed `FORGE:OWNER_ASK` comment for the items with no runnable check, posted once; best effort, nothing in a dry run. */
105
+ async function postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def }) {
106
+ if (dryRun || (iss.comments ?? []).some((c) => isTrusted(c) && String(c?.body ?? "").includes(OWNER_ASK_MARKER))) return;
107
+ const what = ids.length ? missing.map((m) => (m ? `add \`${m}\` to ${CHECKS_FILE} on ${def}` : "name an item's check with `[check:<id>]`")).join("; ") : "name each item's check with `[check:<id>]` and add it to the checks file";
108
+ try { await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", `<!-- ${OWNER_ASK_MARKER} -->\n## Owner ask\nNo lane was started and none will be. Needed from the owner: ${what}, or measure the item by hand.`]); } catch { /* the label still shows it */ }
109
+ }
110
+
111
+ /** #4689: one engineering issue per failing check (never a lane): the output, the fix and the follow-up. Best effort. */
112
+ async function fileFailureIssues({ bad, iss, repo, io, parent, def, fix }) {
113
+ for (const r of bad) {
114
+ const title = `Post-deploy check ${r.id} fails on ${def}${parent ? ` (fix of #${parent})` : ""}`;
115
+ const body = `Post-deploy check \`${r.id}\` failed (exit ${r.exit ?? "timeout"}) for fix \`${fix.slice(0, 12)}\` on \`${def}\`. Follow-up: #${iss.number}.\n\n${fenced(r.output)}`.slice(0, OUTPUT_TAIL * 2);
116
+ const create = (extra) => io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body, ...extra]);
117
+ try { await create(["--label", "followup:engineering"]); } catch { try { await create([]); } catch { /* the follow-up comment carries the output */ } }
118
+ }
119
+ }
120
+
100
121
  /**
101
122
  * Measure a ready post-deploy follow-up. Never runs a command that is not in the default branch's checks file.
102
123
  * @returns {Promise<{state:"awaiting-measurement"|"pass"|"fail"|"recorded-fail"|"dry-run"|"monitoring", detail:string}>}
@@ -107,13 +128,19 @@ export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd,
107
128
  const ids = itemCheckIds(iss.body);
108
129
  const checks = await checksAt(io.git, [`origin/${def}`], cwd);
109
130
  const missing = ids.filter((id) => !id || !checks[id]);
110
- if (!ids.length || missing.length) return { state: "awaiting-measurement", detail: `no runnable check on ${def} for ${ids.length ? `${missing.map((m) => m ?? "an item without a check id").join(", ")}` : "any item"} (${CHECKS_FILE})` };
131
+ if (!ids.length || missing.length) {
132
+ await postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def });
133
+ return { state: "awaiting-measurement", detail: `no runnable check on ${def} for ${ids.length ? `${missing.map((m) => m ?? "an item without a check id").join(", ")}` : "any item"} (${CHECKS_FILE})` };
134
+ }
111
135
  const fix = [...shas].sort().join(",");
112
136
  const prior = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => probeResultOf(c?.body)).filter(Boolean).reverse().find((p) => p.fix === fix);
113
137
  if (prior?.result === "fail") return { state: "recorded-fail", detail: `check failed for ${fix.slice(0, 12)} (FORGE:PROBE); waiting for a new fix of #${parent}` };
114
138
  const windowed = ids.every((id) => checks[id].forMs);
115
139
  const monitored = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => monitorRecord(c?.body)).filter(Boolean).find((p) => p.fix === fix && p.result === "pass" && p.until);
116
- if (windowed && monitored) return { state: "monitoring", detail: `windowed checks ${[...new Set(ids)].join(", ")} monitored until ${new Date(monitored.until).toISOString()}` };
140
+ if (windowed && monitored) {
141
+ await addMonitoringLabel({ io, repo, issue: iss.number }); // retry a label that failed on the first pass; sweepDue selects by it
142
+ return { state: "monitoring", detail: `windowed checks ${[...new Set(ids)].join(", ")} monitored until ${new Date(monitored.until).toISOString()}` };
143
+ }
117
144
  if (dryRun) return { state: "dry-run", detail: `checks ${[...new Set(ids)].join(", ")} not run (dry run)` };
118
145
  let defaultSha = null;
119
146
  try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* recorded as null */ }
@@ -129,16 +156,21 @@ export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd,
129
156
  `Ran after the fix of #${parent} reached \`${def}\`; commands come from \`${CHECKS_FILE}\` on \`${def}\`.${ok ? (win.at ? ` Monitored until \`${win.window_until}\`, re-run on later releases.` : " The follow-up is closed.") : " The parent is reopened."}`,
130
157
  ...rows.filter((r) => r.result === "fail").flatMap((r) => ["", `**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output)]),
131
158
  ].join("\n").slice(0, OUTPUT_TAIL * 4);
132
- await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
133
159
  if (!ok) {
134
160
  const bad = rows.filter((r) => r.result === "fail");
135
- let note = "";
136
- try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) { note = ` (reopen failed: ${String(e?.message ?? e).slice(0, 120)})`; }
161
+ // Reopen before recording: the FORGE:PROBE fail record parks later passes (recorded-fail), so it must only exist once the parent is open.
162
+ try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) {
163
+ return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; reopen of #${parent} failed, no record posted so the next pass retries (${String(e?.message ?? e).slice(0, 120)})` };
164
+ }
165
+ await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
166
+ await fileFailureIssues({ bad, iss, repo, io, parent, def, fix }); // #4689
137
167
  try {
138
168
  await io.gh(["issue", "comment", String(parent), "-R", repo, "--body", `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} failed for the fix on \`${def}\` (follow-up #${iss.number}); reopened.\n\n${bad.map((r) => fenced(r.output)).join("\n\n")}`.slice(0, OUTPUT_TAIL * 4)]);
139
169
  } catch { /* the follow-up comment carries the output */ }
140
- return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened${note}` };
170
+ return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened` };
141
171
  }
172
+ await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
173
+ if (tickItems(iss.body) !== iss.body) { try { await io.gh(["issue", "edit", String(iss.number), "-R", repo, "--body", tickItems(iss.body)]); } catch { /* the PROBE record is the evidence */ } } // #4689
142
174
  }
143
175
  if (windowed) {
144
176
  await addMonitoringLabel({ io, repo, issue: iss.number });
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: AGPL-3.0-or-later
3
3
  import { describe, it } from "node:test";
4
4
  import assert from "node:assert/strict";
5
- import { parseChecks, itemCheckIds, measureFollowup, monitorRecord, CHECK_ID_RE } from "./postdeploy-check.mjs";
5
+ import { parseChecks, itemCheckIds, measureFollowup, monitorRecord, CHECK_ID_RE, tickItems } from "./postdeploy-check.mjs";
6
6
  import { postDeployOnly } from "./prefilter.mjs";
7
7
  import { validateContract, applyAmendments, precheck, followupItems } from "./conformance.mjs";
8
8
  import { PROBE_MARKER } from "./probe.mjs";
@@ -63,7 +63,7 @@ describe("post-deploy named checks (#4626)", () => {
63
63
  for (const [file, b] of [[CHECKS, body([item("R1", "[check:ghost] x")])], [null, body([item("R1", "[check:smoke] x")])], [CHECKS, body([item("R1", "x"), item("R2", "[check:smoke] y")])]]) {
64
64
  const w = world({ file });
65
65
  assert.equal((await run(w, b)).state, "awaiting-measurement");
66
- assert.deepEqual(w.spawned, []); assert.deepEqual(w.gh, []);
66
+ assert.deepEqual(w.spawned, []); assert.ok(w.gh.every((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1])), "only the typed owner ask (#4689)");
67
67
  }
68
68
  });
69
69
 
@@ -142,5 +142,65 @@ describe("windowed post-deploy checks (#4627)", () => {
142
142
  assert.equal((await run(w, body([item("R1", "[check:soak] x")]))).state, "fail");
143
143
  assert.ok(w.gh.some((a) => a[1] === "reopen" && a[2] === "40"));
144
144
  });
145
+ it("a failed parent reopen posts no fail record, so the next pass re-runs and retries", async () => {
146
+ const w = world();
147
+ const gh = w.io.gh;
148
+ let failReopen = true;
149
+ w.io.gh = async (a) => { if (a[1] === "reopen" && failReopen) { w.gh.push(a); throw new Error("api down"); } return gh(a); };
150
+ const b = body([item("R1", "[check:bad] x")]);
151
+ const r = await run(w, b);
152
+ assert.equal(r.state, "fail"); assert.match(r.detail, /reopen of #40 failed/);
153
+ assert.ok(!w.gh.some((a) => a[1] === "comment"));
154
+ failReopen = false;
155
+ const r2 = await run(w, b);
156
+ assert.ok(w.gh.filter((a) => a[1] === "reopen").length === 2 && /#40 reopened/.test(r2.detail));
157
+ const order = w.gh.map((a) => a[1] === "comment" ? `c${a[2]}` : a[1]);
158
+ assert.ok(order.indexOf("reopen", 1) < order.indexOf("c41"));
159
+ });
160
+ it("a recorded windowed pass retries the monitoring label", async () => {
161
+ const w = world({ file: WIN });
162
+ const rec = { body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "abc", result: "pass", at: new Date(NOW).toISOString(), window_until: new Date(NOW + 1e9).toISOString() })}` };
163
+ const b = body([item("R1", "[check:soak] x")]);
164
+ await run(w, b, { iss: { number: 41, body: b, comments: [rec] }, isTrusted: () => true });
165
+ assert.ok(w.gh.some((a) => a[1] === "edit" && a.includes("post-deploy-monitoring")));
166
+ });
145
167
  });
146
168
 
169
+
170
+ describe("verification outcome (#4689)", () => {
171
+ it("a pass ticks the post-deploy items with the evidence in the PROBE record, and writes no code", async () => {
172
+ const w = world();
173
+ const r = await run(w, body([item("R1", "[check:smoke] x")]));
174
+ assert.equal(r.state, "pass");
175
+ const edit = w.gh.find((a) => a[0] === "issue" && a[1] === "edit");
176
+ const text = edit[edit.indexOf("--body") + 1];
177
+ assert.match(text, /- \[x\] `R1: some quote here`/);
178
+ assert.equal(tickItems("- [ ] `R9: other` — defer: code"), "- [ ] `R9: other` — defer: code");
179
+ const probe = w.gh.find((a) => a[1] === "comment" && /FORGE:PROBE/.test(a[a.indexOf("--body") + 1]));
180
+ assert.match(probe[probe.indexOf("--body") + 1], /"id":"smoke","result":"pass"/);
181
+ assert.ok(w.git.every((a) => ["show", "rev-parse"].includes(a[0])), "no git write: " + JSON.stringify(w.git));
182
+ assert.equal(tickItems("no items"), "no items");
183
+ });
184
+
185
+ it("a fail files one engineering issue per failing check and ticks nothing", async () => {
186
+ const w = world({ file: JSON.stringify({ bad: "echo nope", worse: "echo nope", smoke: "echo ok" }) });
187
+ const r = await run(w, body([item("R1", "[check:bad] x"), item("R2", "[check:worse] x"), item("R3", "[check:smoke] x")]));
188
+ assert.equal(r.state, "fail");
189
+ const created = w.gh.filter((a) => a[0] === "issue" && a[1] === "create");
190
+ assert.equal(created.length, 2);
191
+ assert.ok(created.every((a) => a.includes("followup:engineering")));
192
+ assert.ok(created[0][created[0].indexOf("--title") + 1].includes("bad"));
193
+ assert.ok(!w.gh.some((a) => a[1] === "edit" && a.includes("--body")));
194
+ });
195
+
196
+ it("an item with no runnable check posts one typed owner ask, once", async () => {
197
+ const w = world({ file: null });
198
+ assert.equal((await run(w, body([item("R1", "[check:smoke] x")]))).state, "awaiting-measurement");
199
+ const asks = w.gh.filter((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1]));
200
+ assert.equal(asks.length, 1);
201
+ const seen = { number: 41, body: body([item("R1", "[check:smoke] x")]), comments: [{ body: "<!-- FORGE:OWNER_ASK -->\nask", author: { login: "rapiercraft-forgedock" }, authorAssociation: "CONTRIBUTOR" }] };
202
+ const w2 = world({ file: null });
203
+ await run(w2, seen.body, { iss: seen, isTrusted: () => true });
204
+ assert.equal(w2.gh.filter((a) => a[1] === "comment").length, 0);
205
+ });
206
+ });
@@ -13,6 +13,15 @@ import { deferredFrom, isTrustedIssueComment } from "./prefilter.mjs";
13
13
 
14
14
  export const REGRESSION_MARKER = "FORGE:REGRESSION";
15
15
  const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
16
+ /** An existing regression issue (any state) for this follow-up and fix commit, so a retry after a failed finish() reuses it (#4653). */
17
+ export async function findRegression(io, repo, issue, fix) {
18
+ const found = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "all", "--search", `"${REGRESSION_MARKER}" "Monitored follow-up: #${issue}" in:body`, "--limit", "30", "--json", "number,body"])));
19
+ const hit = (Array.isArray(found) ? found : []).find((i) => {
20
+ const b = String(i?.body ?? "");
21
+ return b.includes(`<!-- ${REGRESSION_MARKER} -->`) && new RegExp(`Monitored follow-up: #${issue}(?!\\d)`).test(b) && b.includes(`Fix commit: \`${fix}\``);
22
+ });
23
+ return hit ? String(hit.number) : null;
24
+ }
16
25
  const short = (s) => String(s ?? "unknown").slice(0, 12);
17
26
 
18
27
  /** One monitored follow-up. Outcomes: skipped | unrunnable | not-due | due (dry run) | pass | window-ended | fail. */
@@ -53,8 +62,9 @@ export async function monitorOne({ issue, repo, def, defaultSha, cwd, io, now, d
53
62
  `- Fix commit: \`${last.fix}\``, `- Last passing release (\`${def}\`): \`${last.defaultSha ?? "unknown"}\``, `- Breaking release (\`${def}\`): \`${defaultSha ?? "unknown"}\``,
54
63
  `- Monitored follow-up: #${issue}${parent ? `; parent: #${parent}` : ""}`, "", ...bad.flatMap((r) => [`**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output), ""]),
55
64
  ].join("\n").slice(0, OUTPUT_TAIL * 4);
56
- const created = String(await io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body]));
57
- const num = /\/issues\/(\d+)/.exec(created)?.[1] ?? null;
65
+ const existing = await findRegression(io, repo, issue, last.fix);
66
+ const created = existing ? "" : String(await io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body]));
67
+ const num = existing ?? /\/issues\/(\d+)/.exec(created)?.[1] ?? null;
58
68
  await finish(`<!-- ${PROBE_MARKER} -->\n## Post-deploy check: fail\n${JSON.stringify({ ...rec, regression: num ? Number(num) : null, checks: sum })}\n\nRegression issue ${num ? `#${num}` : "filed"}; monitoring stops.`);
59
69
  return row("fail", `regression ${num ? `#${num}` : "filed"}: ${bad.map((r) => r.id).join(", ")} broke at ${short(defaultSha)} (fix ${short(last.fix)})`, { fix: last.fix, regression: num });
60
70
  }