forgedock 1.36.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/bot-token-status.mjs +26 -0
- package/bin/engine/events.mjs +10 -1
- package/bin/engine/events.test.mjs +15 -1
- package/bin/engine/outcome-codes.mjs +39 -1
- package/bin/engine/phases.mjs +65 -5
- package/bin/engine/telemetry-seam.mjs +1 -1
- package/bin/engine/telemetry-seam.test.mjs +4 -0
- package/bin/engine-cli.mjs +268 -37
- package/bin/engine.mjs +3 -1
- package/bin/forgedock.mjs +108 -5
- package/bin/labels.json +50 -0
- package/bin/lean/batch.mjs +10 -4
- package/bin/lean/batch.test.mjs +75 -1
- package/bin/lean/conformance.e2e.test.mjs +1 -1
- package/bin/lean/conformance.mjs +125 -11
- package/bin/lean/conformance.test.mjs +122 -1
- package/bin/lean/decisions.mjs +181 -0
- package/bin/lean/decisions.test.mjs +71 -0
- package/bin/lean/effects.mjs +29 -14
- package/bin/lean/execution.mjs +60 -16
- package/bin/lean/execution.test.mjs +59 -2
- package/bin/lean/gatewatch.mjs +31 -0
- package/bin/lean/gatewatch.test.mjs +59 -0
- package/bin/lean/guardgate.mjs +170 -0
- package/bin/lean/guardgate.test.mjs +153 -0
- package/bin/lean/guards.mjs +243 -0
- package/bin/lean/guards.test.mjs +177 -0
- package/bin/lean/heavy.mjs +54 -8
- package/bin/lean/heavy.test.mjs +74 -1
- package/bin/lean/hostkill.mjs +1 -1
- package/bin/lean/integration.mjs +276 -0
- package/bin/lean/integration.test.mjs +189 -0
- package/bin/lean/lean.mjs +134 -28
- package/bin/lean/lean.test.mjs +239 -22
- package/bin/lean/lifecycle.mjs +129 -0
- package/bin/lean/lifecycle.test.mjs +118 -0
- package/bin/lean/light.mjs +1 -1
- package/bin/lean/milestonesync.mjs +130 -0
- package/bin/lean/milestonesync.test.mjs +133 -0
- package/bin/lean/orchestrate.mjs +83 -15
- package/bin/lean/parked-sweep.mjs +45 -0
- package/bin/lean/parked-sweep.test.mjs +71 -0
- package/bin/lean/postdeploy-check.mjs +38 -6
- package/bin/lean/postdeploy-check.test.mjs +62 -2
- package/bin/lean/postdeploy-monitor.mjs +12 -2
- package/bin/lean/postdeploy-monitor.test.mjs +27 -1
- package/bin/lean/prefilter.mjs +83 -23
- package/bin/lean/prefilter.test.mjs +73 -1
- package/bin/lean/probe.mjs +2 -2
- package/bin/lean/probe.test.mjs +1 -1
- package/bin/lean/prompts.mjs +15 -9
- package/bin/lean/resume.test.mjs +1 -1
- package/bin/lean/retry.mjs +28 -0
- package/bin/lean/sandbox-provider.contract.test.mjs +8 -3
- package/bin/lean/sandbox-remote.mjs +4 -4
- package/bin/lean/schedd.mjs +53 -9
- package/bin/lean/schedd.test.mjs +77 -5
- package/bin/lean/sequence.mjs +24 -0
- package/bin/lean/sequence.test.mjs +31 -1
- package/bin/lean/signoff.mjs +66 -0
- package/bin/lean/signoff.test.mjs +102 -0
- package/bin/lean/train.mjs +177 -0
- package/bin/lean/train.test.mjs +113 -0
- package/bin/pool/cli.mjs +2 -0
- package/bin/pool/health.mjs +30 -0
- package/bin/review-context.mjs +298 -0
- package/bin/runner.mjs +26 -3
- package/bin/stats/compounding.mjs +157 -0
- package/bin/stats/compounding.test.mjs +74 -0
- package/bin/stats/index.mjs +21 -2
- package/commands/bootstrap.md +10 -8
- package/commands/deploy-info.md +3 -0
- package/commands/issue.md +2 -0
- package/commands/milestone.md +43 -54
- package/commands/orchestrate/phase-4-execution.md +2 -0
- package/commands/orchestrate.md +8 -1
- package/commands/pipeline-status.md +20 -0
- package/commands/review-pr.md +1 -0
- package/commands/work-on/build/implement.md +1 -1
- package/commands/work-on/build.md +4 -2
- package/commands/work-on/investigate.md +3 -1
- package/commands/work-on/remediate.md +1 -1
- package/docs/CONFIG.md +71 -2
- package/docs/EVENTS.md +1 -4
- package/docs/OUTCOME-CODES.md +9 -1
- package/docs/STATS.md +20 -0
- package/docs/site/start-a-new-project.md +1 -1
- package/docs/spec/label-state-machine.md +6 -0
- package/package.json +1 -1
- package/pool/install/install-agent.sh +4 -2
- package/pool/install/install-router.sh +4 -2
- package/pool/install/install-schedd.sh +28 -0
- package/pool/install/unit-hardening.sh +34 -0
- package/scripts/acceptance-red-green.sh +19 -2
- package/scripts/acceptance-red-green.test.sh +2 -0
- package/scripts/bootstrap-create.mjs +3 -2
- package/scripts/bootstrap-validate.mjs +31 -4
- package/scripts/file-deferred-followups.sh +285 -77
- package/scripts/file-deferred-followups.test.sh +111 -4
- package/scripts/hotspots.sh +23 -2
- package/scripts/hotspots.test.sh +9 -0
- package/scripts/plan-validate-live.mjs +111 -0
- package/scripts/readiness.mjs +141 -0
- package/scripts/review-prepare.sh +17 -1
- package/scripts/review-prepare.test.sh +17 -0
- package/scripts/settle-scope.mjs +74 -0
- package/scripts/systemd/forgedock-bot-token-refresh.service +7 -0
- package/scripts/systemd/forgedock-bot-token-refresh.timer +11 -0
- package/scripts/wait-ci-green.sh +14 -3
- package/scripts/wait-ci-green.test.sh +3 -1
- package/templates/bootstrap/needs-human-issue.md +4 -1
- package/templates/github-actions/migration-journal-guard.test.mjs +26 -0
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -37,6 +37,8 @@
|
|
|
37
37
|
* issue while maintenance starts stay within `share` of all lane starts (`dispatch: {started, maint}`, persisted), then
|
|
38
38
|
* P2+ features, then (`whenIdle`) maintenance once no feature is pending. Maintenance the budget never admits ends `held`.
|
|
39
39
|
*
|
|
40
|
+
* Milestone lifecycle (#4657), off unless `lifecycle` is passed (`bin/lean/lifecycle.mjs` makeLifecycle): at batch end
|
|
41
|
+
* `lifecycle.onBatchEnd(titles)` advances/closes each milestone a `merged` row's result named. Errors never change an outcome.
|
|
40
42
|
* Milestone drift (#4161), off unless `drift` is passed (`bin/lean/drift.mjs` makeDrift): a `merged` milestone-lane run
|
|
41
43
|
* whose result carries `structure` awaits `drift.onMerged(issue, row, result)` before its slot frees. A drift error is
|
|
42
44
|
* reported as a `drift` row and never changes the outcome. With `drift.holds(issue, {body, lane})` (block_dispatch), a
|
|
@@ -100,6 +102,7 @@
|
|
|
100
102
|
*/
|
|
101
103
|
import { parseDependsOn } from "../engine/phases.mjs";
|
|
102
104
|
import { classifyNeedsHuman, recoveryCodeOf, OUTCOME_CODES } from "../engine/outcome-codes.mjs";
|
|
105
|
+
import { detectGateDeadlock, formatGateDeadlock } from "./gatewatch.mjs";
|
|
103
106
|
import { recoveryEvidence } from "./render.mjs";
|
|
104
107
|
import { renderAlerts } from "../stats/render.mjs";
|
|
105
108
|
import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
|
|
@@ -236,9 +239,9 @@ const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k),
|
|
|
236
239
|
*/
|
|
237
240
|
export async function orchestrate({ issues, parallel = 3, runOne, readBody, prefilter = null, resolveLane = null, now = () => Date.now(), onUpdate = () => {}, onCrossLane = () => {},
|
|
238
241
|
store = null, adopt = null, getParallel = null, getAdded = null, getRequeued = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
239
|
-
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
240
|
-
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, stopLane = null,
|
|
241
|
-
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
|
|
242
|
+
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
243
|
+
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, getHeld = null, getGates = null, stopLane = null,
|
|
244
|
+
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null }) {
|
|
242
245
|
const saved = store?.load?.() ?? null;
|
|
243
246
|
const order = saved ? [...saved.order] : [];
|
|
244
247
|
const set = new Set(order);
|
|
@@ -273,7 +276,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
273
276
|
const parkedBase = new Map(); // #4525: issue -> base sha its parked PR was last synced (or reported conflicting) against
|
|
274
277
|
let lastParkedSync = null;
|
|
275
278
|
const bodies = new Map(); // issue -> body, kept in memory for drift.holds only
|
|
279
|
+
const lifecycleTitles = new Set(); // #4657: milestones this batch merged into, re-evaluated at batch end
|
|
276
280
|
const driftHeld = new Map(); // issue -> flagged package dir holding it this pass
|
|
281
|
+
const integHeld = new Map(); // issue -> reason (red integration streak on its milestone lane, #4667) holding it this pass
|
|
277
282
|
const reg = sched?.registry ?? null;
|
|
278
283
|
const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
|
|
279
284
|
const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
|
|
@@ -304,6 +309,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
304
309
|
const recovering = toMap(saved?.recovering); // issue -> {code, attempt, detail, outcome, ...} of the recovery lane queued or running
|
|
305
310
|
const requeueApplied = new Set(saved?.requeueApplied ?? []); // #4550: `--requeue` entries already applied (batch `requeue` file)
|
|
306
311
|
const requeuedInfo = toMap(saved?.requeued); // #4550: issue -> {reason, by, at, prior: [earlier rows]}
|
|
312
|
+
const everRelabelled = new Set(saved?.relabelled ?? []); // #4656: issues the runner already relabelled; never parked on top
|
|
307
313
|
const relabelled = new Set(saved?.relabelled ?? []); // #4315: queued recoveries relabelled needs-human when a batch stopped
|
|
308
314
|
// #4267: decisions are recorded, never awaited: a failing or slow recorder cannot block the batch.
|
|
309
315
|
const decided = (d) => { try { Promise.resolve(decide?.(d)).catch(() => {}); } catch { /* best effort */ } };
|
|
@@ -493,14 +499,23 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
493
499
|
return true;
|
|
494
500
|
};
|
|
495
501
|
|
|
496
|
-
|
|
502
|
+
// #4687: a dependency skip is a waiting state while a blocker re-check can still re-admit it (`waiting`, cleared when the batch ends);
|
|
503
|
+
// the log row names the triggering issue.
|
|
504
|
+
const skip = (n, detail, trigger = null) => {
|
|
505
|
+
results.set(n, { issue: n, outcome: "skipped", pr: null, cost_usd: 0, minutes: 0, detail, ...(trigger !== null && dispatchCheck ? { waiting: true } : {}) });
|
|
506
|
+
update({ ...results.get(n), ...(trigger !== null ? { trigger } : {}) });
|
|
507
|
+
};
|
|
497
508
|
const requeue = (n, delayMs) => { pending.push(n); enqueued.set(n, now()); if (delayMs > 0) notBefore.set(n, now() + delayMs); else notBefore.delete(n); };
|
|
498
|
-
const
|
|
509
|
+
const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]/i;
|
|
510
|
+
const TRANSPORT_COOLDOWN_MS = 5 * 60000;
|
|
511
|
+
let transportErrors = 0; // #4685: engine-errors in the current window that were link failures
|
|
512
|
+
const noteEngineError = (detail = "") => {
|
|
499
513
|
if (!breaker) return;
|
|
500
514
|
const t = now(), windowMs = breaker.window_min * 60000;
|
|
501
515
|
errorTimes = [...errorTimes.filter((x) => t - x < windowMs), t];
|
|
516
|
+
transportErrors = TRANSPORT_TEXT.test(String(detail)) ? Math.min(transportErrors + 1, errorTimes.length) : 0;
|
|
502
517
|
if (!tripped && errorTimes.length >= breaker.errors) {
|
|
503
|
-
tripped = { at: t, detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
|
|
518
|
+
tripped = { at: t, ...(transportErrors >= errorTimes.length ? { transport: true } : {}), detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
|
|
504
519
|
update({ issue: null, outcome: "breaker", detail: tripped.detail });
|
|
505
520
|
}
|
|
506
521
|
};
|
|
@@ -544,11 +559,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
544
559
|
}
|
|
545
560
|
if (parked !== undefined) {
|
|
546
561
|
done();
|
|
547
|
-
const final = { ...row, outcome: "skipped", detail: `dependency #${parked} needs-human
|
|
562
|
+
const final = { ...row, outcome: "skipped", detail: `dependency #${parked} needs-human`, ...(dispatchCheck ? { waiting: true } : {}) };
|
|
548
563
|
results.set(n, final); save();
|
|
549
564
|
decided({ issue: n, what: `skipped: dependency #${parked} needs-human (not pulled into the batch)`, why: firstLine(row.detail) || "the lane reported an open dependency",
|
|
550
565
|
alternative: `pull #${parked} in and run it again` });
|
|
551
|
-
update(final);
|
|
566
|
+
update({ ...final, trigger: parked });
|
|
552
567
|
return;
|
|
553
568
|
}
|
|
554
569
|
}
|
|
@@ -588,7 +603,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
588
603
|
row.detail = `gave up after ${deferrals.get(n) ?? 0} deferrals: ${r?.detail ?? r?.code ?? ""}`;
|
|
589
604
|
}
|
|
590
605
|
if (row.outcome === "engine-error") {
|
|
591
|
-
noteEngineError();
|
|
606
|
+
noteEngineError(r?.detail);
|
|
592
607
|
if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
|
|
593
608
|
engineErrors.set(n, (engineErrors.get(n) ?? 0) + 1);
|
|
594
609
|
requeue(n, 0); done(); save();
|
|
@@ -648,7 +663,16 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
648
663
|
if (!prior && row.outcome === "budget-exceeded" && typeof relabelNeedsHuman === "function") {
|
|
649
664
|
try { await relabelNeedsHuman({ issue: n, evidence: `Implement turn budget overrun (\`budget.exceeded\`) not recovered in this batch:\n${String(row.detail ?? "")}` }); } catch { /* best effort */ }
|
|
650
665
|
}
|
|
666
|
+
// #4656: a mechanical terminal park (recovery spent or none applies) moves to the remediation queue, not the human queue.
|
|
667
|
+
if (row.outcome === "needs-human" && !row.parkedQueued && !everRelabelled.has(n) && typeof relabelNeedsHuman === "function" && (cls => cls === "parked" || (cls === "recoverable" && prior))(classifyNeedsHuman({ ...row, issueEdited: r?.issueEdited === true }, { parked: true }))) { // a recoverable row with a finished recovery has used its attempts
|
|
668
|
+
try { const res = await relabelNeedsHuman({ issue: n, parked: { code: row.code, pr: Number.isInteger(row.pr) ? row.pr : null }, evidence: `Mechanical park \`${row.code}\`${Number.isInteger(row.pr) ? ` (PR #${row.pr})` : ""}:\n${String(row.detail ?? "")}` }); if (res !== false && res !== "failed" && res !== "skipped") row.parkedQueued = true; } catch { /* best effort */ }
|
|
669
|
+
}
|
|
651
670
|
results.set(n, row);
|
|
671
|
+
if (row.outcome === "merged" && lifecycle && r?.milestone) lifecycleTitles.add(String(r.milestone)); // #4657
|
|
672
|
+
if (row.outcome === "merged" && integration?.onMerged && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
|
|
673
|
+
// #4667: counts the merge toward the milestone's integration run; an error is a row, never the outcome.
|
|
674
|
+
try { await integration.onMerged(n, { ...row, lane: rec?.lane ?? r?.lane }, r); } catch (e) { update({ issue: n, outcome: "integration", detail: `integration check failed: ${String(e?.message ?? e).slice(0, 300)}` }); }
|
|
675
|
+
}
|
|
652
676
|
if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
|
|
653
677
|
save(); update(row);
|
|
654
678
|
// Before the slot frees: the batch never ends with a fold pending, and block_dispatch sees the new flags.
|
|
@@ -746,7 +770,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
746
770
|
}
|
|
747
771
|
save();
|
|
748
772
|
|
|
749
|
-
const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run");
|
|
773
|
+
const heldDetail = (n) => cancelled ? "cancelled by operator" : opHeld.has(n) ? "held by operator" : tripped?.detail ?? (integHeld.has(n) ? `integration: ${integHeld.get(n)}` : (driftHeld.has(n) ? `drift: ${driftHeld.get(n)} flagged` : budgetStop ? `execution budget: ${budgetStop}` : budgetHeld ? "maintenance share" : "not run"));
|
|
750
774
|
// #4315: a recovery queued but never launched (the batch stopped: breaker, budget, hold; or a resume that did not relaunch a
|
|
751
775
|
// lost run) left the issue without needs-human, since a non-final attempt does not relabel. Relabel it once, with the full reason.
|
|
752
776
|
// #4318: also on a throw out of the loop and on a signal (`onExit`); `cause` overrides the held reason. Runs once per process.
|
|
@@ -759,7 +783,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
759
783
|
queued: q.attempt, cause: why, recoveryMax });
|
|
760
784
|
let res;
|
|
761
785
|
try { res = await relabelNeedsHuman({ issue: n, evidence }); } catch { res = "failed"; }
|
|
762
|
-
if (res !== false && res !== "failed") relabelled.add(n); // a failed label add is retried by a resume
|
|
786
|
+
if (res !== false && res !== "failed") { relabelled.add(n); everRelabelled.add(n); } // a failed label add is retried by a resume
|
|
763
787
|
}
|
|
764
788
|
save();
|
|
765
789
|
})());
|
|
@@ -817,6 +841,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
817
841
|
decided({ issue: e.issue, what: `requeued by ${e.by ?? "operator"} (forgedock orchestrate --requeue); prior outcome ${row.outcome}${list.length ? `; dependents back to pending: ${list.join(", ")}` : ""}`,
|
|
818
842
|
why: e.reason || "operator decision", alternative: "start a second batch" });
|
|
819
843
|
update({ issue: e.issue, outcome: "requeued", detail: `${e.reason || "operator decision"} (was ${row.outcome})` });
|
|
844
|
+
for (const m of moved) if (m !== e.issue) update({ issue: m, outcome: "requeued", detail: `dependency #${e.issue} requeued`, trigger: e.issue }); // #4687
|
|
820
845
|
n++;
|
|
821
846
|
}
|
|
822
847
|
if (fresh.length) save();
|
|
@@ -847,6 +872,22 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
847
872
|
update({ issue: null, outcome: "release", detail: fmt(freed) });
|
|
848
873
|
}
|
|
849
874
|
};
|
|
875
|
+
// #4656: a gate (hold list, settle, ship pre-flight) waiting on an issue the scheduler will not admit: one decision and row per (gate, issue).
|
|
876
|
+
const gateSeen = new Map(), gateReported = new Set();
|
|
877
|
+
const checkGates = () => {
|
|
878
|
+
const gates = [];
|
|
879
|
+
if (opHeld.size) gates.push({ name: "hold", waitsOn: [...opHeld] });
|
|
880
|
+
if (getGates) { try { gates.push(...(getGates() ?? [])); } catch { /* best effort */ } }
|
|
881
|
+
if (!gates.length) return;
|
|
882
|
+
const unadmittable = (n) => { const x = results.get(n); return !(x && (x.outcome === "needs-human" || x.parkedQueued)); };
|
|
883
|
+
for (const d of detectGateDeadlock({ gates, admissible: unadmittable, now: now(), firstSeen: gateSeen })) {
|
|
884
|
+
const key = `${d.gate}#${d.issue}`;
|
|
885
|
+
if (gateReported.has(key)) continue;
|
|
886
|
+
gateReported.add(key);
|
|
887
|
+
decided({ issue: d.issue, what: formatGateDeadlock(d), why: "the gate waits on a parked issue the scheduler will not admit", alternative: "/work-on:remediate the parked issue or release the gate" });
|
|
888
|
+
update({ issue: d.issue, outcome: "GATE_DEADLOCK", detail: formatGateDeadlock(d) });
|
|
889
|
+
}
|
|
890
|
+
};
|
|
850
891
|
const checkCancel = () => {
|
|
851
892
|
if (!getCancelled) return;
|
|
852
893
|
let c = null;
|
|
@@ -878,6 +919,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
878
919
|
// #4595: a skipped row whose blocker (`dependency #B ...`) was finished outside the lane is re-admitted: B is re-checked
|
|
879
920
|
// through `dispatchCheck` (at most every WATCH_EVERY_MS, and once before the batch would end); fail-open.
|
|
880
921
|
let lastWatch = null;
|
|
922
|
+
const settled = new Set(); // #4687: blockers whose close is final (not planned): their dependents stay skipped, no more re-checks
|
|
881
923
|
const readmitted = new Set(); // "n:b": re-admit n on blocker b once; a lane that re-skips on b stays skipped
|
|
882
924
|
const blockerOf = (n) => { const r = results.get(n); const m = r?.outcome === "skipped" ? /^dependency #(\d+) /.exec(r.detail ?? "") : null; return m ? Number(m[1]) : null; };
|
|
883
925
|
const watchable = () => !!dispatchCheck && [...results.keys()].some((n) => blockerOf(n) !== null);
|
|
@@ -887,10 +929,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
887
929
|
if (!final && lastWatch !== null && t0 - lastWatch < WATCH_EVERY_MS) return;
|
|
888
930
|
lastWatch = t0;
|
|
889
931
|
const resolved = new Set();
|
|
890
|
-
const roots = new Set([...results.keys()].filter((n) => !readmitted.has(`${n}:${blockerOf(n)}`)).map(blockerOf).filter((b) => b !== null && blockerOf(b) === null));
|
|
932
|
+
const roots = new Set([...results.keys()].filter((n) => !readmitted.has(`${n}:${blockerOf(n)}`)).map(blockerOf).filter((b) => b !== null && blockerOf(b) === null && !settled.has(b)));
|
|
891
933
|
for (const b of roots) {
|
|
892
934
|
let g = null;
|
|
893
935
|
try { g = (await dispatchCheck(b)) ?? null; } catch { g = null; }
|
|
936
|
+
if (g?.final) { // #4687: closed as not planned: terminal for its dependents
|
|
937
|
+
settled.add(b);
|
|
938
|
+
for (const n of results.keys()) {
|
|
939
|
+
const r = results.get(n);
|
|
940
|
+
if (blockerOf(n) === b && r.waiting) { const { waiting, ...rest } = r; results.set(n, rest); update({ ...rest, trigger: b }); }
|
|
941
|
+
}
|
|
942
|
+
continue;
|
|
943
|
+
}
|
|
894
944
|
if (!g || !SATISFIED.has(g.outcome)) continue;
|
|
895
945
|
resolved.add(b);
|
|
896
946
|
const row = results.get(b);
|
|
@@ -909,6 +959,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
909
959
|
readmitted.add(`${n}:${b}`);
|
|
910
960
|
results.delete(n);
|
|
911
961
|
pending.push(n);
|
|
962
|
+
update({ issue: n, outcome: "readmitted", detail: `dependency #${b} merged or closed`, trigger: b }); // #4687
|
|
912
963
|
decided({ issue: n, what: `re-admitted: dependency #${b} merged or closed after this issue was skipped`, why: "the blocker was finished outside this batch lane",
|
|
913
964
|
alternative: "leave it skipped until the next batch" });
|
|
914
965
|
}
|
|
@@ -930,18 +981,25 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
930
981
|
const n = pending[i];
|
|
931
982
|
if (results.has(n)) { pending.splice(i, 1); continue; }
|
|
932
983
|
const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
|
|
933
|
-
if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}
|
|
984
|
+
if (bad !== undefined) { pending.splice(i, 1); skip(n, `dependency #${bad} ended ${results.get(bad).outcome}`, bad); again = true; }
|
|
934
985
|
}
|
|
935
986
|
}
|
|
936
987
|
const t = now();
|
|
937
988
|
const holdUntil = heldUntil();
|
|
938
989
|
let started = 0;
|
|
939
990
|
driftHeld.clear();
|
|
991
|
+
integHeld.clear();
|
|
940
992
|
opHeldHit.clear();
|
|
941
993
|
refreshHeld();
|
|
994
|
+
checkGates();
|
|
942
995
|
const isReady = (n) => {
|
|
943
996
|
if (opHeld.has(n)) { opHeldHit.add(n); return false; }
|
|
944
997
|
if (!((notBefore.get(n) ?? 0) <= t && edges(n).every((d) => results.has(d) && SATISFIED.has(results.get(d).outcome)))) return false;
|
|
998
|
+
if (integration?.holds) {
|
|
999
|
+
let why = null;
|
|
1000
|
+
try { why = integration.holds(n, { lane: lanes.get(n)?.lane ?? null }); } catch { why = null; }
|
|
1001
|
+
if (why) { integHeld.set(n, why); return false; }
|
|
1002
|
+
}
|
|
945
1003
|
if (!drift?.holds) return true;
|
|
946
1004
|
let pkg = null;
|
|
947
1005
|
try { pkg = drift.holds(n, { body: bodies.get(n) ?? "", lane: lanes.get(n)?.lane ?? null }); } catch { pkg = null; }
|
|
@@ -987,6 +1045,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
987
1045
|
}
|
|
988
1046
|
}
|
|
989
1047
|
if (reg) withdrawExcept(wanted);
|
|
1048
|
+
// #4685: a breaker tripped only by link failures (a router restart) closes itself after a cooldown
|
|
1049
|
+
if (tripped?.transport && now() - tripped.at >= TRANSPORT_COOLDOWN_MS) { tripped = null; errorTimes = []; transportErrors = 0; save(); }
|
|
990
1050
|
if (tripped && !running.size) { await drainAdds(); break; } // breaker: let running lanes finish, keep the rest pending for a resume
|
|
991
1051
|
if (cancelled && !running.size) { await drainAdds(); break; } // #4376: cancel: the rest end held for a resume
|
|
992
1052
|
if (budgetStop && !running.size) { await drainAdds(); break; } // execution budget (stop): the same, the rest end held
|
|
@@ -1025,7 +1085,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1025
1085
|
else if (pending.length && waits.length) await nap(Math.min(wake, getCancelled ? pollMs : Infinity)); // wake is poll-capped by a hold or a live K; #4454: nap wakes on a stop
|
|
1026
1086
|
else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
|
|
1027
1087
|
else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume
|
|
1028
|
-
else if (pending.length && driftHeld.size) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch): pending for a later batch
|
|
1088
|
+
else if (pending.length && (driftHeld.size || integHeld.size)) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch) or a red integration streak (#4667): pending for a later batch
|
|
1029
1089
|
else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
|
|
1030
1090
|
for (const n of pending.splice(0)) skip(n, "dependency cycle");
|
|
1031
1091
|
save();
|
|
@@ -1038,6 +1098,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1038
1098
|
}
|
|
1039
1099
|
if (reg) withdrawExcept(new Set());
|
|
1040
1100
|
await relabelQueued();
|
|
1101
|
+
// #4657: a milestone whose `close_on` state is reached closes here; a failure is a row, never an outcome change.
|
|
1102
|
+
if (lifecycle?.onBatchEnd && lifecycleTitles.size && !halted) {
|
|
1103
|
+
try { await lifecycle.onBatchEnd([...lifecycleTitles]); } catch (e) { onUpdate({ issue: 0, outcome: "lifecycle", detail: `lifecycle check failed: ${String(e?.message ?? e).slice(0, 300)}` }); }
|
|
1104
|
+
}
|
|
1105
|
+
for (const [n, r] of results) if (r?.waiting) { const { waiting, ...rest } = r; results.set(n, rest); } // #4687: the batch ended: skipped for good
|
|
1041
1106
|
save();
|
|
1042
1107
|
const withRequeue = (r) => (r && requeuedInfo.has(r.issue) ? { ...r, requeued: { reason: requeuedInfo.get(r.issue).reason, by: requeuedInfo.get(r.issue).by, prior: requeuedInfo.get(r.issue).prior.map((p) => ({ outcome: p.outcome, detail: p.detail ?? null, pr: p.pr ?? null })) } } : r);
|
|
1043
1108
|
return order.map((n) => withRequeue(results.get(n)) ?? { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: heldDetail(n),
|
|
@@ -1076,9 +1141,12 @@ export function formatOutcomeLists(rows) {
|
|
|
1076
1141
|
for (const r of rows ?? []) {
|
|
1077
1142
|
if (!r || r.recovered) continue;
|
|
1078
1143
|
const human = r.outcome === "needs-human" && classifyNeedsHuman(r) === "human";
|
|
1144
|
+
if (r.parkedQueued && r.outcome === "needs-human") continue; // #4656: listed on the remediation line below, not as unrecovered
|
|
1079
1145
|
if (!human && (r.outcome === "needs-human" || r.outcome === "budget-exceeded" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
|
|
1080
1146
|
}
|
|
1081
1147
|
const esc = (rows ?? []).filter((r) => r?.outcome === "needs-human" && classifyNeedsHuman(r) === "human");
|
|
1148
|
+
const rq = (rows ?? []).filter((r) => r?.outcome === "needs-human" && r.parkedQueued); // #4656: parked past recovery: the single remediation queue, not escalation
|
|
1149
|
+
if (rq.length) out.push(`remediation queue (/work-on:remediate): ${rq.map((r) => `#${r.issue} (parked:${r.code}${r.pr ? ` PR #${r.pr}` : ""})`).join(" ")}`);
|
|
1082
1150
|
const parked = formatParkedLine(rows); // #4389: one line for every parked post-deploy follow-up, never an escalation
|
|
1083
1151
|
if (parked) out.push(parked);
|
|
1084
1152
|
if (esc.length) out.push(["escalations (human decisions, asked once):", ...esc.map((r) => `- #${r.issue} (${r.code}): ${one(r.detail)}`)].join("\n"));
|
|
@@ -1089,7 +1157,7 @@ export function formatTable(rows) {
|
|
|
1089
1157
|
// #4255: sandbox minutes and cost next to the token cost, only when some lane ran in a sandbox.
|
|
1090
1158
|
const sbx = rows.some((r) => r?.sandbox);
|
|
1091
1159
|
const head = ["issue", "outcome", "PR", "cost", ...(sbx ? ["sbx-min", "sbx-$"] : []), "min"];
|
|
1092
|
-
const body = rows.map((r) => [`#${r.issue}`, r.outcome + (["skipped", "held"].includes(r.outcome) && r.detail ? ` (${r.detail})` : r.prefilter?.closed || r.prefilter?.parked ? " (pre-filter)" : ""), r.pr ? `#${r.pr}` : "-", `$${(r.cost_usd ?? 0).toFixed(2)}`,
|
|
1160
|
+
const body = rows.map((r) => [`#${r.issue}`, (r.waiting && /^dependency #(\d+) /.test(r.detail ?? "") ? `waiting on #${/^dependency #(\d+) /.exec(r.detail)[1]}` : r.outcome) + (["skipped", "held"].includes(r.outcome) && r.detail ? ` (${r.detail})` : r.prefilter?.closed || r.prefilter?.parked ? " (pre-filter)" : ""), r.pr ? `#${r.pr}` : "-", `$${(r.cost_usd ?? 0).toFixed(2)}`,
|
|
1093
1161
|
...(sbx ? (r.sandbox ? [(Number(r.sandbox.minutes) || 0).toFixed(1), `$${(Number(r.sandbox.cost_usd) || 0).toFixed(2)}`] : ["-", "-"]) : []), String(r.minutes ?? 0)]);
|
|
1094
1162
|
const w = head.map((h, i) => Math.max(h.length, ...body.map((b) => b[i].length)));
|
|
1095
1163
|
const line = (c) => c.map((x, i) => x.padEnd(w[i])).join(" ").trimEnd();
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
/**
|
|
4
|
+
* Parked post-deploy follow-up sweep (#4689). A follow-up parked by the #4389 gate (`awaiting-promotion`, `awaiting-due`,
|
|
5
|
+
* `awaiting-measurement`) is re-evaluated on the daemon timer, next to `sweepDue`, instead of waiting for the next batch:
|
|
6
|
+
* `prefilterIssue` with `gateOnly` re-runs the same deterministic gate (git ancestry, due date, soak window, precondition,
|
|
7
|
+
* named checks; no model call, no lane). A released issue loses its wait labels and is picked up by the next scheduler pass;
|
|
8
|
+
* one that is still waiting is left as it is.
|
|
9
|
+
*/
|
|
10
|
+
import { prefilterIssue, lazyDefaultBranch, AWAITING_PROMOTION_LABEL, AWAITING_DUE_LABEL } from "./prefilter.mjs";
|
|
11
|
+
import { AWAITING_MEASUREMENT_LABEL } from "./postdeploy-check.mjs";
|
|
12
|
+
|
|
13
|
+
export const PARKED_LABELS = [AWAITING_PROMOTION_LABEL, AWAITING_DUE_LABEL, AWAITING_MEASUREMENT_LABEL];
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Re-evaluate every parked follow-up of the repo. Sequential; an error is an `error` row, never a throw.
|
|
17
|
+
* Outcomes: released | closed | still-parked | skipped | error.
|
|
18
|
+
*/
|
|
19
|
+
export async function sweepParked({ repo, cwd = process.cwd(), io, defaultBranch = null, preconditionCmd = null, minAge = null, dryRun = false, onResult = null } = {}) {
|
|
20
|
+
const found = new Set();
|
|
21
|
+
for (const label of PARKED_LABELS) {
|
|
22
|
+
try {
|
|
23
|
+
const list = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "open", "--label", label, "--limit", "100", "--json", "number"])));
|
|
24
|
+
for (const i of Array.isArray(list) ? list : []) if (Number.isInteger(i?.number)) found.add(i.number);
|
|
25
|
+
} catch { /* one unreadable label list: the others still run */ }
|
|
26
|
+
}
|
|
27
|
+
const resolve = defaultBranch ?? lazyDefaultBranch({ io, repo, cwd });
|
|
28
|
+
const rows = [];
|
|
29
|
+
for (const issue of [...found].sort((a, b) => a - b)) {
|
|
30
|
+
let row;
|
|
31
|
+
try {
|
|
32
|
+
const r = await prefilterIssue({ issue, repo, cwd, io, dryRun, defaultBranch: resolve, preconditionCmd, minAge, gateOnly: true });
|
|
33
|
+
row = { issue, outcome: /^pre-filter error/.test(r.detail) ? "error" : r.closed ? "closed" : r.parked ? "still-parked" : /^released/.test(r.detail) ? "released" : "skipped", detail: r.detail, kind: r.kind ?? null };
|
|
34
|
+
} catch (e) { row = { issue, outcome: "error", detail: String(e?.message ?? e).slice(0, 160) }; }
|
|
35
|
+
rows.push(row);
|
|
36
|
+
try { if (!dryRun) await onResult?.(row); } catch { /* an event sink never fails the sweep */ }
|
|
37
|
+
}
|
|
38
|
+
return rows;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** One line, e.g. `parked: released #4 #5; closed #6`; "" when nothing changed. */
|
|
42
|
+
export function formatParkedSweepSummary(rows) {
|
|
43
|
+
const parts = [["released", "released"], ["closed", "closed"], ["error", "error"]].map(([k, l]) => [l, (rows ?? []).filter((r) => r.outcome === k).map((r) => `#${r.issue}`)]).filter(([, a]) => a.length).map(([l, a]) => `${l} ${a.join(" ")}`);
|
|
44
|
+
return parts.length ? `parked: ${parts.join("; ")}` : "";
|
|
45
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
import { describe, it } from "node:test";
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { sweepParked, formatParkedSweepSummary } from "./parked-sweep.mjs";
|
|
6
|
+
|
|
7
|
+
const MERGE = "a".repeat(40);
|
|
8
|
+
const item = (extra = "") => `- [ ] \`R2: the window shows a row\` — post-deploy: [check:smoke] ${extra} after promotion`;
|
|
9
|
+
const bodyOf = (extra) => `## AC\n\n${item(extra)}\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n`;
|
|
10
|
+
const CHECKS = JSON.stringify({ smoke: "echo ok" });
|
|
11
|
+
|
|
12
|
+
/** Fake repo: `parked` maps issue -> {labels, body}; `promoted` says whether the parent's merge SHA is on origin/main. */
|
|
13
|
+
function world({ parked, promoted, now = Date.parse("2026-11-04T00:00:00Z") }) {
|
|
14
|
+
const w = { edits: [], closed: [], comments: [], commits: [] };
|
|
15
|
+
w.io = {
|
|
16
|
+
now: () => now,
|
|
17
|
+
gh: async (a) => {
|
|
18
|
+
const k = a.slice(0, 2).join(" ");
|
|
19
|
+
if (k === "issue list") { const l = a[a.indexOf("--label") + 1]; return JSON.stringify(Object.entries(parked).filter(([, v]) => v.labels.includes(l)).map(([n]) => ({ number: Number(n) }))); }
|
|
20
|
+
if (k === "issue view") { const n = Number(a[2]); return JSON.stringify({ number: n, state: "OPEN", title: "", comments: [], ...parked[n], labels: parked[n].labels.map((name) => ({ name })) }); }
|
|
21
|
+
if (k === "pr list") return JSON.stringify([{ number: 90, title: "fix (#40)", body: "", mergeCommit: { oid: MERGE } }]);
|
|
22
|
+
if (k === "issue edit") { w.edits.push(a); return ""; }
|
|
23
|
+
if (k === "issue comment") { w.comments.push(a); return ""; }
|
|
24
|
+
if (k === "issue close") { w.closed.push(Number(a[2])); return ""; }
|
|
25
|
+
throw new Error(`unexpected gh ${a.join(" ")}`);
|
|
26
|
+
},
|
|
27
|
+
git: async (a) => {
|
|
28
|
+
if (a[0] === "fetch") return "";
|
|
29
|
+
if (a[0] === "rev-parse") return "deadbeef\n";
|
|
30
|
+
if (a[0] === "show") return CHECKS;
|
|
31
|
+
if (a[0] === "merge-base") { if (promoted) return ""; throw Object.assign(new Error("exit 1"), { code: 1 }); }
|
|
32
|
+
throw new Error(`unexpected git ${a.join(" ")}`);
|
|
33
|
+
},
|
|
34
|
+
spawnCheck: async () => ({ code: 0, stdout: "ok", stderr: "" }),
|
|
35
|
+
};
|
|
36
|
+
return w;
|
|
37
|
+
}
|
|
38
|
+
const sweep = (w, extra = {}) => sweepParked({ repo: "o/r", cwd: "/repo", io: w.io, defaultBranch: "main", ...extra });
|
|
39
|
+
|
|
40
|
+
describe("parked follow-up sweep (#4689)", () => {
|
|
41
|
+
it("one pass after the parent's commit reaches the default branch removes awaiting-promotion and measures; before it nothing changes", async () => {
|
|
42
|
+
const parked = { 41: { labels: ["awaiting-promotion"], body: bodyOf() } };
|
|
43
|
+
const before = world({ parked, promoted: false });
|
|
44
|
+
const rows0 = await sweep(before);
|
|
45
|
+
assert.deepEqual(rows0.map((r) => [r.issue, r.outcome, r.kind]), [[41, "still-parked", "awaiting-promotion"]]);
|
|
46
|
+
assert.deepEqual(before.closed, []);
|
|
47
|
+
const after = world({ parked, promoted: true });
|
|
48
|
+
const rows = await sweep(after, { dryRun: false });
|
|
49
|
+
assert.ok(after.edits.some((a) => a.includes("--remove-label") && a.includes("awaiting-promotion")), "label removed");
|
|
50
|
+
assert.equal(rows[0].issue, 41); assert.notEqual(rows[0].outcome, "still-parked");
|
|
51
|
+
assert.equal(formatParkedSweepSummary([{ issue: 41, outcome: "released" }, { issue: 5, outcome: "error" }]), "parked: released #41; error #5");
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("a due date in the past is released, a future one stays parked", async () => {
|
|
55
|
+
const parked = { 50: { labels: ["awaiting-due", "due:2026-11-03"], body: bodyOf("due:2026-11-03") }, 51: { labels: ["awaiting-due"], body: bodyOf("due:2026-12-01") } };
|
|
56
|
+
const w = world({ parked, promoted: true });
|
|
57
|
+
const rows = await sweep(w);
|
|
58
|
+
assert.equal(rows.find((r) => r.issue === 51).outcome, "still-parked");
|
|
59
|
+
assert.notEqual(rows.find((r) => r.issue === 50).outcome, "still-parked");
|
|
60
|
+
assert.ok(w.edits.some((a) => a[2] === "50" && a.includes("--remove-label") && /awaiting-due,due:2026-11-03/.test(a.join(" "))));
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("an unreadable issue is an error row, never a throw; a dry run changes nothing", async () => {
|
|
64
|
+
const w = world({ parked: { 60: { labels: ["awaiting-promotion"], body: bodyOf() } }, promoted: true });
|
|
65
|
+
const g = w.io.gh; w.io.gh = async (a) => { if (a[1] === "view") throw new Error("boom"); return g(a); };
|
|
66
|
+
assert.equal((await sweep(w))[0].outcome, "error");
|
|
67
|
+
const w2 = world({ parked: { 61: { labels: ["awaiting-promotion"], body: bodyOf() } }, promoted: true });
|
|
68
|
+
await sweep(w2, { dryRun: true });
|
|
69
|
+
assert.deepEqual(w2.edits, []); assert.deepEqual(w2.closed, []);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
@@ -17,6 +17,7 @@ export const CHECKS_FILE = ".forge/post-deploy-checks.json";
|
|
|
17
17
|
export const CHECK_ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
18
18
|
export const AWAITING_MEASUREMENT_LABEL = "awaiting-measurement";
|
|
19
19
|
export const MONITORING_LABEL = "post-deploy-monitoring";
|
|
20
|
+
export const OWNER_ASK_MARKER = "FORGE:OWNER_ASK"; // #4689
|
|
20
21
|
export const DEFAULT_EVERY_MS = 6 * 3600e3;
|
|
21
22
|
const ITEM_RE = /^\s*[-*]\s+\[[ xX]\]\s+(.*)$/;
|
|
22
23
|
const CHECK_TOKEN_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:\s*\[check:\s*([^\]\s]*)\s*\]/;
|
|
@@ -97,6 +98,26 @@ export async function addMonitoringLabel({ io, repo, issue }) {
|
|
|
97
98
|
}
|
|
98
99
|
}
|
|
99
100
|
|
|
101
|
+
/** #4689: tick every unticked post-deploy checklist item of a follow-up body (the evidence is in the FORGE:PROBE comment). */
|
|
102
|
+
export const tickItems = (body) => String(body ?? "").replace(/^(\s*[-*]\s+)\[ \](\s+`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:)/gm, "$1[x]$2");
|
|
103
|
+
|
|
104
|
+
/** #4689: one typed `FORGE:OWNER_ASK` comment for the items with no runnable check, posted once; best effort, nothing in a dry run. */
|
|
105
|
+
async function postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def }) {
|
|
106
|
+
if (dryRun || (iss.comments ?? []).some((c) => isTrusted(c) && String(c?.body ?? "").includes(OWNER_ASK_MARKER))) return;
|
|
107
|
+
const what = ids.length ? missing.map((m) => (m ? `add \`${m}\` to ${CHECKS_FILE} on ${def}` : "name an item's check with `[check:<id>]`")).join("; ") : "name each item's check with `[check:<id>]` and add it to the checks file";
|
|
108
|
+
try { await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", `<!-- ${OWNER_ASK_MARKER} -->\n## Owner ask\nNo lane was started and none will be. Needed from the owner: ${what}, or measure the item by hand.`]); } catch { /* the label still shows it */ }
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** #4689: one engineering issue per failing check (never a lane): the output, the fix and the follow-up. Best effort. */
|
|
112
|
+
async function fileFailureIssues({ bad, iss, repo, io, parent, def, fix }) {
|
|
113
|
+
for (const r of bad) {
|
|
114
|
+
const title = `Post-deploy check ${r.id} fails on ${def}${parent ? ` (fix of #${parent})` : ""}`;
|
|
115
|
+
const body = `Post-deploy check \`${r.id}\` failed (exit ${r.exit ?? "timeout"}) for fix \`${fix.slice(0, 12)}\` on \`${def}\`. Follow-up: #${iss.number}.\n\n${fenced(r.output)}`.slice(0, OUTPUT_TAIL * 2);
|
|
116
|
+
const create = (extra) => io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body, ...extra]);
|
|
117
|
+
try { await create(["--label", "followup:engineering"]); } catch { try { await create([]); } catch { /* the follow-up comment carries the output */ } }
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
100
121
|
/**
|
|
101
122
|
* Measure a ready post-deploy follow-up. Never runs a command that is not in the default branch's checks file.
|
|
102
123
|
* @returns {Promise<{state:"awaiting-measurement"|"pass"|"fail"|"recorded-fail"|"dry-run"|"monitoring", detail:string}>}
|
|
@@ -107,13 +128,19 @@ export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd,
|
|
|
107
128
|
const ids = itemCheckIds(iss.body);
|
|
108
129
|
const checks = await checksAt(io.git, [`origin/${def}`], cwd);
|
|
109
130
|
const missing = ids.filter((id) => !id || !checks[id]);
|
|
110
|
-
if (!ids.length || missing.length)
|
|
131
|
+
if (!ids.length || missing.length) {
|
|
132
|
+
await postOwnerAsk({ iss, repo, io, dryRun, isTrusted, ids, missing, def });
|
|
133
|
+
return { state: "awaiting-measurement", detail: `no runnable check on ${def} for ${ids.length ? `${missing.map((m) => m ?? "an item without a check id").join(", ")}` : "any item"} (${CHECKS_FILE})` };
|
|
134
|
+
}
|
|
111
135
|
const fix = [...shas].sort().join(",");
|
|
112
136
|
const prior = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => probeResultOf(c?.body)).filter(Boolean).reverse().find((p) => p.fix === fix);
|
|
113
137
|
if (prior?.result === "fail") return { state: "recorded-fail", detail: `check failed for ${fix.slice(0, 12)} (FORGE:PROBE); waiting for a new fix of #${parent}` };
|
|
114
138
|
const windowed = ids.every((id) => checks[id].forMs);
|
|
115
139
|
const monitored = (iss.comments ?? []).filter((c) => isTrusted(c)).map((c) => monitorRecord(c?.body)).filter(Boolean).find((p) => p.fix === fix && p.result === "pass" && p.until);
|
|
116
|
-
if (windowed && monitored)
|
|
140
|
+
if (windowed && monitored) {
|
|
141
|
+
await addMonitoringLabel({ io, repo, issue: iss.number }); // retry a label that failed on the first pass; sweepDue selects by it
|
|
142
|
+
return { state: "monitoring", detail: `windowed checks ${[...new Set(ids)].join(", ")} monitored until ${new Date(monitored.until).toISOString()}` };
|
|
143
|
+
}
|
|
117
144
|
if (dryRun) return { state: "dry-run", detail: `checks ${[...new Set(ids)].join(", ")} not run (dry run)` };
|
|
118
145
|
let defaultSha = null;
|
|
119
146
|
try { defaultSha = String(await io.git(["rev-parse", `origin/${def}`], { cwd })).trim() || null; } catch { /* recorded as null */ }
|
|
@@ -129,16 +156,21 @@ export async function measureFollowup({ iss, repo, parent, def, shas = [], cwd,
|
|
|
129
156
|
`Ran after the fix of #${parent} reached \`${def}\`; commands come from \`${CHECKS_FILE}\` on \`${def}\`.${ok ? (win.at ? ` Monitored until \`${win.window_until}\`, re-run on later releases.` : " The follow-up is closed.") : " The parent is reopened."}`,
|
|
130
157
|
...rows.filter((r) => r.result === "fail").flatMap((r) => ["", `**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output)]),
|
|
131
158
|
].join("\n").slice(0, OUTPUT_TAIL * 4);
|
|
132
|
-
await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
|
|
133
159
|
if (!ok) {
|
|
134
160
|
const bad = rows.filter((r) => r.result === "fail");
|
|
135
|
-
|
|
136
|
-
try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) {
|
|
161
|
+
// Reopen before recording: the FORGE:PROBE fail record parks later passes (recorded-fail), so it must only exist once the parent is open.
|
|
162
|
+
try { await io.gh(["issue", "reopen", String(parent), "-R", repo]); } catch (e) {
|
|
163
|
+
return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; reopen of #${parent} failed, no record posted so the next pass retries (${String(e?.message ?? e).slice(0, 120)})` };
|
|
164
|
+
}
|
|
165
|
+
await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
|
|
166
|
+
await fileFailureIssues({ bad, iss, repo, io, parent, def, fix }); // #4689
|
|
137
167
|
try {
|
|
138
168
|
await io.gh(["issue", "comment", String(parent), "-R", repo, "--body", `Post-deploy check ${bad.map((r) => `\`${r.id}\``).join(", ")} failed for the fix on \`${def}\` (follow-up #${iss.number}); reopened.\n\n${bad.map((r) => fenced(r.output)).join("\n\n")}`.slice(0, OUTPUT_TAIL * 4)]);
|
|
139
169
|
} catch { /* the follow-up comment carries the output */ }
|
|
140
|
-
return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened
|
|
170
|
+
return { state: "fail", detail: `check ${bad.map((r) => r.id).join(", ")} failed; #${parent} reopened` };
|
|
141
171
|
}
|
|
172
|
+
await io.gh(["issue", "comment", String(iss.number), "-R", repo, "--body", body]);
|
|
173
|
+
if (tickItems(iss.body) !== iss.body) { try { await io.gh(["issue", "edit", String(iss.number), "-R", repo, "--body", tickItems(iss.body)]); } catch { /* the PROBE record is the evidence */ } } // #4689
|
|
142
174
|
}
|
|
143
175
|
if (windowed) {
|
|
144
176
|
await addMonitoringLabel({ io, repo, issue: iss.number });
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
4
|
import assert from "node:assert/strict";
|
|
5
|
-
import { parseChecks, itemCheckIds, measureFollowup, monitorRecord, CHECK_ID_RE } from "./postdeploy-check.mjs";
|
|
5
|
+
import { parseChecks, itemCheckIds, measureFollowup, monitorRecord, CHECK_ID_RE, tickItems } from "./postdeploy-check.mjs";
|
|
6
6
|
import { postDeployOnly } from "./prefilter.mjs";
|
|
7
7
|
import { validateContract, applyAmendments, precheck, followupItems } from "./conformance.mjs";
|
|
8
8
|
import { PROBE_MARKER } from "./probe.mjs";
|
|
@@ -63,7 +63,7 @@ describe("post-deploy named checks (#4626)", () => {
|
|
|
63
63
|
for (const [file, b] of [[CHECKS, body([item("R1", "[check:ghost] x")])], [null, body([item("R1", "[check:smoke] x")])], [CHECKS, body([item("R1", "x"), item("R2", "[check:smoke] y")])]]) {
|
|
64
64
|
const w = world({ file });
|
|
65
65
|
assert.equal((await run(w, b)).state, "awaiting-measurement");
|
|
66
|
-
assert.deepEqual(w.spawned, []); assert.
|
|
66
|
+
assert.deepEqual(w.spawned, []); assert.ok(w.gh.every((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1])), "only the typed owner ask (#4689)");
|
|
67
67
|
}
|
|
68
68
|
});
|
|
69
69
|
|
|
@@ -142,5 +142,65 @@ describe("windowed post-deploy checks (#4627)", () => {
|
|
|
142
142
|
assert.equal((await run(w, body([item("R1", "[check:soak] x")]))).state, "fail");
|
|
143
143
|
assert.ok(w.gh.some((a) => a[1] === "reopen" && a[2] === "40"));
|
|
144
144
|
});
|
|
145
|
+
it("a failed parent reopen posts no fail record, so the next pass re-runs and retries", async () => {
|
|
146
|
+
const w = world();
|
|
147
|
+
const gh = w.io.gh;
|
|
148
|
+
let failReopen = true;
|
|
149
|
+
w.io.gh = async (a) => { if (a[1] === "reopen" && failReopen) { w.gh.push(a); throw new Error("api down"); } return gh(a); };
|
|
150
|
+
const b = body([item("R1", "[check:bad] x")]);
|
|
151
|
+
const r = await run(w, b);
|
|
152
|
+
assert.equal(r.state, "fail"); assert.match(r.detail, /reopen of #40 failed/);
|
|
153
|
+
assert.ok(!w.gh.some((a) => a[1] === "comment"));
|
|
154
|
+
failReopen = false;
|
|
155
|
+
const r2 = await run(w, b);
|
|
156
|
+
assert.ok(w.gh.filter((a) => a[1] === "reopen").length === 2 && /#40 reopened/.test(r2.detail));
|
|
157
|
+
const order = w.gh.map((a) => a[1] === "comment" ? `c${a[2]}` : a[1]);
|
|
158
|
+
assert.ok(order.indexOf("reopen", 1) < order.indexOf("c41"));
|
|
159
|
+
});
|
|
160
|
+
it("a recorded windowed pass retries the monitoring label", async () => {
|
|
161
|
+
const w = world({ file: WIN });
|
|
162
|
+
const rec = { body: `<!-- ${PROBE_MARKER} -->\n${JSON.stringify({ fix: "abc", result: "pass", at: new Date(NOW).toISOString(), window_until: new Date(NOW + 1e9).toISOString() })}` };
|
|
163
|
+
const b = body([item("R1", "[check:soak] x")]);
|
|
164
|
+
await run(w, b, { iss: { number: 41, body: b, comments: [rec] }, isTrusted: () => true });
|
|
165
|
+
assert.ok(w.gh.some((a) => a[1] === "edit" && a.includes("post-deploy-monitoring")));
|
|
166
|
+
});
|
|
145
167
|
});
|
|
146
168
|
|
|
169
|
+
|
|
170
|
+
describe("verification outcome (#4689)", () => {
|
|
171
|
+
it("a pass ticks the post-deploy items with the evidence in the PROBE record, and writes no code", async () => {
|
|
172
|
+
const w = world();
|
|
173
|
+
const r = await run(w, body([item("R1", "[check:smoke] x")]));
|
|
174
|
+
assert.equal(r.state, "pass");
|
|
175
|
+
const edit = w.gh.find((a) => a[0] === "issue" && a[1] === "edit");
|
|
176
|
+
const text = edit[edit.indexOf("--body") + 1];
|
|
177
|
+
assert.match(text, /- \[x\] `R1: some quote here`/);
|
|
178
|
+
assert.equal(tickItems("- [ ] `R9: other` — defer: code"), "- [ ] `R9: other` — defer: code");
|
|
179
|
+
const probe = w.gh.find((a) => a[1] === "comment" && /FORGE:PROBE/.test(a[a.indexOf("--body") + 1]));
|
|
180
|
+
assert.match(probe[probe.indexOf("--body") + 1], /"id":"smoke","result":"pass"/);
|
|
181
|
+
assert.ok(w.git.every((a) => ["show", "rev-parse"].includes(a[0])), "no git write: " + JSON.stringify(w.git));
|
|
182
|
+
assert.equal(tickItems("no items"), "no items");
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("a fail files one engineering issue per failing check and ticks nothing", async () => {
|
|
186
|
+
const w = world({ file: JSON.stringify({ bad: "echo nope", worse: "echo nope", smoke: "echo ok" }) });
|
|
187
|
+
const r = await run(w, body([item("R1", "[check:bad] x"), item("R2", "[check:worse] x"), item("R3", "[check:smoke] x")]));
|
|
188
|
+
assert.equal(r.state, "fail");
|
|
189
|
+
const created = w.gh.filter((a) => a[0] === "issue" && a[1] === "create");
|
|
190
|
+
assert.equal(created.length, 2);
|
|
191
|
+
assert.ok(created.every((a) => a.includes("followup:engineering")));
|
|
192
|
+
assert.ok(created[0][created[0].indexOf("--title") + 1].includes("bad"));
|
|
193
|
+
assert.ok(!w.gh.some((a) => a[1] === "edit" && a.includes("--body")));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("an item with no runnable check posts one typed owner ask, once", async () => {
|
|
197
|
+
const w = world({ file: null });
|
|
198
|
+
assert.equal((await run(w, body([item("R1", "[check:smoke] x")]))).state, "awaiting-measurement");
|
|
199
|
+
const asks = w.gh.filter((a) => a[1] === "comment" && /FORGE:OWNER_ASK/.test(a[a.indexOf("--body") + 1]));
|
|
200
|
+
assert.equal(asks.length, 1);
|
|
201
|
+
const seen = { number: 41, body: body([item("R1", "[check:smoke] x")]), comments: [{ body: "<!-- FORGE:OWNER_ASK -->\nask", author: { login: "rapiercraft-forgedock" }, authorAssociation: "CONTRIBUTOR" }] };
|
|
202
|
+
const w2 = world({ file: null });
|
|
203
|
+
await run(w2, seen.body, { iss: seen, isTrusted: () => true });
|
|
204
|
+
assert.equal(w2.gh.filter((a) => a[1] === "comment").length, 0);
|
|
205
|
+
});
|
|
206
|
+
});
|
|
@@ -13,6 +13,15 @@ import { deferredFrom, isTrustedIssueComment } from "./prefilter.mjs";
|
|
|
13
13
|
|
|
14
14
|
export const REGRESSION_MARKER = "FORGE:REGRESSION";
|
|
15
15
|
const fenced = (s) => { const f = "`".repeat(Math.max(3, ...[...String(s).matchAll(/`+/g)].map((m) => m[0].length + 1))); return `${f}\n${String(s).trim() || "(no output)"}\n${f}`; };
|
|
16
|
+
/** An existing regression issue (any state) for this follow-up and fix commit, so a retry after a failed finish() reuses it (#4653). */
|
|
17
|
+
export async function findRegression(io, repo, issue, fix) {
|
|
18
|
+
const found = JSON.parse(String(await io.gh(["issue", "list", "-R", repo, "--state", "all", "--search", `"${REGRESSION_MARKER}" "Monitored follow-up: #${issue}" in:body`, "--limit", "30", "--json", "number,body"])));
|
|
19
|
+
const hit = (Array.isArray(found) ? found : []).find((i) => {
|
|
20
|
+
const b = String(i?.body ?? "");
|
|
21
|
+
return b.includes(`<!-- ${REGRESSION_MARKER} -->`) && new RegExp(`Monitored follow-up: #${issue}(?!\\d)`).test(b) && b.includes(`Fix commit: \`${fix}\``);
|
|
22
|
+
});
|
|
23
|
+
return hit ? String(hit.number) : null;
|
|
24
|
+
}
|
|
16
25
|
const short = (s) => String(s ?? "unknown").slice(0, 12);
|
|
17
26
|
|
|
18
27
|
/** One monitored follow-up. Outcomes: skipped | unrunnable | not-due | due (dry run) | pass | window-ended | fail. */
|
|
@@ -53,8 +62,9 @@ export async function monitorOne({ issue, repo, def, defaultSha, cwd, io, now, d
|
|
|
53
62
|
`- Fix commit: \`${last.fix}\``, `- Last passing release (\`${def}\`): \`${last.defaultSha ?? "unknown"}\``, `- Breaking release (\`${def}\`): \`${defaultSha ?? "unknown"}\``,
|
|
54
63
|
`- Monitored follow-up: #${issue}${parent ? `; parent: #${parent}` : ""}`, "", ...bad.flatMap((r) => [`**${r.id}** (exit ${r.exit ?? "timeout"})`, fenced(r.output), ""]),
|
|
55
64
|
].join("\n").slice(0, OUTPUT_TAIL * 4);
|
|
56
|
-
const
|
|
57
|
-
const
|
|
65
|
+
const existing = await findRegression(io, repo, issue, last.fix);
|
|
66
|
+
const created = existing ? "" : String(await io.gh(["issue", "create", "-R", repo, "--title", title, "--body", body]));
|
|
67
|
+
const num = existing ?? /\/issues\/(\d+)/.exec(created)?.[1] ?? null;
|
|
58
68
|
await finish(`<!-- ${PROBE_MARKER} -->\n## Post-deploy check: fail\n${JSON.stringify({ ...rec, regression: num ? Number(num) : null, checks: sum })}\n\nRegression issue ${num ? `#${num}` : "filed"}; monitoring stops.`);
|
|
59
69
|
return row("fail", `regression ${num ? `#${num}` : "filed"}: ${bad.map((r) => r.id).join(", ")} broke at ${short(defaultSha)} (fix ${short(last.fix)})`, { fix: last.fix, regression: num });
|
|
60
70
|
}
|