forgedock 1.31.0 → 1.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/phases.mjs +28 -7
- package/bin/engine-cli.mjs +79 -22
- package/bin/hooks/reread-nudge.mjs +65 -0
- package/bin/lean/batch.mjs +9 -3
- package/bin/lean/batch.test.mjs +75 -2
- package/bin/lean/context.mjs +12 -1
- package/bin/lean/context.test.mjs +18 -1
- package/bin/lean/effects.mjs +41 -4
- package/bin/lean/execution.mjs +4 -2
- package/bin/lean/lean.mjs +83 -18
- package/bin/lean/lean.test.mjs +265 -4
- package/bin/lean/lightlane.mjs +219 -0
- package/bin/lean/lightlane.test.mjs +237 -0
- package/bin/lean/orchestrate.mjs +75 -15
- package/bin/lean/precheck.test.mjs +5 -0
- package/bin/lean/prefilter.mjs +69 -8
- package/bin/lean/prefilter.test.mjs +109 -2
- package/bin/lean/prompts.mjs +3 -0
- package/bin/lean/retry.mjs +15 -1
- package/bin/lean/sched.mjs +16 -1
- package/bin/lean/sched.test.mjs +48 -0
- package/bin/lean/schedd.mjs +50 -14
- package/bin/lean/schedd.test.mjs +153 -20
- package/bin/stats/efficiency.mjs +22 -6
- package/bin/stats/efficiency.test.mjs +35 -2
- package/bin/stats/index.mjs +25 -3
- package/bin/stats/local-source.mjs +59 -5
- package/bin/stats/query.mjs +133 -3
- package/bin/stats/stats.test.mjs +181 -5
- package/docs/CONFIG.md +3 -3
- package/docs/STATS.md +13 -0
- package/docs/eval/corpus/lightlane-v1.json +55 -0
- package/docs/eval/lightlane-bench.md +91 -0
- package/package.json +1 -1
- package/scripts/bench-lightlane.mjs +286 -0
- package/scripts/refresh-bot-token.test.sh +56 -0
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
* lane is refused ends `lane-refused` without a run, and a dependency edge between two lanes goes to `onCrossLane`.
|
|
15
15
|
*
|
|
16
16
|
* Durable batch runner (#4121): with a `store` every transition is persisted, so a restarted dispatcher continues the
|
|
17
|
-
* batch; lanes it launched that are still alive are re-attached through `adopt(issue, rec)` instead of relaunched.
|
|
17
|
+
* batch; lanes it launched that are still alive are re-attached through `adopt(issue, rec, {onLive})` instead of relaunched.
|
|
18
|
+
* A seam calls `onLive()` before it waits on a live lane (#4504); an adopt that resolves first is settled (or requeued on null)
|
|
19
|
+
* before scheduling, so a live lane does not hold up the others.
|
|
18
20
|
* `deferred` is a scheduling fact, not an end: each typed code is requeued with its retry-at (`deferPolicy`), a usage
|
|
19
21
|
* limit holds admissions for that account until its reset, and non-host deferrals stop after `maxDeferrals`.
|
|
20
22
|
* Usage-limit holds (#4190) are keyed by backend identity in `limits` (the host store, shared by every batch on the
|
|
@@ -41,6 +43,13 @@
|
|
|
41
43
|
* `priorities`) ranks it host-wide, and the batch's ready issues are tried in priority order. Ready issues it cannot
|
|
42
44
|
* start yet stay offered in the registry (polled every `pollMs`) and are withdrawn once no longer ready; a settled lane
|
|
43
45
|
* releases its slot, an adopted one re-records it.
|
|
46
|
+
* Per-issue lock (#4481), on with `sched.exclusive` (registry entries ask `exclusive: true`): when another batch holds a live slot for the same repo and issue (`tryAcquire` `conflict`, or
|
|
47
|
+
* `registry.holder` for a lane placed remote), the issue is not started: it is retried after `pollMs` with one `deferred`
|
|
48
|
+
* row naming that batch (no deferral count, no breaker), and admission goes on with the next ready issue.
|
|
49
|
+
*
|
|
50
|
+
* Dispatch check (#4481), off unless `dispatchCheck(issue)` is passed: it runs before EVERY lane start (first and
|
|
51
|
+
* requeued, unlike the once-per-issue pre-filter) and returns null (run) or `{outcome: "merged", pr}` / `{outcome:
|
|
52
|
+
* "invalid"}` for an issue already closed or whose PR merged; the row ends with that outcome and no lane starts. Fail-open.
|
|
44
53
|
*
|
|
45
54
|
* Remote lanes (#4253), off unless `remote: {max, isRemote(issue)}` is passed (execution.mjs): a lane `isRemote` places
|
|
46
55
|
* in a sandbox takes no host registry slot; it starts only while fewer than `max` remote lanes of this batch run (the
|
|
@@ -217,7 +226,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
217
226
|
store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
218
227
|
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
219
228
|
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, stopLane = null,
|
|
220
|
-
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null }) {
|
|
229
|
+
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null }) {
|
|
221
230
|
const saved = store?.load?.() ?? null;
|
|
222
231
|
const order = saved ? [...saved.order] : [];
|
|
223
232
|
const set = new Set(order);
|
|
@@ -380,7 +389,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
380
389
|
return ready.map((n) => ({ n, p: prioOf(n), b: heads.has(n) || starved(n) ? 0 : 1, s: starved(n) ? enqueued.get(n) : 0, w: weight(n), i: idx.get(n) ?? 0 }))
|
|
381
390
|
.sort((a, b) => a.p - b.p || a.b - b.b || a.s - b.s || b.w - a.w || a.i - b.i).map((x) => x.n);
|
|
382
391
|
};
|
|
383
|
-
const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n) });
|
|
392
|
+
const entryOf = (n) => ({ key: keyOf(n), batch: sched?.batch ?? batchId ?? null, repo: sched?.repo ?? null, issue: n, priority: prioOf(n), ...(sched?.exclusive ? { exclusive: true } : {}) });
|
|
384
393
|
const releaseSlot = (n) => { if (reg && slotted.delete(n)) reg.release(keyOf(n)); };
|
|
385
394
|
const offerFresh = (ns) => { const fresh = ns.filter((n) => !offered.has(n)); if (!fresh.length) return; reg.offer(fresh.map(entryOf)); for (const n of fresh) offered.add(n); };
|
|
386
395
|
const withdrawExcept = (keep) => {
|
|
@@ -407,10 +416,28 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
407
416
|
if (budgetHit === null) update({ issue: null, outcome: "budget", detail: `execution budget reached: ${reason}; ${remote?.onBudget === "local" || remote?.fallbackLocal ? "remaining lanes run locally" : "no further remote lanes admitted"}` });
|
|
408
417
|
budgetHit = reason;
|
|
409
418
|
};
|
|
419
|
+
// #4481: another batch runs a lane for `n` now: retry after a poll, one row (and decision) per other batch.
|
|
420
|
+
const busyNoted = new Map(); // issue -> the other batch last reported
|
|
421
|
+
const busy = (n, other, wanted) => {
|
|
422
|
+
const delayMs = Math.max(pollMs, 1);
|
|
423
|
+
notBefore.set(n, now() + delayMs); offered.delete(n); wanted.delete(n);
|
|
424
|
+
if (busyNoted.get(n) !== other) {
|
|
425
|
+
busyNoted.set(n, other); save();
|
|
426
|
+
decided({ issue: n, what: `deferred: batch ${other} runs a lane for #${n}`, why: "one lane per issue on the host (two batches include it)", alternative: "run a second lane on the same branch" });
|
|
427
|
+
update({ issue: n, outcome: "deferred", pr: null, cost_usd: 0, minutes: 0, detail: `#${n} has a live lane in batch ${other}; retried in ${Math.round(delayMs / 1000)}s` });
|
|
428
|
+
}
|
|
429
|
+
return "busy";
|
|
430
|
+
};
|
|
410
431
|
// A host slot for `n` (a lane-refused issue runs nothing and needs none); refused: `n` stays queued in the registry.
|
|
432
|
+
// true: start it; false: no capacity now; "busy" (#4481): another batch runs `n`, try the next issue.
|
|
411
433
|
const grant = (n, wanted) => {
|
|
412
434
|
if (lanes.get(n)?.ok === false) return true;
|
|
413
435
|
let p = placeOf(n);
|
|
436
|
+
if (p?.where === "remote" && reg?.holder) {
|
|
437
|
+
let other = null;
|
|
438
|
+
try { other = reg.holder(entryOf(n)) ?? null; } catch { other = null; }
|
|
439
|
+
if (other !== null) return busy(n, other, wanted);
|
|
440
|
+
}
|
|
414
441
|
if (p?.where === "remote") {
|
|
415
442
|
const adm = admitRemote();
|
|
416
443
|
if (adm.ok) {
|
|
@@ -426,6 +453,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
426
453
|
}
|
|
427
454
|
if (!reg) { if (p) placed.set(n, p); return true; }
|
|
428
455
|
const r = reg.tryAcquire(keyOf(n), entryOf(n), sched.cap);
|
|
456
|
+
if (!r?.ok && r?.conflict != null) return busy(n, r.conflict, wanted);
|
|
429
457
|
if (!r?.ok) { offered.add(n); wanted.add(n); return false; }
|
|
430
458
|
offered.delete(n); slotted.add(n);
|
|
431
459
|
if (p) placed.set(n, p);
|
|
@@ -452,6 +480,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
452
480
|
const pfCost = recovering.has(n) ? 0 : (pf?.cost_usd ?? 0);
|
|
453
481
|
const row = { issue: n, outcome: r?.terminalReason ?? "engine-error", pr: r?.pr ?? null, cost_usd: (r?.cost_usd ?? 0) + pfCost, minutes: Math.round((now() - rec.startedAt) / 6000) / 10, detail: r?.detail, ...(r?.code ? { code: r.code } : {}), ...(pf ? { prefilter: pf } : {}),
|
|
454
482
|
...(r?.waits && typeof r.waits === "object" ? { waits: r.waits } : {}), ...(Number.isInteger(r?.heavyQueueMax) ? { heavyQueueMax: r.heavyQueueMax } : {}),
|
|
483
|
+
...(Number.isFinite(r?.wall_ms) ? { wall_ms: r.wall_ms } : {}), ...(Number.isFinite(r?.model_ms) ? { model_ms: r.model_ms } : {}), // #4451
|
|
455
484
|
...(Number.isFinite(r?.precheckMs) ? { precheck_ms: r.precheckMs } : {}), ...(Number.isInteger(r?.precheckDeferred) && r.precheckDeferred > 0 ? { precheck_deferred: r.precheckDeferred } : {}), // #4331
|
|
456
485
|
...(typeof r?.firstPass === "boolean" ? { first_pass: r.firstPass } : {}), // #4362
|
|
457
486
|
...(r?.turns && typeof r.turns === "object" ? { turns: r.turns } : {}), // #4392
|
|
@@ -604,17 +633,24 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
604
633
|
if (relabelled.delete(n)) save(); // #4315: the recovery lane launches after all; it removes needs-human itself
|
|
605
634
|
const pl = placed.get(n);
|
|
606
635
|
placed.delete(n);
|
|
607
|
-
heads.delete(n); enqueued.delete(n);
|
|
636
|
+
heads.delete(n); enqueued.delete(n); busyNoted.delete(n);
|
|
608
637
|
const rec = { startedAt: now(), ...(lane ? { lane } : {}), ...(remoteRun.has(n) ? { remote: true } : {}), ...(pl ? { placement: pl } : {}) };
|
|
609
638
|
runRecs.set(n, rec); notBefore.delete(n); save();
|
|
610
639
|
track(n, (async () => {
|
|
611
640
|
// Pre-filter once per issue (a requeued issue is not re-checked); fail-open: an error means a normal lane.
|
|
612
641
|
if (prefilter && !checked.has(n)) { let pf = null; try { pf = await prefilter(n, lane); } catch { pf = null; } checked.set(n, pf); save(); }
|
|
613
642
|
const pf = checked.get(n) ?? null;
|
|
643
|
+
// #4481: every start re-checks GitHub: a closed issue or a merged PR ends the row without a lane (fail-open).
|
|
644
|
+
let gone = null;
|
|
645
|
+
if (dispatchCheck && !cancelled && !pf?.closed && !pf?.parked) {
|
|
646
|
+
try { gone = (await dispatchCheck(n)) ?? null; } catch { gone = null; }
|
|
647
|
+
if (!["merged", "invalid"].includes(gone?.outcome)) gone = null;
|
|
648
|
+
}
|
|
614
649
|
if (halted) return; // #4454: stopped before launch; no pid, so a resume queues it again
|
|
615
650
|
let r;
|
|
616
651
|
if (cancelled) r = { terminalReason: "deferred", code: "run.interrupted", cost_usd: 0, detail: "cancelled before launch" }; // #4376
|
|
617
652
|
else if (pf?.closed) r = { terminalReason: "invalid", cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
|
|
653
|
+
else if (gone) r = { terminalReason: gone.outcome, pr: Number.isInteger(gone.pr) ? gone.pr : null, cost_usd: 0, detail: `skipped at dispatch: ${gone.detail ?? gone.outcome}` };
|
|
618
654
|
// #4389: a parked post-deploy follow-up starts no lane; it ends awaiting-promotion|awaiting-precondition, re-checked next batch.
|
|
619
655
|
else if (pf?.parked) r = { terminalReason: pf.kind, cost_usd: 0, detail: `pre-filter: ${pf.detail}` };
|
|
620
656
|
else {
|
|
@@ -632,15 +668,34 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
632
668
|
};
|
|
633
669
|
|
|
634
670
|
// Lanes launched by an earlier dispatcher: re-attach to the live ones, queue the rest again (no result was recorded).
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
671
|
+
// #4504: every adopt starts at once and races its `onLive` signal, so a live lane never holds up the others: one that
|
|
672
|
+
// already ended is settled from its result (or requeued without one) before scheduling; a live one is awaited in the background.
|
|
673
|
+
const requeueLost = (n) => { runRecs.delete(n); if (!pending.includes(n) && !results.has(n)) { pending.unshift(n); enqueued.set(n, now()); } };
|
|
674
|
+
const adopting = [...runRecs].map(([n, rec]) => {
|
|
675
|
+
if (!adopt || !(rec?.pid || rec?.handle)) return { n, rec, live: false, p: Promise.resolve(null) };
|
|
676
|
+
let onLive;
|
|
677
|
+
const isLive = new Promise((r) => { onLive = () => r(true); });
|
|
678
|
+
const p = Promise.resolve().then(() => adopt(n, rec, { onLive })).catch(() => null);
|
|
679
|
+
return { n, rec, p, first: Promise.race([p.then(() => false), isLive]) };
|
|
680
|
+
});
|
|
681
|
+
for (const a of adopting) a.live = a.first ? await a.first : false;
|
|
682
|
+
for (const { n, rec, p, live } of adopting) {
|
|
683
|
+
if (live) {
|
|
684
|
+
if (rec.remote) remoteRun.add(n);
|
|
685
|
+
else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); }
|
|
686
|
+
track(n, p.then((r) => {
|
|
687
|
+
if (r) return settle(n, r, rec, checked.get(n) ?? null);
|
|
688
|
+
if (halted) return; // #4454: kept for the next resume
|
|
689
|
+
running.delete(n); releaseSlot(n); remoteRun.delete(n); requeueLost(n); save(); // ended without a result after all
|
|
690
|
+
}));
|
|
691
|
+
continue;
|
|
642
692
|
}
|
|
643
|
-
|
|
693
|
+
const r = await p;
|
|
694
|
+
if (!r) { requeueLost(n); continue; }
|
|
695
|
+
if (rec.remote) remoteRun.add(n);
|
|
696
|
+
else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); } // released by the settle
|
|
697
|
+
track(n, Promise.resolve(r).then((x) => settle(n, x, rec, checked.get(n) ?? null)));
|
|
698
|
+
await running.get(n); // settled before scheduling
|
|
644
699
|
}
|
|
645
700
|
// A resume re-derives each unfinished issue's lane from its current milestone (#4153): a re-milestone takes effect.
|
|
646
701
|
if (saved && resolveLane) {
|
|
@@ -766,7 +821,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
766
821
|
const n = pickNext(ready, { classes, share: budget.share, whenIdle: budget.whenIdle, started: dispatch.started, maint: dispatch.maint,
|
|
767
822
|
featuresPending: pending.some((p) => !clsOf(classes, p).maintenance) });
|
|
768
823
|
if (n === null) { budgetHeld = ready.length > 0; break; }
|
|
769
|
-
|
|
824
|
+
const g = grant(n, wanted);
|
|
825
|
+
if (g === "busy") continue; // #4481: not ready any more this pass (notBefore), so `ready` drops it
|
|
826
|
+
if (!g) break;
|
|
770
827
|
pending.splice(pending.indexOf(n), 1);
|
|
771
828
|
if (lanes.get(n)?.ok !== false) { dispatch.started++; if (clsOf(classes, n).maintenance) dispatch.maint++; }
|
|
772
829
|
start(n); started++;
|
|
@@ -778,13 +835,16 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
778
835
|
offerFresh(want);
|
|
779
836
|
for (const n of want) wanted.add(n);
|
|
780
837
|
for (const n of ready) {
|
|
781
|
-
if (running.size >= K()
|
|
838
|
+
if (running.size >= K()) break;
|
|
839
|
+
const g = grant(n, wanted);
|
|
840
|
+
if (g === "busy") continue; // #4481
|
|
841
|
+
if (!g) break;
|
|
782
842
|
pending.splice(pending.indexOf(n), 1); start(n); started++;
|
|
783
843
|
}
|
|
784
844
|
} else if (!tripped && !cancelled && !holdUntil) {
|
|
785
845
|
for (const n of rank(pending.filter(isReady))) {
|
|
786
846
|
if (running.size >= K()) break;
|
|
787
|
-
if (grant(n, new Set())) { pending.splice(pending.indexOf(n), 1); start(n); started++; }
|
|
847
|
+
if (grant(n, new Set()) === true) { pending.splice(pending.indexOf(n), 1); start(n); started++; }
|
|
788
848
|
}
|
|
789
849
|
}
|
|
790
850
|
if (reg) withdrawExcept(wanted);
|
|
@@ -118,6 +118,11 @@ describe("formatPrecheckSummary (#4331)", () => {
|
|
|
118
118
|
assert.deepEqual(rows.map((r) => [r.issue, r.precheck_ms, r.precheck_deferred]), [[1, 90000, 1], [2, 30000, undefined], [3, undefined, undefined]]);
|
|
119
119
|
assert.equal(formatPrecheckSummary(rows), "precheck: 2.0m total over 2 lanes, 1 deferred to CI");
|
|
120
120
|
});
|
|
121
|
+
it("#4451: orchestrate rows carry the lane's wall_ms and model_ms, so wall vs model time can be measured", async () => {
|
|
122
|
+
const rows = await orchestrate({ issues: [1, 2], parallel: 1, readBody: async () => "",
|
|
123
|
+
runOne: async (n) => ({ terminalReason: "merged", ...({ 1: { wall_ms: 90000, model_ms: 60000 } })[n] }) });
|
|
124
|
+
assert.deepEqual(rows.map((r) => [r.issue, r.wall_ms, r.model_ms]), [[1, 90000, 60000], [2, undefined, undefined]]);
|
|
125
|
+
});
|
|
121
126
|
it("#4362: orchestrate rows carry the lane's firstPass as first_pass for the batch first-pass line", async () => {
|
|
122
127
|
const rows = await orchestrate({ issues: [1, 2, 3], parallel: 1, readBody: async () => "",
|
|
123
128
|
runOne: async (n) => ({ terminalReason: "merged", ...({ 1: { firstPass: true }, 2: { firstPass: false } })[n] }) });
|
package/bin/lean/prefilter.mjs
CHANGED
|
@@ -12,7 +12,9 @@
|
|
|
12
12
|
* ALREADY-FIXED/INVALID verdict with evidence closes the issue; everything else, and every error, goes to the normal lane.
|
|
13
13
|
* Post-deploy gate (#4389), with `defaultBranch`: a `FORGE:DEFERRED_FROM: #P` follow-up whose checklist items are all
|
|
14
14
|
* post-deploy is parked (`parked: true`, label `awaiting-promotion`, no lane) while a merge SHA of #P is not an ancestor
|
|
15
|
-
* of origin/<default>, or while forge.yaml `post_deploy.precondition_cmd` exits 1.
|
|
15
|
+
* of origin/<default>, or while forge.yaml `post_deploy.precondition_cmd` exits 1. With `post_deploy.min_age` (#4425) it also
|
|
16
|
+
* stays parked (`awaiting-min-age`) until that long (or the longest window an item's why states) after the fix landed on
|
|
17
|
+
* the default branch. Anything unresolvable fails open.
|
|
16
18
|
* All effects are injected (gh, git, runner, transitionLabel, projector?, execCode?) so it is unit-testable.
|
|
17
19
|
*/
|
|
18
20
|
import { execFile } from "node:child_process";
|
|
@@ -196,12 +198,52 @@ function execCode(cmd, args, opts) {
|
|
|
196
198
|
try { execFile(cmd, args, opts, (err) => res(err ? (typeof err.code === "number" ? err.code : -1) : 0)); } catch { res(-1); }
|
|
197
199
|
});
|
|
198
200
|
}
|
|
201
|
+
// #4425: post_deploy.min_age soak window. Deterministic: a strict duration parse, and git for the landing time.
|
|
202
|
+
const UNIT_MS = { m: 60e3, h: 3600e3, d: 86400e3, w: 604800e3 };
|
|
203
|
+
/** forge.yaml `post_deploy.min_age` ("30m", "24h", "7d", "2w"; a bare number is minutes, so "0" is allowed) -> ms, or null. */
|
|
204
|
+
export function parseMinAge(s) {
|
|
205
|
+
if (s == null) return null;
|
|
206
|
+
const m = /^(\d+(?:\.\d+)?)\s*([mhdw]?)$/i.exec(String(s).trim());
|
|
207
|
+
return m ? Math.round(Number(m[1]) * UNIT_MS[(m[2] || "m").toLowerCase()]) : null;
|
|
208
|
+
}
|
|
209
|
+
const WINDOW_RE = /\b(\d+(?:\.\d+)?)\s*(h|hrs?|hours?|d|days?|w|wks?|weeks?)\b/gi;
|
|
210
|
+
const POST_DEPLOY_ITEM_RE = /^`[^`]*`\s+(?:\*\*known-flaw\*\*\s+)?[—-]+\s*post-deploy:(.*)$/;
|
|
211
|
+
/** Per post-deploy checklist item: the largest duration its why states (ms), or null when it states none. */
|
|
212
|
+
export function itemWindowsMs(body) {
|
|
213
|
+
return noComments(String(body ?? ""), " ").split("\n").map((l) => CHECK_RE.exec(l)?.[1]).filter((x) => x != null)
|
|
214
|
+
.map((t) => POST_DEPLOY_ITEM_RE.exec(t)).filter(Boolean).map((m) => {
|
|
215
|
+
const ms = [...m[1].matchAll(WINDOW_RE)].map((w) => Number(w[1]) * UNIT_MS[w[2][0].toLowerCase()]);
|
|
216
|
+
return ms.length ? Math.round(Math.max(...ms)) : null;
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Unix seconds when `sha` reached origin/<def>: the oldest first-parent commit of <def> that descends from it (the merge
|
|
221
|
+
* that brought it in), or the SHA's own commit time when it is on the mainline itself. Null on any git error.
|
|
222
|
+
*/
|
|
223
|
+
export async function landedAt({ sha, def, io, cwd }) {
|
|
224
|
+
try {
|
|
225
|
+
// --first-parent with --ancestry-path drops merges that brought the SHA in through a second parent (git 2.55): two walks
|
|
226
|
+
const range = `${sha}..origin/${def}`;
|
|
227
|
+
const below = new Set(String(await io.git(["rev-list", "--ancestry-path", range], { cwd })).trim().split(/\s+/).filter(Boolean));
|
|
228
|
+
const chain = String(await io.git(["log", "--first-parent", "--format=%H %P %ct", range], { cwd })).trim().split("\n").map((l) => l.trim().split(/\s+/));
|
|
229
|
+
const oldest = chain.filter((c) => below.has(c[0])).at(-1);
|
|
230
|
+
let own = !oldest;
|
|
231
|
+
if (!own) {
|
|
232
|
+
const full = /^[0-9a-f]{40}$/i.test(sha) ? sha : String(await io.git(["rev-parse", `${sha}^{commit}`], { cwd })).trim();
|
|
233
|
+
own = oldest[1]?.toLowerCase() === full.toLowerCase();
|
|
234
|
+
}
|
|
235
|
+
const ct = own ? String(await io.git(["log", "-1", "--format=%ct", sha], { cwd })).trim() : oldest.at(-1);
|
|
236
|
+
return /^\d+$/.test(ct) ? Number(ct) : null;
|
|
237
|
+
} catch { return null; }
|
|
238
|
+
}
|
|
239
|
+
const fmtAge = (ms) => (ms % UNIT_MS.d === 0 && ms ? `${ms / UNIT_MS.d}d` : ms % UNIT_MS.h === 0 && ms ? `${ms / UNIT_MS.h}h` : `${Math.round(ms / UNIT_MS.m)}m`);
|
|
199
240
|
/**
|
|
200
241
|
* #4389: never throws, no model call. `defaultBranch` is a name or an (async) function resolving it (and fetching it).
|
|
201
|
-
*
|
|
242
|
+
* #4425: `minAge` (ms, opt-in) parks `awaiting-min-age` until max(item window ?? minAge) after the fix landed on <default>.
|
|
243
|
+
* @returns {Promise<{state:"not-applicable"|"unresolved"|"awaiting-promotion"|"awaiting-min-age"|"awaiting-precondition"|"ready", parent:?number, shas:string[], default?:string, eligibleAt?:string, detail:string}>}
|
|
202
244
|
* Fail-open: no merge SHA, a git answer other than yes/no, an unknown default branch or a hook error give "unresolved".
|
|
203
245
|
*/
|
|
204
|
-
export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd = null }) {
|
|
246
|
+
export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd = null, minAge = null, now = null }) {
|
|
205
247
|
const parent = deferredFrom(iss?.body);
|
|
206
248
|
const res = (state, detail, extra = {}) => ({ state, parent, shas: [], detail, ...extra });
|
|
207
249
|
if (!parent || !postDeployOnly(iss.body)) return res("not-applicable", "");
|
|
@@ -218,6 +260,20 @@ export async function postDeployGate({ iss, repo, lane, defaultBranch, cwd, io,
|
|
|
218
260
|
if (!on) off.push(sha);
|
|
219
261
|
}
|
|
220
262
|
if (off.length) return res("awaiting-promotion", `fix ${off.map((s) => s.slice(0, 12)).join(", ")} of #${parent} is not on ${def}${lane ? ` (lane ${lane})` : ""}`, { shas, default: def });
|
|
263
|
+
if (minAge != null) {
|
|
264
|
+
const required = Math.max(0, ...itemWindowsMs(iss.body).map((w) => w ?? minAge));
|
|
265
|
+
let landed = 0;
|
|
266
|
+
for (const sha of shas) {
|
|
267
|
+
const t = await landedAt({ sha, def, io, cwd });
|
|
268
|
+
if (t == null) return res("unresolved", `landing time of ${sha.slice(0, 12)} on ${def} unknown`, { shas, default: def });
|
|
269
|
+
landed = Math.max(landed, t);
|
|
270
|
+
}
|
|
271
|
+
const at = now ?? io.now?.() ?? Date.now(), until = landed * 1000 + required;
|
|
272
|
+
if (at < until) {
|
|
273
|
+
const eligibleAt = new Date(until).toISOString();
|
|
274
|
+
return res("awaiting-min-age", `fix of #${parent} on ${def} since ${new Date(landed * 1000).toISOString()}; min age ${fmtAge(required)} until ${eligibleAt}`, { shas, default: def, eligibleAt });
|
|
275
|
+
}
|
|
276
|
+
}
|
|
221
277
|
if (preconditionCmd) {
|
|
222
278
|
const code = await (io.execCode ?? execCode)("sh", ["-c", preconditionCmd, "precondition", String(iss.number)],
|
|
223
279
|
{ cwd, timeout: 60000, env: { ...process.env, FORGE_ISSUE: String(iss.number) } });
|
|
@@ -252,12 +308,12 @@ async function markParked({ issue, repo, io, had, gate }) {
|
|
|
252
308
|
} catch { /* the comment and the report still show it */ }
|
|
253
309
|
}
|
|
254
310
|
if (had) return;
|
|
255
|
-
const what = gate.state === "awaiting-promotion" ? "awaiting promotion" : "awaiting its post-deploy precondition";
|
|
311
|
+
const what = gate.state === "awaiting-promotion" ? "awaiting promotion" : gate.state === "awaiting-min-age" ? "awaiting its post-deploy soak window" : "awaiting its post-deploy precondition";
|
|
256
312
|
const body = `<!-- ${PRECONDITION_MARKER} -->\n## Parked: ${what}\n${sanitizeMd(gate.detail)}.\n\nEvery item here is a post-deploy check, so no lane was started. The next orchestrate or triage run checks again and launches this issue once the precondition holds.`;
|
|
257
313
|
try { await io.gh(["issue", "comment", String(issue), "-R", repo, "--body", capComment(body)]); } catch { /* the label still shows it */ }
|
|
258
314
|
}
|
|
259
315
|
|
|
260
|
-
/** #4389 batch-report lines: one `awaiting promotion:` line (and one `awaiting precondition:` line) for the parked rows, or "". */
|
|
316
|
+
/** #4389 batch-report lines: one `awaiting promotion:` line (and one `awaiting min age:` / `awaiting precondition:` line) for the parked rows, or "". */
|
|
261
317
|
export function formatParkedLine(rows) {
|
|
262
318
|
const parked = (rows ?? []).filter((r) => r?.prefilter?.parked);
|
|
263
319
|
const uniq = (xs) => [...new Set(xs.filter(Boolean))];
|
|
@@ -267,6 +323,11 @@ export function formatParkedLine(rows) {
|
|
|
267
323
|
const lanes = uniq(promo.map((r) => r.prefilter.lane)), defs = uniq(promo.map((r) => r.prefilter.default));
|
|
268
324
|
out.push(`awaiting promotion: ${promo.map((r) => `#${r.issue}`).join(" ")} (fixes on ${lanes.join(", ") || "the lane"}, not on ${defs.join(", ") || "the default branch"})`);
|
|
269
325
|
}
|
|
326
|
+
const soak = parked.filter((r) => r.prefilter.kind === "awaiting-min-age"); // #4425
|
|
327
|
+
if (soak.length) {
|
|
328
|
+
const defs = uniq(soak.map((r) => r.prefilter.default));
|
|
329
|
+
out.push(`awaiting min age: ${soak.map((r) => `#${r.issue}`).join(" ")} (fix on ${defs.join(", ") || "the default branch"}, soak window not elapsed)`);
|
|
330
|
+
}
|
|
270
331
|
const pre = parked.filter((r) => r.prefilter.kind === "awaiting-precondition");
|
|
271
332
|
if (pre.length) out.push(`awaiting precondition: ${pre.map((r) => `#${r.issue}`).join(" ")} (post_deploy.precondition_cmd not ready)`);
|
|
272
333
|
return out.join("\n");
|
|
@@ -276,7 +337,7 @@ export function formatParkedLine(rows) {
|
|
|
276
337
|
* Pre-filter one issue. Never throws.
|
|
277
338
|
* @returns {Promise<{issue:number, closed:boolean, kind:?string, candidate:boolean, verdict:?string, cost_usd:number, detail:string}>}
|
|
278
339
|
*/
|
|
279
|
-
export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAULT_PREFILTER_MODEL, dryRun = false, agentId = null, maxMinutes = 5, selfLogins = null, defaultBranch = null, preconditionCmd = null }) {
|
|
340
|
+
export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAULT_PREFILTER_MODEL, dryRun = false, agentId = null, maxMinutes = 5, selfLogins = null, defaultBranch = null, preconditionCmd = null, minAge = null }) {
|
|
280
341
|
const out = { issue, closed: false, kind: null, candidate: false, verdict: null, cost_usd: 0, detail: "" };
|
|
281
342
|
try {
|
|
282
343
|
const iss = JSON.parse(String(await io.gh(["issue", "view", String(issue), "-R", repo, "--json", "number,title,body,state,labels,comments"])));
|
|
@@ -292,9 +353,9 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
|
|
|
292
353
|
if (st?.lease && st.lease.until > Date.now() && st.lease.by !== agentId) return { ...out, detail: `leased by ${st.lease.by}` };
|
|
293
354
|
}
|
|
294
355
|
if (gated) {
|
|
295
|
-
const gate = await postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd });
|
|
356
|
+
const gate = await postDeployGate({ iss, repo, lane, defaultBranch, cwd, io, preconditionCmd, minAge });
|
|
296
357
|
const had = names.includes(AWAITING_PROMOTION_LABEL);
|
|
297
|
-
if (
|
|
358
|
+
if (["awaiting-promotion", "awaiting-min-age", "awaiting-precondition"].includes(gate.state)) {
|
|
298
359
|
if (!dryRun) await markParked({ issue, repo, io, had, gate });
|
|
299
360
|
return { ...out, parked: true, kind: gate.state, parent: gate.parent, lane, default: gate.default, detail: gate.detail };
|
|
300
361
|
}
|
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
4
|
import assert from "node:assert/strict";
|
|
5
|
-
import { prefilterIssue, flakyReason, makePrefilter, postDeployGate, postDeployOnly, deferredFrom, formatParkedLine, lazyDefaultBranch, ownLogins, resolveOwnLogins, formatPrefilterSummary, claimsFix, ciFailureRun, namedPaths, isTrustedIssueComment, DEFAULT_PREFILTER_MODEL } from "./prefilter.mjs";
|
|
5
|
+
import { prefilterIssue, flakyReason, makePrefilter, postDeployGate, postDeployOnly, deferredFrom, formatParkedLine, lazyDefaultBranch, ownLogins, resolveOwnLogins, formatPrefilterSummary, claimsFix, ciFailureRun, namedPaths, isTrustedIssueComment, DEFAULT_PREFILTER_MODEL, parseMinAge, itemWindowsMs, landedAt } from "./prefilter.mjs";
|
|
6
|
+
import { writeFileSync } from "node:fs";
|
|
7
|
+
import { tmpDir } from "../tests/_tmp.mjs";
|
|
8
|
+
import { execFileSync } from "node:child_process";
|
|
9
|
+
import { join } from "node:path";
|
|
6
10
|
import { parsePrefilterVerdict, prefilterPrompt } from "./prompts.mjs";
|
|
7
11
|
import { orchestrate, formatTable, formatOutcomeLists } from "./orchestrate.mjs";
|
|
8
|
-
import { triageFromCli } from "../engine-cli.mjs";
|
|
12
|
+
import { triageFromCli, readPostDeployMinAge } from "../engine-cli.mjs";
|
|
9
13
|
|
|
10
14
|
const fence = (o) => "```json\n" + JSON.stringify(o) + "\n```";
|
|
11
15
|
|
|
@@ -477,6 +481,109 @@ describe("post-deploy precondition gate (#4389)", () => {
|
|
|
477
481
|
});
|
|
478
482
|
});
|
|
479
483
|
|
|
484
|
+
describe("post_deploy.min_age soak window (#4425)", () => {
|
|
485
|
+
const MERGE = "a".repeat(40), PROMO = "b".repeat(40), MAIN = "c".repeat(40);
|
|
486
|
+
const H = 3600, LANDED = 1_800_000_000; // unix seconds of the promotion merge
|
|
487
|
+
const item = (id, why) => `- [ ] \`${id}: a health row exists\` — post-deploy: ${why}`;
|
|
488
|
+
const followup = (items) => `## Acceptance Criteria\n\n${items.join("\n")}\n\n<!-- FORGE:DEFERRED_FROM: #40 -->\n`;
|
|
489
|
+
const parentPr = [{ number: 90, title: "fix(x): health row (#40)", body: "", mergeCommit: { oid: MERGE } }];
|
|
490
|
+
// the parent fix reached main through promotion merge PROMO (first parent MAIN) at LANDED
|
|
491
|
+
const soakWorld = (o = {}) => {
|
|
492
|
+
const w = world({ mergedPrs: parentPr, ancestry: { [MERGE]: 0 }, ...o });
|
|
493
|
+
const git = w.io.git;
|
|
494
|
+
w.io.git = async (a, opts) => {
|
|
495
|
+
if (a[0] === "log" && a[1] === "--first-parent") { w.gitCalls.push(a); return o.walk ?? `${"d".repeat(40)} ${PROMO} ${LANDED + 9 * H}\n${PROMO} ${MAIN} ${MERGE} ${LANDED}\n${MAIN} ${"e".repeat(40)} ${LANDED - H}\n`; }
|
|
496
|
+
if (a[0] === "rev-list") { w.gitCalls.push(a); return o.below ?? `${"d".repeat(40)}\n${PROMO}\n`; }
|
|
497
|
+
if (a[0] === "log" && a[1] === "-1") { w.gitCalls.push(a); return String(LANDED - 5 * H); }
|
|
498
|
+
return git(a, opts);
|
|
499
|
+
};
|
|
500
|
+
return w;
|
|
501
|
+
};
|
|
502
|
+
const gate = (w, body, extra = {}) => postDeployGate({ iss: { number: 41, body }, repo: "o/r", lane: "staging", defaultBranch: "main", cwd: "/repo", io: w.io, ...extra });
|
|
503
|
+
|
|
504
|
+
it("parses min_age and the windows each post-deploy item's why states (largest wins, none -> null)", () => {
|
|
505
|
+
assert.equal(parseMinAge("24h"), 24 * H * 1000); assert.equal(parseMinAge("7d"), 7 * 24 * H * 1000); assert.equal(parseMinAge("2w"), 14 * 24 * H * 1000);
|
|
506
|
+
assert.equal(parseMinAge("30m"), 30 * 60e3); assert.equal(parseMinAge("0"), 0); assert.equal(parseMinAge(90), 90 * 60e3);
|
|
507
|
+
assert.equal(parseMinAge("soon"), null); assert.equal(parseMinAge(null), null);
|
|
508
|
+
assert.deepEqual(itemWindowsMs(followup([item("R1", "after 24h of traffic, then 7 days of retention"), item("R2", "only in prod")])), [7 * 24 * H * 1000, null]);
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
it("landedAt: the promotion merge time, or the SHA's own time when it is on the mainline", async () => {
|
|
512
|
+
const w = soakWorld();
|
|
513
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: w.io, cwd: "/repo" }), LANDED);
|
|
514
|
+
assert.deepEqual(w.gitCalls.slice(0, 2), [["rev-list", "--ancestry-path", `${MERGE}..origin/main`], ["log", "--first-parent", "--format=%H %P %ct", `${MERGE}..origin/main`]]);
|
|
515
|
+
const own = soakWorld({ walk: `${PROMO} ${MERGE} ${LANDED}\n`, below: `${PROMO}\n` });
|
|
516
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: own.io, cwd: "/repo" }), LANDED - 5 * H);
|
|
517
|
+
const tip = soakWorld({ walk: "", below: "" });
|
|
518
|
+
assert.equal(await landedAt({ sha: MERGE, def: "main", io: tip.io, cwd: "/repo" }), LANDED - 5 * H);
|
|
519
|
+
});
|
|
520
|
+
|
|
521
|
+
it("landedAt on a real repo: the promotion merge that brought a staging fix in through its second parent", async () => {
|
|
522
|
+
const d = tmpDir("landed-");
|
|
523
|
+
const run = (args, t) => execFileSync("git", args, { cwd: d, encoding: "utf8", env: { ...process.env, GIT_AUTHOR_DATE: `@${t} +0000`, GIT_COMMITTER_DATE: `@${t} +0000`,
|
|
524
|
+
GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" } }).trim();
|
|
525
|
+
run(["init", "-q", "-b", "main"], 1);
|
|
526
|
+
run(["commit", "-q", "--allow-empty", "-m", "base"], LANDED - 10 * H);
|
|
527
|
+
run(["checkout", "-q", "-b", "staging"], 1);
|
|
528
|
+
run(["commit", "-q", "--allow-empty", "-m", "fix"], LANDED - 8 * H);
|
|
529
|
+
const fix = run(["rev-parse", "HEAD"], 1);
|
|
530
|
+
run(["checkout", "-q", "main"], 1);
|
|
531
|
+
run(["commit", "-q", "--allow-empty", "-m", "hotfix on main"], LANDED - 2 * H);
|
|
532
|
+
run(["merge", "-q", "--no-ff", "-m", "promote", "staging"], LANDED);
|
|
533
|
+
run(["commit", "-q", "--allow-empty", "-m", "later"], LANDED + 5 * H);
|
|
534
|
+
run(["update-ref", "refs/remotes/origin/main", "HEAD"], 1);
|
|
535
|
+
const io = { git: async (a, o) => run(a, 1) + "\n" };
|
|
536
|
+
assert.equal(await landedAt({ sha: fix, def: "main", io, cwd: d }), LANDED);
|
|
537
|
+
assert.equal(await landedAt({ sha: fix.slice(0, 12), def: "main", io, cwd: d }), LANDED);
|
|
538
|
+
assert.equal(await landedAt({ sha: run(["rev-parse", "HEAD~1^1"], 1), def: "main", io, cwd: d }), LANDED - 2 * H); // on the mainline itself
|
|
539
|
+
assert.equal(await landedAt({ sha: "f".repeat(40), def: "main", io, cwd: d }), null);
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
it("parks awaiting-min-age until max(item window ?? min_age) after landing, then is ready", async () => {
|
|
543
|
+
const body = followup([item("R1", "wait 7d of production data"), item("R2", "only in prod")]);
|
|
544
|
+
const g = await gate(soakWorld(), body, { minAge: parseMinAge("24h"), now: (LANDED + 48 * H) * 1000 });
|
|
545
|
+
assert.equal(g.state, "awaiting-min-age");
|
|
546
|
+
assert.equal(g.eligibleAt, new Date((LANDED + 7 * 24 * H) * 1000).toISOString());
|
|
547
|
+
assert.match(g.detail, /^fix of #40 on main since .*; min age 7d until /);
|
|
548
|
+
// no stated window: the min_age floor applies
|
|
549
|
+
const short = followup([item("R2", "only in prod")]);
|
|
550
|
+
assert.equal((await gate(soakWorld(), short, { minAge: parseMinAge("24h"), now: (LANDED + 23 * H) * 1000 })).state, "awaiting-min-age");
|
|
551
|
+
assert.equal((await gate(soakWorld(), short, { minAge: parseMinAge("24h"), now: (LANDED + 25 * H) * 1000 })).state, "ready");
|
|
552
|
+
assert.equal((await gate(soakWorld(), body, { minAge: 0, now: (LANDED + 8 * 24 * H) * 1000 })).state, "ready");
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
it("unset min_age changes nothing (no landing-time git calls); an unknown landing time fails open", async () => {
|
|
556
|
+
const w = soakWorld();
|
|
557
|
+
assert.equal((await gate(w, followup([item("R1", "wait 7d")]))).state, "ready");
|
|
558
|
+
assert.ok(!w.gitCalls.some((a) => a[0] === "log" || a[0] === "rev-list"));
|
|
559
|
+
const broken = soakWorld();
|
|
560
|
+
const git = broken.io.git;
|
|
561
|
+
broken.io.git = async (a, o) => { if (a[0] === "rev-list") throw Object.assign(new Error("bad"), { code: 128 }); return git(a, o); };
|
|
562
|
+
const g = await gate(broken, followup([item("R1", "wait 7d")]), { minAge: 0 });
|
|
563
|
+
assert.equal(g.state, "unresolved"); assert.match(g.detail, /landing time of aaaaaaaaaaaa on main unknown/);
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
it("prefilterIssue parks it with the awaiting-promotion label, one comment, and an 'awaiting min age:' report line", async () => {
|
|
567
|
+
const w = soakWorld({ issues: { 41: { body: followup([item("R1", "wait 7d")]) } } });
|
|
568
|
+
w.io.now = () => (LANDED + H) * 1000;
|
|
569
|
+
const r = await pf(w, 41, { defaultBranch: "main", minAge: parseMinAge("24h") });
|
|
570
|
+
assert.equal(r.parked, true); assert.equal(r.kind, "awaiting-min-age"); assert.equal(r.cost_usd, 0);
|
|
571
|
+
assert.deepEqual(w.edits, [[41, "--add-label", "awaiting-promotion"]]);
|
|
572
|
+
assert.equal(w.comments.length, 1); assert.match(w.comments[0][1], /Parked: awaiting its post-deploy soak window/);
|
|
573
|
+
assert.equal(formatParkedLine([{ issue: 41, prefilter: r }, { issue: 42, prefilter: { parked: true, kind: "awaiting-promotion", lane: "staging", default: "main" } }]),
|
|
574
|
+
"awaiting promotion: #42 (fixes on staging, not on main)\nawaiting min age: #41 (fix on main, soak window not elapsed)");
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
it("readPostDeployMinAge reads forge.yaml post_deploy.min_age; absent or invalid -> null", async () => {
|
|
578
|
+
const d = tmpDir("minage-");
|
|
579
|
+
assert.equal(await readPostDeployMinAge(d), null);
|
|
580
|
+
writeFileSync(join(d, "forge.yaml"), "post_deploy:\n min_age: \"24h\"\n");
|
|
581
|
+
assert.equal(await readPostDeployMinAge(d), 24 * H * 1000);
|
|
582
|
+
writeFileSync(join(d, "forge.yaml"), "post_deploy:\n min_age: later\n");
|
|
583
|
+
assert.equal(await readPostDeployMinAge(d), null);
|
|
584
|
+
});
|
|
585
|
+
});
|
|
586
|
+
|
|
480
587
|
describe("#4467: isTrustedIssueComment", () => {
|
|
481
588
|
it("trusts the App in gh issue view and REST shapes and trusted associations; not other contributors", () => {
|
|
482
589
|
assert.equal(isTrustedIssueComment({ author: { login: "rapiercraft-forgedock" }, authorAssociation: "CONTRIBUTOR" }), true);
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -24,6 +24,9 @@ export const FORGEDOCK_CLI = fileURLToPath(new URL("../forgedock.mjs", import.me
|
|
|
24
24
|
const HOST_RULES = `Host rules (many lanes share this machine): run only targeted tests (specific files or -k filters), never the full suite. Never create extra checkouts, copies or base clones of the repo; compare with git show or git diff origin/<base>. Never install browsers. Keep scratch files under $TMPDIR. Cap verification workers at $FORGE_HOST_PARALLELISM when set: turbo --concurrency, vitest/jest --maxWorkers, playwright --workers, node --test --test-concurrency (eslint via turbo). For any typecheck, lint or test you run, first run \`node ${CHECKS_CLI} --base origin/<base>\` after your edits and run the scoped commands it prints instead of the full tsc/eslint/test scripts (it widens itself when impact is unknown: lockfile, tsconfig, CI config); once they pass, rerun it with --publish to share the incremental caches. Lint changed packages only, never dependents (turbo ...[base], nx affected). node_modules/.venv may be hardlinked read-only from a shared cache: after changing a lockfile, or when an install fails with EACCES, delete that dir first (rm -rf is safe), then install; never chmod or edit files inside it.
|
|
25
25
|
Quiet output (tool results are re-read every later turn): run tests quietly (-q or a dot/summary reporter); on failure read \`tail -40\` plus the failing test ids (grep FAILED / not ok / FAIL), never the whole log. Send output over 8 KB to a $TMPDIR file; tail or grep it. Issue independent read-only calls (reads, greps, git log/blame, gh) in the same turn or one shell call. Read each file's ranges once (sed -n a,bp or Read offset/limit, never a whole large file) and keep notes; don't re-read it.`;
|
|
26
26
|
|
|
27
|
+
/** #4450: the host-rules block of the builder prompts (toolchain, sparse and repo-facts lines included), reused by the light-lane prompt. */
|
|
28
|
+
export const hostRules = ({ toolchain = null, sparse = null, facts = null } = {}) => `${HOST_RULES}${toolchainRule(toolchain)}${sparseRule(sparse)}${factsRule(facts)}`;
|
|
29
|
+
|
|
27
30
|
/** Verify-by-CI (#4167): in ci/hybrid mode the engine runs the repo's CI on GitHub runners; the host runs only light checks. */
|
|
28
31
|
const LIGHT = "run only formatters and linters on the files you changed";
|
|
29
32
|
const VERIFY_RUN = { ci: `${LIGHT} (no type-checks, test suites or test containers)`, hybrid: "run the targeted validate commands" };
|
package/bin/lean/retry.mjs
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
export const PUSH_TRANSIENT_RE = /remote unpack failed|index-pack failed|RPC failed|unexpected disconnect|early EOF|the remote end hung up|HTTP\/?\s?5\d\d|returned error: 5\d\d|\b50[0234]\b|internal server error|bad gateway|service unavailable|gateway time-?out|connection (?:reset|timed out|refused)|timed out|could not resolve host|broken pipe|ETIMEDOUT|ECONNRESET|temporarily unavailable|\[remote rejected\][^\n]*\(failed\)|unable to write file|unable to create temporary object|unpacker error|failed to write object/i;
|
|
6
6
|
export const PUSH_PERMANENT_RE = /protected branch|GH006|GH013|repository rule violations|permission (?:denied|to .* denied)|denied to|\b403\b|authentication failed|not allowed|hook declined|pre-receive hook declined|\[rejected\]|non-fast-forward|fetch first/i;
|
|
7
7
|
export const GH_TRANSIENT_RE = /HTTP 5\d\d|\b50[0234]\b|secondary rate limit|rate limit exceeded|abuse detection|something went wrong|internal server error|bad gateway|service unavailable|ETIMEDOUT|ECONNRESET|connection reset|timed out|\bEOF\b|temporarily unavailable/i;
|
|
8
|
-
|
|
8
|
+
/** #4507: gh failing to parse its own input: a torn HTTP cache entry (JSON) or a half-written config (YAML). */
|
|
9
|
+
export const GH_PARSE_RE = /invalid character .{0,40}looking for|unexpected end of JSON input|cannot unmarshal|yaml: line \d+|yaml: unmarshal/i;
|
|
10
|
+
export const MERGE_TRANSIENT_RE = new RegExp(`${GH_TRANSIENT_RE.source}|${GH_PARSE_RE.source}|base branch was modified|head branch was modified|mergeability.*unknown|try again`, "i");
|
|
9
11
|
export const MERGE_CONFLICT_RE = /not mergeable|merge conflict|conflict/i;
|
|
10
12
|
|
|
11
13
|
export const errText = (e) => `${e?.stderr ?? ""}\n${e?.message ?? ""}\n${e?.stdout ?? ""}`;
|
|
@@ -29,3 +31,15 @@ export async function retrying(fn, { transient, permanent = null, attempts = 4,
|
|
|
29
31
|
last.transientExhausted = true;
|
|
30
32
|
throw last;
|
|
31
33
|
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* #4507: run `call(env)` with the lane's gh cache env; on a gh parse error (corrupt cache) reset the cache and run it
|
|
37
|
+
* once more. Without a cache it still retries once. Any other error is rethrown unchanged.
|
|
38
|
+
*/
|
|
39
|
+
export async function ghCacheRetry(call, cache) {
|
|
40
|
+
try { return await call(cache?.env?.()); } catch (e) {
|
|
41
|
+
if (!GH_PARSE_RE.test(errText(e))) throw e;
|
|
42
|
+
try { await cache?.reset?.(); } catch { /* retry on the old dir */ }
|
|
43
|
+
return await call(cache?.env?.());
|
|
44
|
+
}
|
|
45
|
+
}
|
package/bin/lean/sched.mjs
CHANGED
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
* (`forgedock sched cap N`), else the caller's forge.yaml `orchestration.global_parallel`, else none. Fail-open: an
|
|
18
18
|
* unreadable registry or a lock not taken within `lockWaitMs` grants (`fallback: true`), since a stuck registry would
|
|
19
19
|
* stall every batch on the host.
|
|
20
|
+
* Per-issue lock (#4481), asked for per request with `exclusive: true` in the entry (not stored; the batch runner sets it):
|
|
21
|
+
* such a key is never granted while a live slot of another batch holds the same (non-null) repo and issue, so two batches that both include issue N (a resumed batch and a new one) never run two lanes for N at once;
|
|
22
|
+
* `tryAcquire` answers `{ok:false, conflict:<batch>}` and drops the key from the queue (it would hold a grant rank).
|
|
20
23
|
*/
|
|
21
24
|
import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeSync } from "node:fs";
|
|
22
25
|
import { homedir } from "node:os";
|
|
@@ -50,6 +53,14 @@ const posInt = (k) => (Number.isInteger(k) && k >= 1 ? k : null);
|
|
|
50
53
|
const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
51
54
|
const byRank = ([ka, a], [kb, b]) => (a.priority - b.priority) || (a.seq - b.seq) || (ka < kb ? -1 : ka > kb ? 1 : 0);
|
|
52
55
|
const groupOf = (k, e) => e?.repo ?? e?.batch ?? k;
|
|
56
|
+
/** #4481: the batch of another live slot running the same repo and issue as an `exclusive` `entry`, or null (no repo: never). */
|
|
57
|
+
const holderOf = (s, key, entry) => {
|
|
58
|
+
if (entry?.exclusive !== true || !entry.repo || entry.issue == null) return null;
|
|
59
|
+
for (const [k, e] of Object.entries(s.slots)) {
|
|
60
|
+
if (k !== key && e?.repo === entry.repo && e?.issue === entry.issue && (e.batch ?? null) !== (entry.batch ?? null)) return e.batch ?? k;
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
};
|
|
53
64
|
|
|
54
65
|
/**
|
|
55
66
|
* Queue entries `[key, entry]` in grant order: priority ascending, then a weighted fair share across repos within each
|
|
@@ -88,7 +99,7 @@ export function grantOrder(s) {
|
|
|
88
99
|
|
|
89
100
|
/**
|
|
90
101
|
* Handle on the registry. `pid` is this dispatcher; `alive(pid)` is the liveness probe (injected in tests).
|
|
91
|
-
* @returns {{dir, tryAcquire, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
|
|
102
|
+
* @returns {{dir, tryAcquire, holder, offer, withdraw, release, setLanePid, adoptSlot, cap, setCap, weights, setWeight, list}}
|
|
92
103
|
*/
|
|
93
104
|
export function openSched(dir = schedDir(), { now = () => Date.now(), pid = process.pid, alive = pidAlive, lockWaitMs = LOCK_WAIT_MS } = {}) {
|
|
94
105
|
const file = join(dir, "registry.json"), lockFile = join(dir, "registry.lock");
|
|
@@ -167,6 +178,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
|
|
|
167
178
|
try {
|
|
168
179
|
return mutate((s) => {
|
|
169
180
|
if (s.slots[key]) return { ok: true };
|
|
181
|
+
const holder = holderOf(s, key, entry);
|
|
182
|
+
if (holder !== null) { delete s.queue[key]; return { ok: false, conflict: holder, reason: `#${entry.issue} has a live lane in batch ${holder}` }; }
|
|
170
183
|
enqueue(s, key, entry);
|
|
171
184
|
const cap = effCap(s, callerCap);
|
|
172
185
|
if (cap !== null) {
|
|
@@ -182,6 +195,8 @@ export function openSched(dir = schedDir(), { now = () => Date.now(), pid = proc
|
|
|
182
195
|
});
|
|
183
196
|
} catch (e) { return { ok: true, fallback: true, reason: e?.message ?? String(e) }; }
|
|
184
197
|
},
|
|
198
|
+
/** #4481: the other batch holding a live slot for `{repo, issue}` (not `batch`), or null; null on any error. */
|
|
199
|
+
holder: (entry = {}) => { try { return mutate((s) => holderOf(s, null, entry)); } catch { return null; } },
|
|
185
200
|
/** Queue entries `[{key, batch, repo, issue, priority}]` (their FIFO position is kept on a re-offer). */
|
|
186
201
|
offer: (entries) => { try { mutate((s) => { for (const e of entries) enqueue(s, e.key, e); }); } catch { /* fail-open */ } },
|
|
187
202
|
withdraw: (keys) => { if (!keys.length) return; try { mutate((s) => { for (const k of keys) delete s.queue[k]; }); } catch { /* fail-open */ } },
|
package/bin/lean/sched.test.mjs
CHANGED
|
@@ -434,3 +434,51 @@ describe("longest-first admission (#4409)", () => {
|
|
|
434
434
|
assert.equal(estCalls, 0);
|
|
435
435
|
});
|
|
436
436
|
});
|
|
437
|
+
|
|
438
|
+
describe("#4481: one lane per repo and issue across batches", () => {
|
|
439
|
+
it("the registry refuses a second batch's key for a live repo+issue slot, names the batch and drops it from the queue", () => {
|
|
440
|
+
const { handle, live } = world();
|
|
441
|
+
const a = handle(201), b = handle(202);
|
|
442
|
+
assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
|
|
443
|
+
const r = b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4);
|
|
444
|
+
assert.deepEqual([r.ok, r.conflict], [false, "A"]);
|
|
445
|
+
assert.match(r.reason, /#5 has a live lane in batch A/);
|
|
446
|
+
assert.deepEqual(b.list().queue, [], "a refused key holds no grant rank");
|
|
447
|
+
assert.equal(b.holder({ batch: "B", repo: "o/r", issue: 5, exclusive: true }), "A");
|
|
448
|
+
assert.equal(a.holder({ batch: "A", repo: "o/r", issue: 5, exclusive: true }), null, "its own batch is no conflict");
|
|
449
|
+
assert.equal(b.tryAcquire("B#6", { batch: "B", repo: "o/r", issue: 6, exclusive: true }, 4).ok, true, "another issue");
|
|
450
|
+
assert.equal(b.tryAcquire("C#5", { batch: "C", repo: "o/other", issue: 5, exclusive: true }, 4).ok, true, "another repo");
|
|
451
|
+
assert.equal(b.tryAcquire("D#5", { batch: "D", repo: null, issue: 5, exclusive: true }, 4).ok, true, "no repo: unchanged");
|
|
452
|
+
assert.equal(a.tryAcquire("A#5", { batch: "A", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true, "a granted key stays granted");
|
|
453
|
+
assert.equal(b.tryAcquire("E#5", { batch: "E", repo: "o/r", issue: 5 }, null).ok, true, "not exclusive: unchanged");
|
|
454
|
+
b.release("E#5");
|
|
455
|
+
live.delete(201); // A's dispatcher dies with no lane: its slot is reclaimed
|
|
456
|
+
assert.equal(b.tryAcquire("B#5", { batch: "B", repo: "o/r", issue: 5, exclusive: true }, 4).ok, true);
|
|
457
|
+
});
|
|
458
|
+
|
|
459
|
+
it("two batches that both include issue N never run two lanes for N; the second defers naming the first and runs its other issues", async () => {
|
|
460
|
+
const { handle } = world();
|
|
461
|
+
const regA = handle(301), regB = handle(302);
|
|
462
|
+
const active = new Map(), maxActive = new Map(), ranB = [];
|
|
463
|
+
let releaseA;
|
|
464
|
+
const gateA = new Promise((r) => { releaseA = r; });
|
|
465
|
+
const lane = (n) => { active.set(n, (active.get(n) ?? 0) + 1); maxActive.set(n, Math.max(maxActive.get(n) ?? 0, active.get(n))); };
|
|
466
|
+
const pA = orchestrate({ issues: [5], readBody: async () => "", pollMs: 2, batchId: "bA", sched: { registry: regA, batch: "bA", repo: "o/r", cap: null, exclusive: true },
|
|
467
|
+
runOne: async (n) => { lane(n); await gateA; active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
|
|
468
|
+
await until(() => active.get(5) === 1, "batch A's lane");
|
|
469
|
+
const rowsB = [];
|
|
470
|
+
const pB = orchestrate({ issues: [5, 6], readBody: async () => "", pollMs: 2, batchId: "bB", sched: { registry: regB, batch: "bB", repo: "o/r", cap: null, exclusive: true },
|
|
471
|
+
onUpdate: (r) => rowsB.push(r),
|
|
472
|
+
runOne: async (n) => { lane(n); ranB.push(n); active.set(n, active.get(n) - 1); return { terminalReason: "merged", cost_usd: 0 }; } });
|
|
473
|
+
await until(() => ranB.includes(6), "batch B's other issue");
|
|
474
|
+
const deferred = rowsB.filter((r) => r.issue === 5 && r.outcome === "deferred");
|
|
475
|
+
assert.equal(deferred.length, 1, "one row per other batch");
|
|
476
|
+
assert.match(deferred[0].detail, /live lane in batch bA/);
|
|
477
|
+
assert.ok(!ranB.includes(5), "no second lane while batch A runs #5");
|
|
478
|
+
releaseA();
|
|
479
|
+
await pA;
|
|
480
|
+
const done = await pB;
|
|
481
|
+
assert.deepEqual(done.map((r) => [r.issue, r.outcome]).sort(), [[5, "merged"], [6, "merged"]]);
|
|
482
|
+
assert.equal(maxActive.get(5), 1);
|
|
483
|
+
});
|
|
484
|
+
});
|