forgedock 1.50.0 → 1.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +2 -0
- package/bin/engine-cli.mjs +90 -9
- package/bin/forgedock.mjs +30 -2
- package/bin/lean/batch-health.mjs +219 -0
- package/bin/lean/batch-health.test.mjs +156 -0
- package/bin/lean/batch.mjs +3 -3
- package/bin/lean/batch.test.mjs +236 -8
- package/bin/lean/conformance.mjs +50 -5
- package/bin/lean/conformance.test.mjs +39 -1
- package/bin/lean/context.mjs +8 -2
- package/bin/lean/context.test.mjs +11 -1
- package/bin/lean/credentials.mjs +18 -0
- package/bin/lean/execution.mjs +11 -6
- package/bin/lean/execution.test.mjs +28 -0
- package/bin/lean/fixclosereasons.mjs +52 -0
- package/bin/lean/fixclosereasons.test.mjs +64 -0
- package/bin/lean/lean.mjs +28 -17
- package/bin/lean/lean.test.mjs +74 -13
- package/bin/lean/orchestrate-blocked.test.mjs +47 -0
- package/bin/lean/orchestrate.mjs +124 -12
- package/bin/lean/prefilter.mjs +4 -3
- package/bin/lean/prefilter.test.mjs +5 -4
- package/bin/lean/prompts.mjs +4 -2
- package/bin/lean/recovery.test.mjs +5 -5
- package/bin/lean/sandbox-provider.contract.test.mjs +15 -1
- package/bin/pool/agent.mjs +3 -1
- package/bin/pool/router.mjs +72 -7
- package/bin/review-staging.mjs +2 -0
- package/bin/runner.mjs +24 -0
- package/commands/review-pr-staging.md +2 -2
- package/docs/CONFIG.md +39 -1
- package/docs/OUTCOME-CODES.md +2 -1
- package/package.json +1 -1
- package/pool/README.md +2 -0
- package/pool/image/cs-cred +8 -1
- package/scripts/review-staging-post.sh +70 -25
- package/scripts/review-staging-post.test.sh +40 -8
package/bin/lean/lean.test.mjs
CHANGED
|
@@ -188,7 +188,7 @@ function world(o = {}) {
|
|
|
188
188
|
fs: { ...(o.wtExists ? { exists: async (p) => p.startsWith("/wt/lean-7") } : {}), stat: async (p) => (p in w.files ? { mtimeMs: w.mtimes[p] ?? 1 } : null), read: async (p) => { if (!(p in w.files)) throw new Error("ENOENT"); return w.files[p]; }, append: async (p, t) => { w.files[p] = (w.files[p] ?? "") + t; }, write: async (p, t) => { w.files[p] = t; }},
|
|
189
189
|
fileFollowups: async ({ parent, items }) => { w.followups.push({ parent, items }); if (o.followupError) throw Object.assign(new Error(o.followupError), { stdout: o.followupStdout ?? "" });
|
|
190
190
|
if (o.followupRet) return o.followupRet(w.followups.length, items);
|
|
191
|
-
if (w.followups.length <= (o.followupFailTimes ?? 0)) throw new Error("DEFERRED_LOOKUP=FAILED"); if (w.followups.length <= (o.followupMissingTimes ?? 0)) return {}; return Object.fromEntries(items.map((i) => [i.key, o.followupIssue ?? 900])); },
|
|
191
|
+
if (w.followups.length <= (o.followupFailTimes ?? 0)) throw new Error("DEFERRED_LOOKUP=FAILED"); if (w.followups.length <= (o.followupMissingTimes ?? 0)) return {}; return Object.fromEntries(items.filter((i) => !(o.followupSkip ?? []).some((q) => JSON.stringify(i).includes(q))).map((i) => [i.key, o.followupIssue ?? 900])); },
|
|
192
192
|
};
|
|
193
193
|
return w;
|
|
194
194
|
}
|
|
@@ -281,13 +281,16 @@ describe("core work-on", () => {
|
|
|
281
281
|
const t = traj(w)[0];
|
|
282
282
|
for (const needle of ["| Validate | INVALID |", "| Plan | skipped |", "| Implement | skipped |", "| Review | skipped |", "| PR | none |", "| Close | issue closed (not planned) |"]) assert.ok(t.includes(needle), needle);
|
|
283
283
|
assert.equal(w.closed.length, 1);
|
|
284
|
+
assert.equal(w.closed[0][w.closed[0].indexOf("--reason") + 1], "not planned", "#5152: invalid keeps not planned");
|
|
284
285
|
assert.equal(w.prs.length, 0);
|
|
285
286
|
assert.equal(reviewers(w).length, 0);
|
|
286
287
|
});
|
|
287
288
|
|
|
288
|
-
it("already-fixed with evidence also closes", async () => {
|
|
289
|
+
it("already-fixed with evidence also closes, as completed (#5152)", async () => {
|
|
289
290
|
const w = world({ changed: [], result: { status: "already-fixed", summary: "done upstream", evidence: ["a.js:1"], tests: [] } });
|
|
290
291
|
assert.equal((await run(w)).terminalReason, "invalid");
|
|
292
|
+
assert.deepEqual(w.closed.map((a) => a[a.indexOf("--reason") + 1]), ["completed"], "#5152: already-fixed is delivered, never not planned");
|
|
293
|
+
assert.ok(traj(w)[0].includes("| Close | issue closed (completed) |"));
|
|
291
294
|
});
|
|
292
295
|
|
|
293
296
|
it("a failed needs-human comment is recorded as an anomaly and the lane still ends needs-human (#4905)", async () => {
|
|
@@ -383,6 +386,21 @@ describe("core work-on", () => {
|
|
|
383
386
|
assert.equal(r2.code, "review.incomplete"); assert.equal(reviewers(w2).length, 3, "at most three reviewer runs"); assert.equal(w2.merges.length, 0);
|
|
384
387
|
});
|
|
385
388
|
|
|
389
|
+
it("#5164 a coverage misquote gets one more targeted re-ask with the issue's criterion lines; a near-verbatim one is requoted without a re-ask", async () => {
|
|
390
|
+
const mis = () => conf({ issue_coverage: [{ quote: "foo copes when nothing at all is passed", contract_ids: ["R1"], status: "covered" }] });
|
|
391
|
+
const w = world({ reviews: [mis(), mis(), ok()] });
|
|
392
|
+
assert.equal((await run(w)).terminalReason, "merged");
|
|
393
|
+
const rv = reviewers(w);
|
|
394
|
+
assert.equal(rv.length, 3); assert.equal(builders(w).length, 2, "never charged to the builder");
|
|
395
|
+
for (const i of [1, 2]) { assert.match(rv[i][4], /<issue_criteria>/); assert.match(rv[i][4], /- foo must handle a null argument without throwing\./); }
|
|
396
|
+
assert.match(rv[1][4], /answer ONLY for these ids or sections: issue_coverage/);
|
|
397
|
+
const w2 = world({ reviews: [mis(), mis(), mis(), ok()] });
|
|
398
|
+
const r2 = await run(w2);
|
|
399
|
+
assert.equal(r2.code, "review.incomplete"); assert.equal(reviewers(w2).length, 3, "at most three reviewer runs"); assert.equal(w2.merges.length, 0);
|
|
400
|
+
const w3 = world({ reviews: [conf({ issue_coverage: [{ quote: "foo must handle null arguments without throwing", contract_ids: ["R1"], status: "covered" }] })] });
|
|
401
|
+
assert.equal((await run(w3)).terminalReason, "merged"); assert.equal(reviewers(w3).length, 1);
|
|
402
|
+
});
|
|
403
|
+
|
|
386
404
|
it("a CONFORMS verdict whose rows fail is not conforming (the engine computes the verdict)", async () => {
|
|
387
405
|
const w = world({ reviews: [bad("half done", "CONFORMS"), ok()] });
|
|
388
406
|
assert.equal((await run(w)).terminalReason, "merged");
|
|
@@ -689,13 +707,13 @@ describe("core work-on", () => {
|
|
|
689
707
|
assert.ok(w.labels.includes("decomposed"));
|
|
690
708
|
});
|
|
691
709
|
|
|
692
|
-
it("#4337 with decomposition.close_parent a split closes the parent
|
|
710
|
+
it("#4337 #5152 with decomposition.close_parent a split closes the parent completed after its FORGE:DECOMPOSED trail and label, listing every sub-issue", async () => {
|
|
693
711
|
const w = depWorld({ changed: [], result: splitBlock }, {});
|
|
694
712
|
const r = await run(w, { closeParent: true });
|
|
695
713
|
assert.equal(r.terminalReason, "decomposed");
|
|
696
714
|
assert.equal(w.closed.length, 1);
|
|
697
715
|
const c = w.closed[0];
|
|
698
|
-
assert.equal(c[c.indexOf("--reason") + 1], "not
|
|
716
|
+
assert.equal(c[c.indexOf("--reason") + 1], "completed", "#5152: the scope moved to the sub-issues, it was not dropped");
|
|
699
717
|
const note = c[c.indexOf("--comment") + 1];
|
|
700
718
|
assert.match(note, /#201/); assert.match(note, /#202/);
|
|
701
719
|
assert.ok(w.labels.includes("decomposed"));
|
|
@@ -998,6 +1016,9 @@ describe("conformance gate (frozen contract, precheck, structured review)", () =
|
|
|
998
1016
|
const note = w.comments.find((c) => c.includes("FORGE:FOLLOWUP_PENDING"));
|
|
999
1017
|
assert.match(note, /R2 \(defer\): Existing callers of foo keep working\..*`FORGE:DEFERRED_FROM: #7`/);
|
|
1000
1018
|
assert.ok(!w.edits.some((e) => e.includes("--add-label") && e.includes("needs-human")));
|
|
1019
|
+
assert.ok(!w.comments.some((c) => /Closed as/.test(c)), "#5159: no note says the kept-open issue was closed");
|
|
1020
|
+
assert.match(traj(w).join("\n"), /left open \(follow-up pending\)/, "#5159: the trajectory Close row says left open");
|
|
1021
|
+
assert.doesNotMatch(traj(w).join("\n"), /issue closed \(/);
|
|
1001
1022
|
});
|
|
1002
1023
|
|
|
1003
1024
|
it("#5141 a follow-up that keeps failing before the PR is filed by the close step after the merge", async () => {
|
|
@@ -1053,6 +1074,21 @@ describe("conformance gate (frozen contract, precheck, structured review)", () =
|
|
|
1053
1074
|
assert.equal(r2.code, "followup.unavailable"); assert.equal(w2.followups.length, 1); assert.equal(w2.prs.length, 0);
|
|
1054
1075
|
});
|
|
1055
1076
|
|
|
1077
|
+
it("#5159 one follow-up tracked, another still pending: FORGE:TRACKED says left open, never 'Closed as', and the trajectory says left open", async () => {
|
|
1078
|
+
const k0 = two("defer", { why: "w" });
|
|
1079
|
+
const k = { ...k0, requirements: [...k0.requirements, { id: "R3", quote: "foo crashes on null input.", disposition: "defer", bucket: "post-deploy", why: "w" }] };
|
|
1080
|
+
const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupSkip: ["foo crashes on null input."] });
|
|
1081
|
+
const r = await run(w);
|
|
1082
|
+
assert.equal(r.terminalReason, "merged");
|
|
1083
|
+
assert.deepEqual((r.pendingFollowups ?? []).map((p) => p.id), ["R3"]);
|
|
1084
|
+
assert.equal(w.closed.length, 0);
|
|
1085
|
+
const tracked = w.comments.find((c) => c.includes("FORGE:TRACKED"));
|
|
1086
|
+
assert.ok(tracked, "the tracked item is still recorded");
|
|
1087
|
+
assert.match(tracked, /Left open: follow-up pending/); assert.match(tracked, /#900/);
|
|
1088
|
+
assert.ok(!w.comments.some((c) => /Closed as/.test(c)));
|
|
1089
|
+
assert.match(traj(w).join("\n"), /left open \(follow-up pending\)/);
|
|
1090
|
+
});
|
|
1091
|
+
|
|
1056
1092
|
it("a tracking issue that is not open is not a tracking issue: the item stays pending and is recorded", async () => {
|
|
1057
1093
|
const k = two("defer", { why: "w" });
|
|
1058
1094
|
const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], issueStates: { 900: "CLOSED" } });
|
|
@@ -1712,13 +1748,13 @@ describe("conformance shadow mode (the default)", () => {
|
|
|
1712
1748
|
assert.deepEqual(rs.conformance, { mode: "shadow", would: "nonconforming", risk: false });
|
|
1713
1749
|
});
|
|
1714
1750
|
|
|
1715
|
-
it("#4398 an investigation whose question is deferred closes partial (
|
|
1751
|
+
it("#4398 #5152 an investigation whose question is deferred closes partial (completed), carried forward in the tracking issue", async () => {
|
|
1716
1752
|
const k = { ...CONTRACT, preserve: [], requirements: [CONTRACT.requirements[0], { id: "R2", quote: "Existing callers of foo keep working.", disposition: "defer", why: "w", bucket: "new-issue" }] };
|
|
1717
1753
|
const w = world({ title: "investigate(stealth): which signal is rejected", contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)] });
|
|
1718
1754
|
const r = await runShadow(w);
|
|
1719
1755
|
assert.equal(r.terminalReason, "merged"); assert.equal(r.close, "partial");
|
|
1720
|
-
assert.deepEqual(w.closed.map((a) => a[a.indexOf("--reason") + 1]), ["
|
|
1721
|
-
assert.ok(w.comments.some((c) => c.includes("<!-- FORGE:TRACKED -->") && /Closed as partial[\s\S]*carried forward in #900/.test(c)));
|
|
1756
|
+
assert.deepEqual(w.closed.map((a) => a[a.indexOf("--reason") + 1]), ["completed"], "#5152: merged with a tracked remainder is completed");
|
|
1757
|
+
assert.ok(w.comments.some((c) => c.includes("<!-- FORGE:TRACKED -->") && /Closed as partial[\s\S]*Closed as completed by PR #[\s\S]*carried forward in #900/.test(c)));
|
|
1722
1758
|
assert.ok(traj(w)[0].includes("| Close | issue closed (partial) |"));
|
|
1723
1759
|
});
|
|
1724
1760
|
|
|
@@ -1747,7 +1783,7 @@ describe("conformance shadow mode (the default)", () => {
|
|
|
1747
1783
|
assert.equal(r.terminalReason, "merged"); assert.equal(r.close, "partial");
|
|
1748
1784
|
assert.equal(w.followups.length, 1); assert.match(w.followups[0].items[0].key, /^R1: /); assert.match(w.followups[0].items[0].why, /investigation question not answered/);
|
|
1749
1785
|
assert.match(w.prs[0].join(" "), /#900/);
|
|
1750
|
-
assert.deepEqual(w.closed.map((a) => a[a.indexOf("--reason") + 1]), ["
|
|
1786
|
+
assert.deepEqual(w.closed.map((a) => a[a.indexOf("--reason") + 1]), ["completed"]);
|
|
1751
1787
|
});
|
|
1752
1788
|
|
|
1753
1789
|
it("#4398 an answered investigation and a non-investigation with a deferral both close completed", async () => {
|
|
@@ -2649,10 +2685,10 @@ describe("orchestrate", () => {
|
|
|
2649
2685
|
assert.ok(h.log.ends.indexOf(1) < h.log.starts.indexOf(2), JSON.stringify(h.log));
|
|
2650
2686
|
});
|
|
2651
2687
|
|
|
2652
|
-
it("a dependent is
|
|
2688
|
+
it("a dependent is held, without running, when its dependency ends needs-human (transitively)", async () => {
|
|
2653
2689
|
const h = harness({ 2: "**Depends on**: #1", 3: "**Depends on**: #2" }, { 1: "needs-human" });
|
|
2654
2690
|
const rows = await orchestrate({ issues: [1, 2, 3, 4], parallel: 3, ...h });
|
|
2655
|
-
assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "
|
|
2691
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held", "held", "merged"]);
|
|
2656
2692
|
assert.ok(!h.log.starts.includes(2) && !h.log.starts.includes(3));
|
|
2657
2693
|
assert.match(rows[1].detail, /#1/);
|
|
2658
2694
|
});
|
|
@@ -2733,7 +2769,7 @@ describe("orchestrate", () => {
|
|
|
2733
2769
|
const rows = await orchestrate({ issues: [9, 1], parallel: 1, readBody: async (n) => (n === 1 ? "**Depends on**: #9" : ""), ...(dispatchCheck ? { dispatchCheck } : {}),
|
|
2734
2770
|
runOne: async (n) => { log.push(n); return { terminalReason: "needs-human", detail: "ci.red" }; } });
|
|
2735
2771
|
assert.deepEqual(log, [9]);
|
|
2736
|
-
assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[9, "needs-human"], [1, "
|
|
2772
|
+
assert.deepEqual(rows.map((r) => [r.issue, r.outcome]), [[9, "needs-human"], [1, "held"]]);
|
|
2737
2773
|
}
|
|
2738
2774
|
});
|
|
2739
2775
|
|
|
@@ -2744,7 +2780,7 @@ describe("orchestrate", () => {
|
|
|
2744
2780
|
onUpdate: (r) => rowsSeen.push(r),
|
|
2745
2781
|
dispatchCheck: async (n) => (n === 9 && merged ? { outcome: "merged", pr: 90, detail: "PR #90 already merged" } : null),
|
|
2746
2782
|
runOne: async (n) => { if (n === 9) { merged = true; return { terminalReason: "needs-human", detail: "ci.red" }; } return { terminalReason: "merged", pr: 101 }; } });
|
|
2747
|
-
const skipRow = rowsSeen.find((r) => r.issue === 1 && r.outcome === "
|
|
2783
|
+
const skipRow = rowsSeen.find((r) => r.issue === 1 && r.outcome === "held");
|
|
2748
2784
|
assert.equal(skipRow.waiting, true);
|
|
2749
2785
|
assert.equal(skipRow.trigger, 9);
|
|
2750
2786
|
assert.ok(rowsSeen.some((r) => r.issue === 1 && r.outcome === "readmitted" && r.trigger === 9));
|
|
@@ -2756,7 +2792,7 @@ describe("orchestrate", () => {
|
|
|
2756
2792
|
const rows = await orchestrate({ issues: [9, 1], parallel: 1, readBody: async (n) => (n === 1 ? "**Depends on**: #9" : ""),
|
|
2757
2793
|
dispatchCheck: async (n) => { if (n === 9 && ran) { checks++; return { outcome: "invalid", detail: "issue closed (not planned)", final: true }; } return null; },
|
|
2758
2794
|
runOne: async () => { ran = true; return { terminalReason: "needs-human", detail: "ci.red" }; } });
|
|
2759
|
-
assert.deepEqual(rows.map((r) => [r.issue, r.outcome, r.waiting]), [[9, "needs-human", undefined], [1, "
|
|
2795
|
+
assert.deepEqual(rows.map((r) => [r.issue, r.outcome, r.waiting]), [[9, "needs-human", undefined], [1, "held", undefined]]);
|
|
2760
2796
|
assert.equal(checks, 1, "a final blocker is not re-checked");
|
|
2761
2797
|
});
|
|
2762
2798
|
|
|
@@ -3410,6 +3446,31 @@ describe("usage limit, lease takeover, worktree setup, run tmp (#4101-#4104)", (
|
|
|
3410
3446
|
assert.equal(w.merges.length, 0);
|
|
3411
3447
|
});
|
|
3412
3448
|
|
|
3449
|
+
it("#5168 a claude login failure ends the lane deferred/backend.auth with the account, not engine-error", async () => {
|
|
3450
|
+
const w = world(); projector(w, null); const sleeps = [];
|
|
3451
|
+
const s0 = w.io.sleep; w.io.sleep = async (ms) => { sleeps.push(ms); return s0(ms); };
|
|
3452
|
+
const authErr = Object.assign(new Error("claude CLI exited with status 1. Failed to authenticate. API Error: 401"), { code: "CLI_BACKEND_FAILED", authFailure: true });
|
|
3453
|
+
let tries = 0; const orig = w.io.runner; w.io.runner = async (c) => { if (c.role === "builder") { tries++; throw authErr; } return orig(c); };
|
|
3454
|
+
const r = await run(w);
|
|
3455
|
+
assert.equal(r.terminalReason, "deferred");
|
|
3456
|
+
assert.equal(r.code, "backend.auth");
|
|
3457
|
+
assert.equal(r.retryInSeconds, 900);
|
|
3458
|
+
assert.equal(r.backendId, laneBackendId(process.env), "the hold names the account whose login failed");
|
|
3459
|
+
assert.match(r.detail, /claude CLI authentication failed/);
|
|
3460
|
+
assert.equal(tries, 1, "no in-lane retry or pause on a dead login");
|
|
3461
|
+
assert.equal(sleeps.length, 0);
|
|
3462
|
+
assert.equal(w.edits.some((e) => e.includes("needs-human")), false);
|
|
3463
|
+
assert.equal(w.leaseWrites.at(-1).lease, null, "lease released");
|
|
3464
|
+
});
|
|
3465
|
+
|
|
3466
|
+
it("#5168 an error that is both a usage limit and auth-like stays rate_limited", async () => {
|
|
3467
|
+
const w = world();
|
|
3468
|
+
failFirstBuilder(w, limitErr({ authFailure: true, resetAtEpochMs: 3 * 3600 * 1000, resetAt: "8:30pm" }));
|
|
3469
|
+
const r = await run(w);
|
|
3470
|
+
assert.equal(r.terminalReason, "deferred");
|
|
3471
|
+
assert.equal(r.code, "rate_limited");
|
|
3472
|
+
});
|
|
3473
|
+
|
|
3413
3474
|
it("#4101 unknown reset pauses a bounded number of times, then defers", async () => {
|
|
3414
3475
|
const w = world(); let sleeps = 0;
|
|
3415
3476
|
const s0 = w.io.sleep; w.io.sleep = async (ms) => { sleeps++; return s0(ms); };
|
|
@@ -120,3 +120,50 @@ describe("#5048: the pool is unreachable only after a failed capacity probe", ()
|
|
|
120
120
|
assert.deepEqual(notes, ["down", "clear"]);
|
|
121
121
|
});
|
|
122
122
|
});
|
|
123
|
+
|
|
124
|
+
describe("#5162: failed roots hold their subtree; only real cycles are cycles", () => {
|
|
125
|
+
const memStore = () => { let st = null; return { load: () => st, save: (s) => { st = JSON.parse(JSON.stringify(s)); }, get: () => st }; };
|
|
126
|
+
const chain = { 1: "", 2: "**Depends on**: #1", 3: "**Depends on**: #2", 4: "**Depends on**: #1, #3" };
|
|
127
|
+
const outcome = (rows, n) => rows.find((r) => r.issue === n);
|
|
128
|
+
|
|
129
|
+
it("a DAG whose engine-error root ends marks no row dependency cycle; every dependent is held and names the root", async () => {
|
|
130
|
+
const rows = await orchestrate({ issues: [1, 2, 3, 4], parallel: 2, pollMs: 5, readBody: async (n) => chain[n], engineErrorRetries: 0, transientRetries: 0,
|
|
131
|
+
runOne: async () => ({ terminalReason: "engine-error", detail: "boom" }) });
|
|
132
|
+
assert.ok(!rows.some((r) => /dependency cycle/.test(r.detail ?? "")), JSON.stringify(rows));
|
|
133
|
+
for (const n of [2, 3, 4]) { assert.equal(outcome(rows, n).outcome, "held"); assert.match(outcome(rows, n).detail, /^blocked by #1 \(engine-error\)$/); }
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("a genuine cycle is still labelled dependency cycle; its downstream names a member", async () => {
|
|
137
|
+
const bodies = { 1: "**Depends on**: #2", 2: "**Depends on**: #1", 3: "**Depends on**: #2" };
|
|
138
|
+
const rows = await orchestrate({ issues: [1, 2, 3], parallel: 2, pollMs: 5, readBody: async (n) => bodies[n], runOne: async () => merged });
|
|
139
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["skipped", "skipped", "skipped"]);
|
|
140
|
+
assert.equal(outcome(rows, 1).detail, "dependency cycle");
|
|
141
|
+
assert.equal(outcome(rows, 2).detail, "dependency cycle");
|
|
142
|
+
assert.match(outcome(rows, 3).detail, /^blocked by #[12] \(dependency cycle\)$/);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("dependents of a needs-human root end held; after the label is removed a resume admits root then dependents in order", async () => {
|
|
146
|
+
const store = memStore();
|
|
147
|
+
const first = await orchestrate({ issues: [1, 2, 3, 4], parallel: 2, pollMs: 5, store, readBody: async (n) => chain[n], runOne: async () => ({ terminalReason: "needs-human", code: "ci.red", detail: "red" }) });
|
|
148
|
+
assert.equal(outcome(first, 1).outcome, "needs-human");
|
|
149
|
+
for (const n of [2, 3, 4]) assert.deepEqual([outcome(first, n).outcome, outcome(first, n).detail], ["held", "blocked by #1 (needs-human)"]);
|
|
150
|
+
const started = [];
|
|
151
|
+
const rows = await orchestrate({ issues: [1, 2, 3, 4], parallel: 3, pollMs: 5, store, readBody: async (n) => chain[n], issueLabels: async () => [],
|
|
152
|
+
runOne: async (n) => { started.push(n); await new Promise((r) => setTimeout(r, 5)); return merged; } });
|
|
153
|
+
assert.deepEqual(started, [1, 2, 3, 4]);
|
|
154
|
+
assert.ok(rows.every((r) => r.outcome === "merged"), JSON.stringify(rows));
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("a resume with the needs-human label still on the root keeps the subtree held", async () => {
|
|
158
|
+
const store = memStore();
|
|
159
|
+
await orchestrate({ issues: [1, 2], parallel: 1, pollMs: 5, store, readBody: async (n) => chain[n], runOne: async () => ({ terminalReason: "needs-human", detail: "x" }) });
|
|
160
|
+
const rows = await orchestrate({ issues: [1, 2], parallel: 1, pollMs: 5, store, readBody: async (n) => chain[n], issueLabels: async () => ["needs-human"], runOne: async () => merged });
|
|
161
|
+
assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("the report lists a held subtree with its root and one --requeue command", async () => {
|
|
165
|
+
const { formatOutcomeLists } = await import("./orchestrate.mjs");
|
|
166
|
+
const lines = formatOutcomeLists([{ issue: 1, outcome: "engine-error" }, { issue: 2, outcome: "held", root: 1 }, { issue: 3, outcome: "held", root: 1 }]);
|
|
167
|
+
assert.ok(lines.some((l) => /held subtree: #1 \(engine-error\) holds #2 #3; release: forgedock orchestrate --requeue <batch> 1/.test(l)), lines.join("\n"));
|
|
168
|
+
});
|
|
169
|
+
});
|
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -20,8 +20,10 @@
|
|
|
20
20
|
* A seam calls `onLive()` before it waits on a live lane (#4504); an adopt that resolves first is settled (or requeued on null)
|
|
21
21
|
* before scheduling, so a live lane does not hold up the others.
|
|
22
22
|
* `deferred` is a scheduling fact, not an end: each typed code is requeued with its retry-at (`deferPolicy`), a usage
|
|
23
|
-
* limit holds admissions for that account until its reset, and deferrals other than a wait (`host.busy`,
|
|
24
|
-
* `sandbox.capacity`, #4528; `sandbox.unreachable`, #4788; `rate_limited`, #4784) stop after `maxDeferrals`.
|
|
23
|
+
* limit (and #5168 a claude login failure, `backend.auth`) holds admissions for that account until its reset, and deferrals other than a wait (`host.busy`,
|
|
24
|
+
* `sandbox.capacity`, #4528; `sandbox.unreachable`, #4788; `rate_limited`, #4784; `backend.auth`, #5168) stop after `maxDeferrals`.
|
|
25
|
+
* #5161: an engine-error whose detail is a transport/gateway failure (`isTransientEngineError`) is requeued with backoff
|
|
26
|
+
* (`transientRetryDelayMs`) up to `transientRetries` times, not counted against `ENGINE_ERROR_RELAUNCHES`; then it ends engine-error.
|
|
25
27
|
* #5112: with `remote.offlineFallback` (`execution.fallback_local`), an issue deferred `remote.capacityFallbackAfter` times (default
|
|
26
28
|
* `CAPACITY_FALLBACK_AFTER`) for `sandbox.capacity` runs on the host when `remote.headroom()` (host-busy guard) allows it.
|
|
27
29
|
* Usage-limit holds (#4190) are keyed by backend identity in `limits` (the host store, shared by every batch on the
|
|
@@ -128,6 +130,27 @@ import { joinsDecomposed, addsEpicChildren, selectChildren, edgeText, DEFAULT_PO
|
|
|
128
130
|
|
|
129
131
|
// A dependency that finished like this no longer blocks its dependents.
|
|
130
132
|
const SATISFIED = new Set(["merged", "invalid"]);
|
|
133
|
+
// #5162: a dependent of a root that ended like this is held (resumable), not skipped.
|
|
134
|
+
const HOLD_ROOT = new Set(["needs-human", "engine-error", "deferred", "budget-exceeded"]);
|
|
135
|
+
/** #5162: members of a real dependency cycle (a nontrivial SCC or a self-loop) among `nodes`; `edgesOf(n)` lists n's dependencies. */
|
|
136
|
+
export function cyclicNodes(nodes, edgesOf) {
|
|
137
|
+
const inSet = new Set(nodes), idx = new Map(), low = new Map(), stack = [], on = new Set(), out = new Set();
|
|
138
|
+
let i = 0;
|
|
139
|
+
const visit = (v) => {
|
|
140
|
+
idx.set(v, i); low.set(v, i); i++; stack.push(v); on.add(v);
|
|
141
|
+
for (const w of edgesOf(v).filter((d) => inSet.has(d))) {
|
|
142
|
+
if (w === v) out.add(v);
|
|
143
|
+
if (!idx.has(w)) { visit(w); low.set(v, Math.min(low.get(v), low.get(w))); } else if (on.has(w)) low.set(v, Math.min(low.get(v), idx.get(w)));
|
|
144
|
+
}
|
|
145
|
+
if (low.get(v) === idx.get(v)) {
|
|
146
|
+
const comp = [];
|
|
147
|
+
for (let w; w !== v;) { w = stack.pop(); on.delete(w); comp.push(w); }
|
|
148
|
+
if (comp.length > 1) for (const c of comp) out.add(c);
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
for (const n of nodes) if (!idx.has(n)) visit(n);
|
|
152
|
+
return out;
|
|
153
|
+
}
|
|
131
154
|
const WATCH_EVERY_MS = 60_000; // #4595
|
|
132
155
|
// Times one issue may be requeued behind a dependency before its dependency-open row is final.
|
|
133
156
|
export const MAX_REQUEUES = 2;
|
|
@@ -146,6 +169,15 @@ export const CAPACITY_FALLBACK_AFTER = 3;
|
|
|
146
169
|
export const RATE_LIMIT_FALLBACK_S = 15 * 60;
|
|
147
170
|
export const DEFER_BACKOFF_S = 120;
|
|
148
171
|
export const DEFER_BACKOFF_MAX_S = 30 * 60;
|
|
172
|
+
// #4685 + #5161: an engine-error detail that names a link or gateway failure (curl's "returned error: 502", HTTP/1.1 503, ...).
|
|
173
|
+
export const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]|returned error: 5\d\d|HTTP\/\d(\.\d)? 5\d\d|Bad Gateway|Service Unavailable|overloaded/i;
|
|
174
|
+
/** #5161: a transient engine failure (model gateway 5xx, connection reset): retried with backoff, not the single relaunch. */
|
|
175
|
+
export const isTransientEngineError = (detail) => TRANSPORT_TEXT.test(String(detail ?? ""));
|
|
176
|
+
// #5161: transient engine-error retries per issue (forge.yaml `orchestration.transient_retries`) and their backoff: 30s, 2m, 8m, then 15m.
|
|
177
|
+
export const DEFAULT_TRANSIENT_RETRIES = 4;
|
|
178
|
+
export const TRANSIENT_BACKOFF_S = 30;
|
|
179
|
+
export const TRANSIENT_BACKOFF_MAX_S = 15 * 60;
|
|
180
|
+
export const transientRetryDelayMs = (k) => Math.min(TRANSIENT_BACKOFF_S * 4 ** (Math.max(1, k) - 1), TRANSIENT_BACKOFF_MAX_S) * 1000;
|
|
149
181
|
// A dependency-open whose dependencies already finished satisfied (GitHub not yet closed): wait before relaunching.
|
|
150
182
|
export const DEP_REQUEUE_DELAY_S = 60;
|
|
151
183
|
|
|
@@ -204,6 +236,8 @@ export function deferPolicy(r, { now, deferrals = 0, maxDeferrals = DEFAULT_MAX_
|
|
|
204
236
|
const at = posNum(r?.resetAtEpochMs) && r.resetAtEpochMs > now ? r.resetAtEpochMs : now + (posNum(r?.retryInSeconds) ?? RATE_LIMIT_FALLBACK_S) * 1000;
|
|
205
237
|
return { delayMs: at - now, hold: true };
|
|
206
238
|
}
|
|
239
|
+
// #5168: a failed claude login is an account condition, not an engine fault: held like a usage limit, never capped.
|
|
240
|
+
if (code === "backend.auth") return { delayMs: (posNum(r?.retryInSeconds) ?? RATE_LIMIT_FALLBACK_S) * 1000, hold: true };
|
|
207
241
|
const k = deferrals + 1;
|
|
208
242
|
if (k > maxDeferrals) return { capped: true };
|
|
209
243
|
if (posNum(r?.retryInSeconds)) return { delayMs: r.retryInSeconds * 1000 };
|
|
@@ -267,7 +301,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
267
301
|
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, integration = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null, preflight = null,
|
|
268
302
|
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, issueState = null, getCancelled = null, getHeld = null, getBlocked = null, trusted = null, getGates = null, stopLane = null,
|
|
269
303
|
estimate = null, starveMs = DEFAULT_STARVE_MS, setupConcurrency = DEFAULT_SETUP_CONCURRENCY, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS, lifecycle = null,
|
|
270
|
-
getChildren = null, autoChildren = null, baseWait = null, rotator = null }) {
|
|
304
|
+
getChildren = null, autoChildren = null, baseWait = null, rotator = null, transientRetries = DEFAULT_TRANSIENT_RETRIES }) {
|
|
271
305
|
const saved = store?.load?.() ?? null;
|
|
272
306
|
const order = saved ? [...saved.order] : [];
|
|
273
307
|
const set = new Set(order);
|
|
@@ -284,6 +318,8 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
284
318
|
const capDefers = toMap(saved?.capDeferrals); // #5112: issue -> `sandbox.capacity` deferrals (uncapped, so not in `deferrals`)
|
|
285
319
|
const capFellBack = new Set(); // #5112: issues already reported as a capacity fallback (one decision each)
|
|
286
320
|
const engineErrors = toMap(saved?.engineErrors);
|
|
321
|
+
const transients = toMap(saved?.transientRetries); // #5161: issue -> transient engine-error retries taken
|
|
322
|
+
const transientMax = Number.isInteger(transientRetries) && transientRetries >= 0 ? transientRetries : DEFAULT_TRANSIENT_RETRIES;
|
|
287
323
|
const checked = toMap(saved?.prefilter); // issue -> pre-filter result
|
|
288
324
|
const estimates = toMap(saved?.estimates); // #4409: issue -> estimated lane minutes; only with `estimate`
|
|
289
325
|
const enqueued = toMap(saved?.enqueued); // #4409: issue -> when it last entered the queue (starvation guard)
|
|
@@ -369,7 +405,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
369
405
|
const save = () => {
|
|
370
406
|
if (!store || halted) return;
|
|
371
407
|
store.save({ v: 1, ...(setup ? { phase: "setup", setup: { ...setup } } : {}), order, deps: toObj(deps), lanes: toObj(lanes), crossSeen: [...crossSeen], results: toObj(results), running: toObj(runRecs),
|
|
372
|
-
pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors), ...(quota.size ? { quota: toObj(quota) } : {}), ...(capDefers.size ? { capDeferrals: toObj(capDefers) } : {}),
|
|
408
|
+
pending, notBefore: toObj(notBefore), requeues: toObj(requeues), deferrals: toObj(deferrals), engineErrors: toObj(engineErrors), ...(transients.size ? { transientRetries: toObj(transients) } : {}), ...(quota.size ? { quota: toObj(quota) } : {}), ...(capDefers.size ? { capDeferrals: toObj(capDefers) } : {}),
|
|
373
409
|
prefilter: toObj(checked), errorTimes, breaker: tripped, ...(budget ? { classes: toObj(classes), dispatch: { ...dispatch } } : {}),
|
|
374
410
|
...(reg ? { priorities: toObj(priorities) } : {}), ...(sandboxUse.size ? { sandbox: toObj(sandboxUse) } : {}),
|
|
375
411
|
...(recoveries.size || recovering.size ? { recoveries: toObj(recoveries), recovering: toObj(recovering), recoveryClasses: toObj(classUsed) } : {}),
|
|
@@ -633,15 +669,19 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
633
669
|
results.set(n, { issue: n, outcome: "skipped", pr: null, cost_usd: 0, minutes: 0, detail, ...(trigger !== null && dispatchCheck ? { waiting: true } : {}) });
|
|
634
670
|
update({ ...results.get(n), ...(trigger !== null ? { trigger } : {}) });
|
|
635
671
|
};
|
|
672
|
+
// #5162: a dependent of a failed root ends held (trigger = the root, not the parent) and is released with the root.
|
|
673
|
+
const holdBehind = (n, root, rootOutcome) => {
|
|
674
|
+
results.set(n, { issue: n, outcome: "held", pr: null, cost_usd: 0, minutes: 0, detail: `blocked by #${root} (${rootOutcome})`, root, ...(dispatchCheck ? { waiting: true } : {}) });
|
|
675
|
+
update({ ...results.get(n), trigger: root });
|
|
676
|
+
};
|
|
636
677
|
const requeue = (n, delayMs) => { pending.push(n); enqueued.set(n, now()); if (delayMs > 0) notBefore.set(n, now() + delayMs); else notBefore.delete(n); };
|
|
637
|
-
const TRANSPORT_TEXT = /fetch failed|ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up|HTTP 50[234]/i;
|
|
638
678
|
const TRANSPORT_COOLDOWN_MS = 5 * 60000;
|
|
639
679
|
let transportErrors = 0; // #4685: engine-errors in the current window that were link failures
|
|
640
680
|
const noteEngineError = (detail = "") => {
|
|
641
681
|
if (!breaker) return;
|
|
642
682
|
const t = now(), windowMs = breaker.window_min * 60000;
|
|
643
683
|
errorTimes = [...errorTimes.filter((x) => t - x < windowMs), t];
|
|
644
|
-
transportErrors =
|
|
684
|
+
transportErrors = isTransientEngineError(detail) ? Math.min(transportErrors + 1, errorTimes.length) : 0;
|
|
645
685
|
if (!tripped && errorTimes.length >= breaker.errors) {
|
|
646
686
|
tripped = { at: t, ...(transportErrors >= errorTimes.length ? { transport: true } : {}), detail: `circuit breaker: ${errorTimes.length} engine-errors within ${breaker.window_min} min; admissions stopped` };
|
|
647
687
|
update({ issue: null, outcome: "breaker", detail: tripped.detail });
|
|
@@ -757,7 +797,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
757
797
|
resumeAt, rotations: (q.rotations ?? 0) + (rq?.rotated ? 1 : 0) });
|
|
758
798
|
requeue(n, delayMs); done(); save();
|
|
759
799
|
const iso = new Date(resumeAt).toISOString();
|
|
760
|
-
update({ ...row, outcome: "paused", quota: { resumeAt: iso }, detail: `paused (quota) until ${iso}: ${r?.code ?? "deferred"}${r?.detail ? ` ${r.detail}` : ""}` });
|
|
800
|
+
update({ ...row, outcome: "paused", quota: { resumeAt: iso }, detail: `paused (${r?.code === "backend.auth" ? "backend.auth" : "quota"}) until ${iso}: ${r?.code ?? "deferred"}${r?.detail ? ` ${r.detail}` : ""}` });
|
|
761
801
|
return;
|
|
762
802
|
}
|
|
763
803
|
requeue(n, capGated ? 0 : delayMs); done(); save();
|
|
@@ -768,7 +808,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
768
808
|
}
|
|
769
809
|
if (row.outcome === "engine-error") {
|
|
770
810
|
noteEngineError(r?.detail);
|
|
771
|
-
|
|
811
|
+
// #5161: a transient failure (gateway 5xx, reset) waits out the outage with backoff instead of the single immediate relaunch.
|
|
812
|
+
if (transientMax > 0 && isTransientEngineError(row.detail)) {
|
|
813
|
+
const k = (transients.get(n) ?? 0) + 1;
|
|
814
|
+
if (k <= transientMax) {
|
|
815
|
+
const delayMs = transientRetryDelayMs(k), secs = Math.round(delayMs / 1000);
|
|
816
|
+
transients.set(n, k);
|
|
817
|
+
requeue(n, delayMs); done(); save();
|
|
818
|
+
decided({ issue: n, what: `retry after transient engine-error (${k}/${transientMax}) in ${secs}s`, why: `transient: ${firstLine(row.detail) || "engine-error"}`,
|
|
819
|
+
alternative: "end the issue engine-error and skip its dependents" });
|
|
820
|
+
update({ ...row, detail: `transient retry ${k}/${transientMax} in ${secs}s: ${row.detail ?? ""}` });
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
823
|
+
row.detail = `gave up after ${transientMax} transient retries: ${row.detail ?? ""}`;
|
|
824
|
+
} else if ((engineErrors.get(n) ?? 0) < ENGINE_ERROR_RELAUNCHES) {
|
|
772
825
|
engineErrors.set(n, (engineErrors.get(n) ?? 0) + 1);
|
|
773
826
|
requeue(n, 0); done(); save();
|
|
774
827
|
decided({ issue: n, what: "relaunch once after engine-error", why: firstLine(row.detail) || "engine-error", alternative: "end the issue engine-error and skip its dependents" });
|
|
@@ -1039,12 +1092,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1039
1092
|
results.delete(e.issue);
|
|
1040
1093
|
for (let i = 0; i < moved.length; i++) { // dependents skipped behind a requeued issue (fixpoint)
|
|
1041
1094
|
for (const [m, r] of [...results]) {
|
|
1042
|
-
|
|
1095
|
+
const heldBehind = r?.outcome === "held" && r.root === moved[i]; // #5162
|
|
1096
|
+
if (!heldBehind && (r?.outcome !== "skipped" || Number(/^dependency #(\d+)/.exec(r.detail ?? "")?.[1]) !== moved[i])) continue;
|
|
1043
1097
|
priors.set(m, r); results.delete(m); moved.push(m);
|
|
1044
1098
|
}
|
|
1045
1099
|
}
|
|
1046
1100
|
for (const m of moved) {
|
|
1047
|
-
for (const x of [requeues, deferrals, capDefers, engineErrors, recoveries, recovering]) x.delete(m);
|
|
1101
|
+
for (const x of [requeues, deferrals, capDefers, engineErrors, transients, recoveries, recovering]) x.delete(m);
|
|
1048
1102
|
relabelled.delete(m);
|
|
1049
1103
|
if (m === e.issue) requeuedInfo.set(m, { reason: e.reason, by: e.by, at: e.at, prior: [...(requeuedInfo.get(m)?.prior ?? []), priors.get(m)] });
|
|
1050
1104
|
if (!pending.includes(m)) requeue(m, 0);
|
|
@@ -1182,7 +1236,45 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1182
1236
|
const readmitted = new Set(); // "n:b": re-admit n on blocker b once; a lane that re-skips on b stays skipped
|
|
1183
1237
|
const blockerOf = (n) => { const r = results.get(n); const m = r?.outcome === "skipped" ? /^dependency #(\d+) /.exec(r.detail ?? "") : null; return m ? Number(m[1]) : null; };
|
|
1184
1238
|
const watchable = () => !!dispatchCheck && [...results.keys()].some((n) => blockerOf(n) !== null);
|
|
1239
|
+
// #5162: held rows behind `root` return to pending (and the root too when `withRoot`: requeued, counters reset), parents first via the edges.
|
|
1240
|
+
const releaseHeld = (root, withRoot, why) => {
|
|
1241
|
+
const moved = withRoot && results.has(root) ? [root] : [];
|
|
1242
|
+
if (moved.length) results.delete(root);
|
|
1243
|
+
for (const [m, r] of [...results]) if (r?.outcome === "held" && r.root === root) { results.delete(m); moved.push(m); }
|
|
1244
|
+
for (const m of moved) {
|
|
1245
|
+
for (const x of [requeues, deferrals, capDefers, engineErrors, transients, recoveries, recovering]) x.delete(m);
|
|
1246
|
+
relabelled.delete(m);
|
|
1247
|
+
if (!pending.includes(m)) requeue(m, 0);
|
|
1248
|
+
update({ issue: m, outcome: "readmitted", detail: m === root ? why : `dependency #${root} ${why}`, trigger: root });
|
|
1249
|
+
}
|
|
1250
|
+
if (moved.length) { save(); decided({ issue: root, what: `released held subtree of #${root}: ${moved.filter((m) => m !== root).map((m) => `#${m}`).join(", ") || "none"}`, why, alternative: "leave it held until a --requeue" }); }
|
|
1251
|
+
};
|
|
1252
|
+
const restoredHeld = new Set([...results.values()].filter((r) => r?.outcome === "held" && Number.isInteger(r.root)).map((r) => r.root)); // roots held by an earlier run of this batch
|
|
1253
|
+
const watchHeldRoots = async () => {
|
|
1254
|
+
for (const root of new Set([...results.values()].filter((r) => r?.outcome === "held" && Number.isInteger(r.root)).map((r) => r.root))) {
|
|
1255
|
+
const ro = results.get(root)?.outcome;
|
|
1256
|
+
if (ro === undefined || running.has(root) || settled.has(root)) continue;
|
|
1257
|
+
if (restoredHeld.has(root) && ["needs-human", "budget-exceeded"].includes(ro) && typeof issueLabels === "function") { // label removed: requeue the root, its subtree follows
|
|
1258
|
+
let names = null;
|
|
1259
|
+
try { names = await issueLabels(root); } catch { names = null; }
|
|
1260
|
+
if (Array.isArray(names) && !names.includes("needs-human")) { releaseHeld(root, true, "needs-human label removed"); continue; }
|
|
1261
|
+
}
|
|
1262
|
+
if (!dispatchCheck) continue;
|
|
1263
|
+
let g = null;
|
|
1264
|
+
try { g = (await dispatchCheck(root)) ?? null; } catch { g = null; }
|
|
1265
|
+
if (g?.final) { // closed as not planned: terminal for its held dependents
|
|
1266
|
+
settled.add(root);
|
|
1267
|
+
for (const [m, r] of [...results]) if (r?.outcome === "held" && r.root === root && r.waiting) { const { waiting, ...rest } = r; results.set(m, rest); update({ ...rest, trigger: root }); }
|
|
1268
|
+
continue;
|
|
1269
|
+
}
|
|
1270
|
+
if (g && SATISFIED.has(g.outcome)) { // merged or closed outside the batch
|
|
1271
|
+
results.set(root, { ...results.get(root), outcome: g.outcome, pr: Number.isInteger(g.pr) ? g.pr : results.get(root).pr, detail: g.detail ?? results.get(root).detail });
|
|
1272
|
+
update(results.get(root)); releaseHeld(root, false, "merged or closed");
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
};
|
|
1185
1276
|
const watchBlockers = async (final) => {
|
|
1277
|
+
if (!cancelled && !tripped && (final || lastWatch === null || now() - lastWatch >= WATCH_EVERY_MS) && [...results.values()].some((r) => r?.outcome === "held" && Number.isInteger(r.root))) await watchHeldRoots();
|
|
1186
1278
|
if (!watchable() || cancelled || tripped) return;
|
|
1187
1279
|
const t0 = now();
|
|
1188
1280
|
if (!final && lastWatch !== null && t0 - lastWatch < WATCH_EVERY_MS) return;
|
|
@@ -1240,7 +1332,12 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1240
1332
|
const n = pending[i];
|
|
1241
1333
|
if (results.has(n)) { pending.splice(i, 1); continue; }
|
|
1242
1334
|
const bad = edges(n).find((d) => results.has(d) && !SATISFIED.has(results.get(d).outcome));
|
|
1243
|
-
if (bad !== undefined) {
|
|
1335
|
+
if (bad !== undefined) {
|
|
1336
|
+
pending.splice(i, 1); again = true;
|
|
1337
|
+
const br = results.get(bad), root = br.root ?? bad, ro = results.get(root)?.outcome ?? br.outcome;
|
|
1338
|
+
if (HOLD_ROOT.has(ro)) holdBehind(n, root, ro); // #5162
|
|
1339
|
+
else skip(n, `dependency #${bad} ended ${br.outcome}`, bad);
|
|
1340
|
+
}
|
|
1244
1341
|
}
|
|
1245
1342
|
}
|
|
1246
1343
|
const t = now();
|
|
@@ -1373,7 +1470,19 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
1373
1470
|
else if (pending.length && (driftHeld.size || integHeld.size)) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch) or a red integration streak (#4667): pending for a later batch
|
|
1374
1471
|
else if (pending.some((n) => remoteCapped.has(n))) await nap(wake ?? pollMs); // #5138: live max_remote 0: wait for a raise or another placement
|
|
1375
1472
|
else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
|
|
1376
|
-
|
|
1473
|
+
// #5162: only real cycles are labelled so; the rest name the cycle member (or failed root) that blocks them
|
|
1474
|
+
const cyc = cyclicNodes(pending, edges), inP = new Set(pending);
|
|
1475
|
+
const blocker = (n, seen = new Set()) => {
|
|
1476
|
+
for (const d of edges(n)) {
|
|
1477
|
+
if (cyc.has(d)) return d;
|
|
1478
|
+
if (inP.has(d) && !seen.has(d)) { seen.add(d); const b = blocker(d, seen); if (b !== null) return b; }
|
|
1479
|
+
}
|
|
1480
|
+
return null;
|
|
1481
|
+
};
|
|
1482
|
+
for (const n of pending.splice(0)) {
|
|
1483
|
+
const b = cyc.has(n) ? null : blocker(n);
|
|
1484
|
+
skip(n, cyc.has(n) ? "dependency cycle" : b !== null ? `blocked by #${b} (dependency cycle)` : "blocked: no dependency path is runnable");
|
|
1485
|
+
}
|
|
1377
1486
|
save();
|
|
1378
1487
|
}
|
|
1379
1488
|
}
|
|
@@ -1444,6 +1553,9 @@ export function formatOutcomeLists(rows) {
|
|
|
1444
1553
|
const one = (x) => String(x ?? "").split("\n")[0].slice(0, OUTCOME_LINE_MAX);
|
|
1445
1554
|
const requeued = (rows ?? []).filter((r) => r?.requeued); // #4550: operator --requeue after a decision; the row is the latest outcome
|
|
1446
1555
|
if (requeued.length) out.push(`requeued: ${requeued.map((r) => `#${r.issue} (${one(r.requeued.reason)})`).join(" ")}`);
|
|
1556
|
+
const heldBy = new Map(); // #5162: held subtrees by root, one --requeue command each
|
|
1557
|
+
for (const r of rows ?? []) if (r?.outcome === "held" && Number.isInteger(r.root)) heldBy.set(r.root, [...(heldBy.get(r.root) ?? []), r.issue]);
|
|
1558
|
+
for (const [root, xs] of heldBy) { const ro = (rows ?? []).find((r) => r?.issue === root)?.outcome ?? "failed"; out.push(`held subtree: #${root} (${ro}) holds ${xs.map((x) => `#${x}`).join(" ")}; release: forgedock orchestrate --requeue <batch> ${root}`); }
|
|
1447
1559
|
const rec = (rows ?? []).filter((r) => r?.recovered);
|
|
1448
1560
|
if (rec.length) out.push(`recovered: ${rec.map((r) => `#${r.issue} (${r.recovered})`).join(" ")}`);
|
|
1449
1561
|
for (const r of rows ?? []) {
|
package/bin/lean/prefilter.mjs
CHANGED
|
@@ -455,11 +455,12 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
|
|
|
455
455
|
return { ...out, detail: "acceptance probe failed after promotion (FORGE:PROBE): no already-fixed close; lane" };
|
|
456
456
|
}
|
|
457
457
|
const sig = await collectSignals({ iss, repo, lane, io, cwd, selfLogins });
|
|
458
|
-
|
|
458
|
+
// #5152: `completed` for a delivered outcome (CI green again, ALREADY-FIXED); only an INVALID verdict is dropped scope
|
|
459
|
+
const close = async (kind, body, detail, reason = "completed") => {
|
|
459
460
|
let note = "";
|
|
460
461
|
if (!dryRun) {
|
|
461
462
|
// #4315: close first: the marker is posted only on an issue that was closed, so it always means "reopened" later
|
|
462
|
-
await io.gh(["issue", "close", String(issue), "-R", repo, "--reason",
|
|
463
|
+
await io.gh(["issue", "close", String(issue), "-R", repo, "--reason", reason]);
|
|
463
464
|
// #4314: the issue IS closed from here on: the result is closed:true whatever the comment does, so no lane starts on it.
|
|
464
465
|
// The comment is retried once, then a marker-only comment (a second, tiny payload) keeps the #4305 reopen guard working.
|
|
465
466
|
const comment = (text) => io.gh(["issue", "comment", String(issue), "-R", repo, "--body", text]);
|
|
@@ -498,7 +499,7 @@ export async function prefilterIssue({ issue, repo, lane, cwd, io, model = DEFAU
|
|
|
498
499
|
if (!v || v.verdict === "CONFIRMED" || !v.evidence.length) return keep(`candidate (${why}) ${v ? (v.verdict === "CONFIRMED" ? "confirmed" : "without evidence") : "unparseable verdict"}: lane`);
|
|
499
500
|
const ev = v.evidence.map((x) => `- ${sanitizeMd(x).replace(/\s+/g, " ").slice(0, 400)}`).join("\n");
|
|
500
501
|
return await close("validated", `## Pre-filter: ${v.verdict}\n**Signal:** ${why}\n**Summary:** ${sanitizeMd(v.summary).replace(/\s+/g, " ").slice(0, 600) || "not reported"}\n\n**Evidence:**\n${ev}\n\nValidated by a single read-only check (${model}), without a lane. Reopen if this is wrong.`,
|
|
501
|
-
`${v.verdict.toLowerCase()} (${why})
|
|
502
|
+
`${v.verdict.toLowerCase()} (${why})`, v.verdict === "INVALID" ? "not planned" : "completed");
|
|
502
503
|
} catch (e) {
|
|
503
504
|
return { ...out, detail: `pre-filter error (${String(e?.message ?? e).slice(0, 120)}): lane` };
|
|
504
505
|
}
|
|
@@ -73,7 +73,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
73
73
|
const call = w.runner[0];
|
|
74
74
|
assert.equal(call.model, DEFAULT_PREFILTER_MODEL); assert.equal(call.readOnly, true); assert.equal(call.resume, null); assert.equal(call.cwd, "/repo");
|
|
75
75
|
assert.match(call.prompt, /merged PR #90/); assert.doesNotMatch(call.prompt, /PR #91/);
|
|
76
|
-
assert.deepEqual(w.closed, [[12, "
|
|
76
|
+
assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
|
|
77
77
|
assert.deepEqual(w.labels, [[12, "invalid"]]);
|
|
78
78
|
assert.match(w.comments[0][1], /FORGE:PREFILTER/); assert.match(w.comments[0][1], /src\/foo\.js:3 null guard/); assert.match(w.comments[0][1], /PR #90/);
|
|
79
79
|
});
|
|
@@ -120,7 +120,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
120
120
|
assert.equal(r.closed, true); assert.equal(r.kind, "ci-green");
|
|
121
121
|
assert.equal(w.runner.length, 0);
|
|
122
122
|
assert.match(w.comments[0][1], /actions\/runs\/600/); assert.match(w.comments[0][1], /actions\/runs\/500/);
|
|
123
|
-
assert.deepEqual(w.closed, [[34, "
|
|
123
|
+
assert.deepEqual(w.closed, [[34, "completed"]]); // #5152: CI green again is delivered
|
|
124
124
|
});
|
|
125
125
|
|
|
126
126
|
it("#4305 the CI-green shortcut is skipped for intermittent, reopened and same-SHA-flaky CI failures", async () => {
|
|
@@ -204,7 +204,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
204
204
|
assert.equal(r2.closed, true); assert.equal(tiny.comments.length, 1); assert.match(tiny.comments[0][1], /^<!-- FORGE:PREFILTER -->/);
|
|
205
205
|
assert.match(flakyReason({ comments: [{ body: tiny.comments[0][1], author: { login: "forgedock-app[bot]" }, authorAssociation: "NONE" }] }, ["forgedock-app"])?.reason ?? "", /reopened/);
|
|
206
206
|
const dead = flaky(99); const r3 = await pf(dead, 34);
|
|
207
|
-
assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "
|
|
207
|
+
assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "completed"]]); assert.match(r3.detail, /no marker comment/);
|
|
208
208
|
});
|
|
209
209
|
|
|
210
210
|
it("a CI-failure issue whose workflow is still red (or whose latest run is the failing one) goes to a lane", async () => {
|
|
@@ -224,6 +224,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
224
224
|
assert.deepEqual(namedPaths("fix lib/a.js <!-- x/hidden.js --!> visible/e.js and <!-- unclosed c/d.js"), ["lib/a.js", "visible/e.js"]);
|
|
225
225
|
const gone = world({ issues: { 5: { body: "crash in src/old/foo.js:12" } }, verdict: { verdict: "INVALID", evidence: ["src/old/foo.js removed in abc123"] } });
|
|
226
226
|
assert.equal((await pf(gone, 5)).closed, true);
|
|
227
|
+
assert.deepEqual(gone.closed, [[5, "not planned"]], "#5152: only an INVALID verdict closes not planned");
|
|
227
228
|
assert.match(gone.runner[0].prompt, /do not exist on origin\/staging: src\/old\/foo\.js/);
|
|
228
229
|
const kept = world({ issues: { 5: { body: "crash in src/old/foo.js:12 and src/new.js/x.ts" } }, paths: ["src/old/foo.js"] });
|
|
229
230
|
const r = await pf(kept, 5);
|
|
@@ -300,7 +301,7 @@ describe("orchestrate with the pre-filter (#4123)", () => {
|
|
|
300
301
|
assert.deepEqual(res.lanes, [13]);
|
|
301
302
|
assert.equal(logs.at(-1), "lanes: #13");
|
|
302
303
|
assert.ok(logs.some((s) => /^pre-filter: 2 checked, 1 candidate\(s\), 1 closed/.test(s)));
|
|
303
|
-
assert.deepEqual(w.closed, [[12, "
|
|
304
|
+
assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
|
|
304
305
|
await assert.rejects(triageFromCli([], {}), /usage/);
|
|
305
306
|
});
|
|
306
307
|
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -488,8 +488,10 @@ export function reviewerPrompt({ issue, comments = [], contract, result, files =
|
|
|
488
488
|
* #4160: the targeted re-ask after an incomplete or inconsistent reply. Only the missing or invalid ids are asked
|
|
489
489
|
* for, with the exact problems and the reviewer's previous reply; no diff is re-sent (the worktree still holds the change).
|
|
490
490
|
*/
|
|
491
|
-
export function reviewerRepairPrompt({ missing = [], problems = [], rows = null }) {
|
|
492
|
-
|
|
491
|
+
export function reviewerRepairPrompt({ missing = [], problems = [], rows = null, criteria = [] }) {
|
|
492
|
+
// #5164: a coverage misquote is re-asked with the issue's criterion lines verbatim, so the quote is copied, not retyped
|
|
493
|
+
const crit = (criteria ?? []).length ? `\n\n<issue_criteria>\nCopy every issue_coverage quote exactly (character for character, including #numbers and plurals) from one of these issue lines:\n${criteria.slice(0, 80).map((c) => `- ${String(c).slice(0, 400)}`).join("\n")}\n</issue_criteria>` : "";
|
|
494
|
+
return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>${crit}\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is a backtick-quoted repo-relative path:line copied verbatim (e.g. \`apps/web/app/(console)/layout.tsx:12\`); for an evidence path reported as not existing, give the full repo-relative path of that file from the changed files or the repository.`;
|
|
493
495
|
}
|
|
494
496
|
|
|
495
497
|
/** Pre-filter (#4123): one read-only, single-turn check of an issue a cheap signal flagged as possibly already fixed. */
|