forgedock 1.32.1 → 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +7 -3
- package/bin/engine/phases.mjs +3 -3
- package/bin/engine-cli.mjs +56 -11
- package/bin/forgedock.mjs +21 -1
- package/bin/lean/basesync.mjs +98 -0
- package/bin/lean/basesync.test.mjs +151 -0
- package/bin/lean/batch.test.mjs +103 -2
- package/bin/lean/budget.mjs +7 -1
- package/bin/lean/budget.test.mjs +9 -1
- package/bin/lean/conformance.mjs +20 -7
- package/bin/lean/conformance.test.mjs +29 -2
- package/bin/lean/depcache.mjs +23 -3
- package/bin/lean/depcache.test.mjs +42 -1
- package/bin/lean/effects.mjs +1 -1
- package/bin/lean/execution.mjs +102 -12
- package/bin/lean/execution.test.mjs +101 -1
- package/bin/lean/hostkill.mjs +8 -0
- package/bin/lean/lean.mjs +49 -21
- package/bin/lean/lean.test.mjs +133 -7
- package/bin/lean/lightlane.mjs +38 -13
- package/bin/lean/lightlane.test.mjs +35 -1
- package/bin/lean/orchestrate.mjs +63 -17
- package/bin/lean/prompts.mjs +17 -7
- package/bin/lean/recovery.test.mjs +26 -0
- package/bin/lean/sandbox-provider.contract.test.mjs +74 -1
- package/bin/lean/sandbox-remote.mjs +40 -7
- package/bin/lean/seqlease.mjs +49 -2
- package/bin/lean/seqlease.test.mjs +14 -0
- package/bin/lean/sequence.mjs +54 -6
- package/bin/lean/sequence.test.mjs +83 -1
- package/bin/pool/agent.mjs +435 -0
- package/bin/pool/cli.mjs +110 -0
- package/bin/pool/docker.mjs +56 -0
- package/bin/pool/http.mjs +55 -0
- package/bin/pool/router.mjs +299 -0
- package/bin/pool/store.mjs +25 -0
- package/bin/pool/version.mjs +44 -0
- package/bin/runner.mjs +1 -0
- package/bin/stats/aggregate.mjs +6 -0
- package/bin/stats/index.mjs +14 -2
- package/bin/stats/local-source.mjs +1 -1
- package/bin/stats/pool.mjs +51 -0
- package/bin/stats/pool.test.mjs +39 -0
- package/bin/stats/query.mjs +2 -2
- package/bin/stats/stats.test.mjs +15 -1
- package/commands/orchestrate.md +3 -3
- package/docs/CONFIG.md +23 -9
- package/docs/OUTCOME-CODES.md +5 -3
- package/docs/STATS.md +7 -0
- package/package.json +2 -1
- package/pool/README.md +13 -0
- package/pool/image/Dockerfile +46 -0
- package/pool/image/claude-shim +8 -0
- package/pool/image/cs-cred +12 -0
- package/pool/image/entrypoint.sh +61 -0
- package/pool/image/gh-shim +6 -0
- package/pool/install/install-agent-linux.sh +53 -0
- package/pool/install/install-agent.ps1 +225 -0
- package/pool/install/install-agent.sh +101 -0
- package/pool/install/install-router.sh +67 -0
- package/pool/install/keepalive.ps1 +36 -0
- package/pool/install/uninstall-agent.ps1 +20 -0
- package/scripts/hotspots.sh +33 -21
- package/scripts/hotspots.test.sh +22 -3
package/bin/lean/lean.test.mjs
CHANGED
|
@@ -648,6 +648,13 @@ describe("core work-on", () => {
|
|
|
648
648
|
};
|
|
649
649
|
assert.equal(await code(world({ result: { status: "blocked", summary: "need creds", block: { kind: "human", why: "credentials" } } })), "builder.blocked");
|
|
650
650
|
assert.equal(await code(world({ result: { status: "blocked", summary: "need creds" } })), "builder.block_untyped");
|
|
651
|
+
// #4542: a remote lane that names only a declared host capability is the recoverable env code; anything else stays human.
|
|
652
|
+
const rb = (summary, extra) => code(world({ result: { status: "blocked", summary, block: { kind: "human", why: "credentials" } } }), extra);
|
|
653
|
+
const remote = { placement: { where: "remote", reason: "mode remote" }, hostCaps: { hostCapabilities: ["sops"] } };
|
|
654
|
+
assert.equal(await rb("this lane has no sops/age key", remote), "env.capability_remote");
|
|
655
|
+
assert.equal(await rb("owner must create a third-party account", remote), "builder.blocked");
|
|
656
|
+
assert.equal(await rb("this lane has no sops/age key", { ...remote, hostCaps: undefined }), "builder.blocked");
|
|
657
|
+
assert.equal(await rb("this lane has no sops/age key", { ...remote, placement: { where: "local", reason: "x" } }), "builder.blocked");
|
|
651
658
|
assert.equal(await code(world({ changed: [] })), "builder.no_commit");
|
|
652
659
|
assert.equal(await code(world({ waits: [{ rc: 1, text: "lint failed" }] }), { mergeWait: "all" }), "ci.red");
|
|
653
660
|
});
|
|
@@ -1616,6 +1623,20 @@ describe("hotspot prevention (forge#4195): plan facts, build rule, unexempted gr
|
|
|
1616
1623
|
assert.ok(approxTokens(builderImplementPrompt() + hotspotBuildRule(facts([]))) <= 1500);
|
|
1617
1624
|
});
|
|
1618
1625
|
|
|
1626
|
+
it("#4519 generated_paths: both prompts name the globs (even with the gate off), and an amendment adding one blocks", async () => {
|
|
1627
|
+
const G = ["web/public/openapi*.json"];
|
|
1628
|
+
for (const f of [facts([], { generated_paths: G }), facts([HOT], { enabled: false, generated_paths: G })]) {
|
|
1629
|
+
assert.match(hotspotPlanNote(f), /Generated paths \(forge\.yaml maintenance\.generated_paths, owned by the repo's generator\): web\/public\/openapi\*\.json\. Never list or amend them in/);
|
|
1630
|
+
assert.match(hotspotBuildRule(f), /Generated paths .*web\/public\/openapi\*\.json/);
|
|
1631
|
+
}
|
|
1632
|
+
assert.equal(hotspotPlanNote(facts([], { generated_paths: [] })), "", "no generated_paths: unchanged");
|
|
1633
|
+
const w = hworld(facts([], { generated_paths: G }), [facts([])], { amendments: [{ kind: "add-deliverable", path: "web/public/openapi.json", why: "sync the spec" }] });
|
|
1634
|
+
const r = await run(w);
|
|
1635
|
+
assert.equal(r.code, "conformance.unimplemented"); assert.equal(w.merges.length, 0);
|
|
1636
|
+
assert.match(builders(w)[0][4], /Generated paths .*openapi/);
|
|
1637
|
+
assert.match(nh(w), /add-deliverable web\/public\/openapi\.json: generated path/);
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1619
1640
|
it("the plan prompt carries the facts, the implement prompt the configured rule; growth within the limit merges; no extra runner calls", async () => {
|
|
1620
1641
|
const w = hworld(facts([HOT, COLD], { file_lines: 2000, growth_lines: 150 }), [facts([SMALL])]);
|
|
1621
1642
|
assert.equal((await run(w)).terminalReason, "merged");
|
|
@@ -3490,6 +3511,28 @@ describe("batch robustness (#4112-#4115)", () => {
|
|
|
3490
3511
|
}
|
|
3491
3512
|
});
|
|
3492
3513
|
|
|
3514
|
+
it("#4518 a builder paused for longer than maxMs still gets its full ceiling of active time", async () => {
|
|
3515
|
+
const { runCliBackendAsync } = await import("../runner.mjs");
|
|
3516
|
+
const { d, bin } = await fakeCli("paused", `${TOOL}\nsleep 1\n${RESULT}`);
|
|
3517
|
+
const control = {};
|
|
3518
|
+
const call = runCliBackendAsync({ spec: { name: "t" }, userMessage: "go", args: ["1"], cwd: d, bin, logger: { log() {} }, stream: true, idleMs: 0, maxMs: 1200, control });
|
|
3519
|
+
await new Promise((res) => setTimeout(res, 100)); control.pause();
|
|
3520
|
+
await new Promise((res) => setTimeout(res, 2200)); control.resume();
|
|
3521
|
+
assert.equal((await call).text, "all done");
|
|
3522
|
+
});
|
|
3523
|
+
|
|
3524
|
+
it("#4518 external SIGTERM (status 143) restarts the builder once, then ends deferred cli.host_killed, never engine-error", async () => {
|
|
3525
|
+
const kill = () => Object.assign(new Error("claude CLI exited with status 143. No output was captured"), { code: "CLI_BACKEND_FAILED", status: 143, signal: null });
|
|
3526
|
+
const w = world({ dirty: true }); const orig = w.io.runner; let n = 0;
|
|
3527
|
+
w.io.runner = async (c) => { if (c.role === "builder" && n++ === 1) throw kill(); return orig(c); };
|
|
3528
|
+
assert.equal((await run(w)).terminalReason, "merged");
|
|
3529
|
+
const w2 = world({ dirty: true });
|
|
3530
|
+
w2.io.runner = async (c) => { if (c.role === "builder") throw kill(); return { text: "" }; };
|
|
3531
|
+
const r = await run(w2);
|
|
3532
|
+
assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "cli.host_killed");
|
|
3533
|
+
assert.equal(w2.edits.some((e) => e.includes("needs-human")), false);
|
|
3534
|
+
});
|
|
3535
|
+
|
|
3493
3536
|
// ---- #4405 ----
|
|
3494
3537
|
const HEAVY_CHILD = (d) => `printf '#!/bin/sh\\nsleep 1.5\\n' > ${d}/vitest; chmod +x ${d}/vitest`;
|
|
3495
3538
|
it("#4405 a silent session with a running heavy-step child is not idle-killed within the extended window", async () => {
|
|
@@ -3873,6 +3916,17 @@ describe("verify-by-CI (#4167): ci mode runs the heavy checks on CI and feeds fa
|
|
|
3873
3916
|
assert.equal(r.terminalReason, "needs-human"); assert.equal(w.merges.length, 0);
|
|
3874
3917
|
});
|
|
3875
3918
|
|
|
3919
|
+
it("#4527 base-sync re-review FAIL: the fix round's head is CI-verified before the retry review, which accepts validates left to CI and notes them", async () => {
|
|
3920
|
+
const CONFLICTING = { mergeable: "CONFLICTING", mergeStateStatus: "DIRTY" }, OKSTATE = { mergeable: "MERGEABLE", mergeStateStatus: "CLEAN" };
|
|
3921
|
+
const w = withCi(world({ validation: ciValidation, prView: [CONFLICTING, OKSTATE], mergeResults: ["conflict"], reviews: [ok(), bad("src/foo.js:4 lost base change"), ok()] }));
|
|
3922
|
+
const r = await run(w, { verify: CI });
|
|
3923
|
+
assert.equal(r.terminalReason, "merged", r.reason);
|
|
3924
|
+
assert.equal(reviewers(w).length, 3);
|
|
3925
|
+
assert.equal(new Set(w.ci.runs.map((x) => x.headSha)).size, w.ci.runs.length, "every reviewed head was verified once");
|
|
3926
|
+
assert.equal(w.ci.runs[0].headSha, w.head, "the retry head is CI-verified");
|
|
3927
|
+
assert.match(traj(w)[0], /Validate V1 verified by CI at [0-9a-f]{0,12}/);
|
|
3928
|
+
});
|
|
3929
|
+
|
|
3876
3930
|
it("docs-only lane: no push for CI, no dispatch; the decision is recorded", async () => {
|
|
3877
3931
|
const w = withCi(world({ changed: ["README.md", "docs/guide.md"], validation: ciValidation }));
|
|
3878
3932
|
const r = await run(w, { verify: CI, conformance: "shadow" });
|
|
@@ -5106,6 +5160,20 @@ describe("enumerating tests + sequence numbers at hand-off (#4322)", () => {
|
|
|
5106
5160
|
assert.match(traj(w).at(-1), /renumbered deterministically \(no builder\): db\/0007_usage\.sql → db\/0008_usage\.sql/);
|
|
5107
5161
|
});
|
|
5108
5162
|
|
|
5163
|
+
it("#4526: the renumber gets a reserve hook bound to (repo, lane, issue), and the run's reservations are dropped at finish", async () => {
|
|
5164
|
+
const w = dworld({ seq: [COLL, []] }), reserved = [], unreserved = [];
|
|
5165
|
+
w.io.sequenceLease.reserve = async (x) => { reserved.push(x); return { numbers: [9] }; };
|
|
5166
|
+
w.io.sequenceLease.unreserve = async (x) => { unreserved.push(x); };
|
|
5167
|
+
assert.equal((await run(w)).terminalReason, "merged");
|
|
5168
|
+
assert.equal(typeof w.renumbers[0].reserve, "function");
|
|
5169
|
+
assert.deepEqual(await w.renumbers[0].reserve({ dir: "db", floor: 7, count: 1 }), { numbers: [9] });
|
|
5170
|
+
assert.equal(reserved[0].dir, "db");
|
|
5171
|
+
assert.equal(reserved[0].floor, 7);
|
|
5172
|
+
assert.ok(reserved[0].repo && reserved[0].lane && reserved[0].issue, JSON.stringify(reserved[0]));
|
|
5173
|
+
assert.equal(unreserved.length, 1);
|
|
5174
|
+
assert.deepEqual(unreserved[0], { repo: reserved[0].repo, lane: reserved[0].lane, issue: reserved[0].issue });
|
|
5175
|
+
});
|
|
5176
|
+
|
|
5109
5177
|
it("base-sync with conflicts only in sequence files: base side taken and renumbered by the engine, zero builder sessions, zero re-reviews", async () => {
|
|
5110
5178
|
const w = dworld({ seq: [[], COLL, []], prView: [CONFLICTING, OKSTATE], mergeResults: ["conflict"], conflictFiles: ["db/meta/_journal.json", "db/meta/0007_snapshot.json"] });
|
|
5111
5179
|
assert.equal((await run(w)).terminalReason, "merged");
|
|
@@ -5400,10 +5468,10 @@ describe("#4392: implement turn budget", () => {
|
|
|
5400
5468
|
const TB = { S: 25, M: 60, L: 100 }; // the CLI defaults (readCoreLimits); runCore alone is unbudgeted
|
|
5401
5469
|
const stopWith = (digest) => (w) => { w.files[`${WT}/.forge-lean/result.json`] = JSON.stringify({ status: "budget-exceeded", summary: "too big", digest }); };
|
|
5402
5470
|
|
|
5403
|
-
it("an overrun ends
|
|
5471
|
+
it("an overrun ends budget-exceeded (#4505) with the builder's digest, checkpointed, recoverable once", async () => {
|
|
5404
5472
|
const w = budgetWorld(stopWith("done: parser. remaining: 3 adapters. recommend split")); const dir = tmpDir("core-tb-");
|
|
5405
5473
|
const r = await run(w, { dir, turnBudgets: TB });
|
|
5406
|
-
assert.equal(r.terminalReason, "
|
|
5474
|
+
assert.equal(r.terminalReason, "budget-exceeded"); assert.equal(r.code, OUTCOME_CODES.BUDGET_EXCEEDED);
|
|
5407
5475
|
assert.match(r.detail, /size S, 25 turns/); assert.match(r.detail, /remaining: 3 adapters/);
|
|
5408
5476
|
assert.equal(classifyNeedsHuman({ outcome: r.terminalReason, code: r.code }), "recoverable");
|
|
5409
5477
|
assert.deepEqual(r.turns, { size: "S", budget: 25, build: 28, total: 31, exceeded: true });
|
|
@@ -5411,7 +5479,7 @@ describe("#4392: implement turn budget", () => {
|
|
|
5411
5479
|
assert.equal(w.seen[2].resume, null, "the wrap-up is a fresh session");
|
|
5412
5480
|
assert.match(w.seen[2].prompt, /The engine froze \.forge-lean\/contract\.json/); assert.match(w.seen[2].prompt, /at most 5 turns/);
|
|
5413
5481
|
assert.ok(w.calls.some((c) => c[0] === "git" && c[1] === "commit" && c.some((x) => /checkpoint after turn budget/.test(x))), "work checkpointed");
|
|
5414
|
-
assert.ok(w.comments.some((c) => c.includes("FORGE:NEEDS_HUMAN")
|
|
5482
|
+
assert.ok(!w.comments.some((c) => c.includes("FORGE:NEEDS_HUMAN")), "#4505: no human escalation");
|
|
5415
5483
|
const log = readLog(dir, 7);
|
|
5416
5484
|
assert.ok(log.some((e) => e.event === "TURN_BUDGET_EXCEEDED" && e.size === "S" && e.budget === 25 && e.turns === 25 && e.grant === 5));
|
|
5417
5485
|
assert.deepEqual(log.find((e) => e.event === "RUN_TERMINAL").turns, r.turns);
|
|
@@ -5427,6 +5495,35 @@ describe("#4392: implement turn budget", () => {
|
|
|
5427
5495
|
assert.equal(o2.code, OUTCOME_CODES.BUDGET_EXCEEDED); assert.match(o2.detail, /No builder digest; changes so far/);
|
|
5428
5496
|
});
|
|
5429
5497
|
|
|
5498
|
+
const addNH = (w) => w.edits.filter((e) => e.join(" ") === "issue edit 7 -R o/r --add-label needs-human").length;
|
|
5499
|
+
it("#4505 a turn overrun ends budget-exceeded, not needs-human: no label, no NEEDS_HUMAN comment, recoverable", async () => {
|
|
5500
|
+
const w = budgetWorld(stopWith("done: parser. remaining: tests")); const dir = tmpDir("core-tb-");
|
|
5501
|
+
const r = await run(w, { dir, turnBudgets: TB });
|
|
5502
|
+
assert.equal(r.terminalReason, "budget-exceeded"); assert.equal(r.code, OUTCOME_CODES.BUDGET_EXCEEDED);
|
|
5503
|
+
assert.equal(r.branch, "feat/issue-7-lean", "the checkpoint branch rides on the result"); assert.match(r.detail, /remaining: tests/);
|
|
5504
|
+
assert.equal(addNH(w), 0, "no needs-human label"); assert.ok(!w.comments.some((c) => c.includes("FORGE:NEEDS_HUMAN")), "no Needs human comment");
|
|
5505
|
+
assert.equal(classifyNeedsHuman({ outcome: r.terminalReason, code: r.code }), "recoverable");
|
|
5506
|
+
const term = readLog(dir, 7).find((e) => e.event === "RUN_TERMINAL");
|
|
5507
|
+
assert.equal(term.reason, "budget-exceeded"); assert.equal(term.code, OUTCOME_CODES.BUDGET_EXCEEDED);
|
|
5508
|
+
assert.ok(w.comments.some((c) => c.includes("FORGE:TRAJECTORY") && /recoverable budget\.exceeded/.test(c)), "the trajectory records the recoverable end");
|
|
5509
|
+
});
|
|
5510
|
+
|
|
5511
|
+
it("#4505 a budget.exceeded recovery lane runs at the next size budget; non-final stays unlabelled, a final overrun is needs-human", async () => {
|
|
5512
|
+
const nonFinal = budgetWorld(stopWith("still more"));
|
|
5513
|
+
const r1 = await run(nonFinal, { dir: tmpDir("core-tb-"), turnBudgets: TB, recover: { code: OUTCOME_CODES.BUDGET_EXCEEDED, attempt: 1, final: false } });
|
|
5514
|
+
assert.deepEqual(nonFinal.seen.map((x) => x.turnMax), [0, 60, 12], "S lane retried at M: 60 turns, wrap-up grant 12");
|
|
5515
|
+
assert.equal(r1.turns.size, "M"); assert.equal(r1.terminalReason, "budget-exceeded"); assert.equal(r1.recovery, true);
|
|
5516
|
+
assert.equal(addNH(nonFinal), 0);
|
|
5517
|
+
assert.ok(builders(nonFinal).some((b) => /Recovery pass \(budget\.exceeded\): .*Continue from the checkpoint/.test(b[4])));
|
|
5518
|
+
const fin = budgetWorld(stopWith("still more"));
|
|
5519
|
+
const r2 = await run(fin, { dir: tmpDir("core-tb-"), turnBudgets: TB, recover: { code: OUTCOME_CODES.BUDGET_EXCEEDED, attempt: 1, final: true } });
|
|
5520
|
+
assert.equal(r2.terminalReason, "needs-human"); assert.equal(r2.code, OUTCOME_CODES.BUDGET_EXCEEDED);
|
|
5521
|
+
assert.equal(addNH(fin), 1, "the failed recovery is a human escalation"); assert.ok(fin.comments.some((c) => c.includes("FORGE:NEEDS_HUMAN")));
|
|
5522
|
+
const other = budgetWorld(stopWith("still more")); // another code's recovery keeps the lane's size
|
|
5523
|
+
const r3 = await run(other, { dir: tmpDir("core-tb-"), turnBudgets: TB, recover: { code: "ci.red", attempt: 1, final: false } });
|
|
5524
|
+
assert.equal(r3.turns.size, "S"); assert.equal(r3.terminalReason, "budget-exceeded");
|
|
5525
|
+
});
|
|
5526
|
+
|
|
5430
5527
|
it("size comes from a size: label over the contract; 0 disables the budget and nothing is capped", async () => {
|
|
5431
5528
|
const w = world({ labels: [{ name: "size:L" }] }); const caps = []; const orig = w.io.runner;
|
|
5432
5529
|
w.io.runner = async (c) => { if (c.role === "builder") caps.push(c.turnMax ?? 0); return orig(c); };
|
|
@@ -5981,6 +6078,36 @@ describe("#4450 light lane: one builder session, derived contract, upgrade to th
|
|
|
5981
6078
|
assert.equal(typeof events(dir, "LIGHT_DONE")[0].cost_usd, "number");
|
|
5982
6079
|
});
|
|
5983
6080
|
|
|
6081
|
+
it("#4502 a light lane that adds an undeclared package edge gets the structure gate: one builder resume, then merges", async () => {
|
|
6082
|
+
const EDGE = { from: "@s/db", to: "@s/core", kind: "dependencies", manifest: "packages/db/package.json" };
|
|
6083
|
+
const quiet = { package_edges_added: [], package_edges_removed: [], import_pairs: [], boundary_config_changed: { changed: false, files: [], loosens: false, tightens: false } };
|
|
6084
|
+
const w = lworld(), dir = tmpDir("light-struct-"); const q = [{ ...quiet, package_edges_added: [EDGE] }, quiet];
|
|
6085
|
+
w.io.structureDelta = async () => (q.length > 1 ? q.shift() : q[0]);
|
|
6086
|
+
const r = await run(w, { dir, ...ON });
|
|
6087
|
+
assert.equal(r.terminalReason, "merged");
|
|
6088
|
+
assert.equal(events(dir, "LIGHT_DONE").length, 1, "stayed light");
|
|
6089
|
+
assert.equal(builders(w).length, 2, "the light session plus one structure resume");
|
|
6090
|
+
assert.match(builders(w)[1][4], /@s\/db -> @s\/core/);
|
|
6091
|
+
assert.equal(events(dir, "STRUCTURE_DELTA").length >= 1, true);
|
|
6092
|
+
});
|
|
6093
|
+
|
|
6094
|
+
it("#4502 a typed budget error in the light step ends as in the full lane: no second builder session, no upgrade", async () => {
|
|
6095
|
+
const w = lworld(), dir = tmpDir("light-typed-"); const orig = w.io.runner;
|
|
6096
|
+
let tries = 0;
|
|
6097
|
+
w.io.runner = async (c) => { if (c.role === "builder") { tries++; throw Object.assign(new Error("turn budget spent"), { laneCode: OUTCOME_CODES.BUDGET_EXCEEDED }); } return orig(c); };
|
|
6098
|
+
const r = await run(w, { dir, ...ON });
|
|
6099
|
+
assert.equal(r.code, OUTCOME_CODES.BUDGET_EXCEEDED);
|
|
6100
|
+
assert.equal(tries, 1, "no upgrade session");
|
|
6101
|
+
assert.equal(events(dir, "LIGHT_UPGRADE").length, 0);
|
|
6102
|
+
});
|
|
6103
|
+
|
|
6104
|
+
it("#4502 the targeted light tests run with the configured precheck timeout", async () => {
|
|
6105
|
+
const w = lworld(), dir = tmpDir("light-timeout-"); const seen = [];
|
|
6106
|
+
w.io.runCommand = async (a) => { seen.push(a.timeoutMs); return { rc: 0, output: "" }; };
|
|
6107
|
+
await run(w, { dir, ...ON });
|
|
6108
|
+
assert.ok(seen.length && seen.every((t) => Number.isFinite(t) && t > 0 && t < 2 ** 40), `timeouts: ${seen}`);
|
|
6109
|
+
});
|
|
6110
|
+
|
|
5984
6111
|
for (const [name, code, o] of [
|
|
5985
6112
|
["oversized diff", "diff_lines", { numstat: "80\t3\tsrc/foo.js\n20\t0\ttest/foo.test.js\n" }],
|
|
5986
6113
|
["non-red repro", "repro_not_red", { repro: { base: "PASSED", head: "RAN" } }],
|
|
@@ -6039,15 +6166,14 @@ describe("#4450 light lane: one builder session, derived contract, upgrade to th
|
|
|
6039
6166
|
w.io.structureDelta = async () => { w.over.structure.push(w.calls.length); return { package_edges_added: [], package_edges_removed: [], import_pairs: [] }; };
|
|
6040
6167
|
return w;
|
|
6041
6168
|
};
|
|
6042
|
-
it("#4451 a light lane that is not upgraded makes no context-pack
|
|
6169
|
+
it("#4451/#4502 a light lane that is not upgraded makes no context-pack call but runs the structure and hotspot gates", async () => {
|
|
6043
6170
|
const w = overhead(), dir = tmpDir("light-over-");
|
|
6044
6171
|
const r = await run(w, { dir, ...ON, contextPack: { mode: "on" } });
|
|
6045
6172
|
assert.equal(r.terminalReason, "merged");
|
|
6046
6173
|
assert.equal(events(dir, "LIGHT_DONE").length, 1);
|
|
6047
|
-
assert.deepEqual(w.over
|
|
6174
|
+
assert.deepEqual(w.over.pack, []);
|
|
6175
|
+
assert.ok(w.over.hot.length >= 1 && w.over.structure.length >= 1, "#4502: the full lane's gates run");
|
|
6048
6176
|
assert.equal(events(dir, "CONTEXT_PACK").length, 0);
|
|
6049
|
-
// only the merge-time record (local numstat, no structure-delta call) remains; no implement-time delta
|
|
6050
|
-
assert.deepEqual(events(dir, "STRUCTURE_DELTA").map((e) => e.phase), ["merge"]);
|
|
6051
6177
|
assert.equal(events(dir, "RUN_TERMINAL")[0].light, true);
|
|
6052
6178
|
});
|
|
6053
6179
|
it("#4451 an upgraded light lane computes the hotspots and the context pack before the plan call and runs the structure delta", async () => {
|
package/bin/lean/lightlane.mjs
CHANGED
|
@@ -23,14 +23,14 @@ const one = (s, max = DETAIL_MAX) => {
|
|
|
23
23
|
|
|
24
24
|
/** The one builder session of a light lane: failing test, fix, targeted tests, result.json with what the contract is derived from. */
|
|
25
25
|
export function lightBuildPrompt(n, opts = {}) {
|
|
26
|
-
return `Resolve issue #${n} (read it with gh issue view ${n} --comments) in ONE session; this is a light lane for a small, well-specified issue. There is no plan step and no frozen contract: the engine derives the contract from
|
|
26
|
+
return `Resolve issue #${n} (read it with gh issue view ${n} --comments) in ONE session; this is a light lane for a small, well-specified issue. There is no plan step and no frozen contract: the engine derives the contract from the files the issue names and your test.
|
|
27
27
|
|
|
28
28
|
${hostRules(opts)}
|
|
29
29
|
|
|
30
30
|
1. Write the failing regression test first and run it: it must fail on the current code for the reason the issue describes.
|
|
31
31
|
2. Make the smallest fix that makes it pass. Keep the public API shape unchanged (signatures, sync/async, generator/function, exported names); the engine checks it.
|
|
32
32
|
3. Run the targeted tests (the new test plus the existing tests of the files you changed), run the repo's formatter on the files you changed, and commit. Do not push or touch GitHub.
|
|
33
|
-
4.
|
|
33
|
+
4. Re-run the targeted tests once more on the committed tree and make sure \`git status\` is clean (the engine runs its own gates afterwards; there is no contract file to check yet).
|
|
34
34
|
|
|
35
35
|
When done and committed, write ${IMPLEMENTATION_FILE} (markdown, no HTML comments):
|
|
36
36
|
### Approach
|
|
@@ -79,19 +79,22 @@ export function apiShapeReasons(r) {
|
|
|
79
79
|
* issue sentence, else the issue title; deliverables: the changed non-test files; validate: the repro command plus each
|
|
80
80
|
* targeted test command; repro_test: the builder's, else "none" with a reason. validateContract judges it like any other.
|
|
81
81
|
*/
|
|
82
|
-
export function deriveLightContract({ title = "", issueText = "", result = null, files = [] } = {}) {
|
|
82
|
+
export function deriveLightContract({ title = "", issueText = "", result = null, files = [], predictedFiles = null } = {}) {
|
|
83
83
|
const r = result && typeof result === "object" ? result : {};
|
|
84
84
|
const said = typeof r.requirement === "string" ? r.requirement.trim() : "";
|
|
85
85
|
const quote = said && inIssue(said, issueText) ? said : String(title ?? "").trim();
|
|
86
86
|
const paths = [...new Set((Array.isArray(files) ? files : []).map((f) => String(f ?? "").trim()).filter((f) => f && f !== LEAN_DIR && !f.startsWith(`${LEAN_DIR}/`)))];
|
|
87
|
-
const
|
|
87
|
+
const changed = paths.filter((f) => classifyPath(f) !== "test");
|
|
88
|
+
// #4502: deliverables are what the issue names (the diff cannot fail its own scope check); no named files: the diff, as before
|
|
89
|
+
const named = Array.isArray(predictedFiles) ? new Set(predictedFiles.map((f) => String(f ?? "").trim())) : null;
|
|
90
|
+
const code = named ? changed.filter((f) => named.has(f)) : changed;
|
|
88
91
|
const t = r.repro_test;
|
|
89
92
|
const repro = t && typeof t === "object" && !Array.isArray(t) && String(t.path ?? "").trim() && String(t.command ?? "").trim()
|
|
90
93
|
? { path: String(t.path).trim(), id: String(t.id ?? "").trim(), command: String(t.command).trim() } : null;
|
|
91
94
|
const commands = [...new Set([repro?.command, ...(Array.isArray(r.tests) ? r.tests : []).map((x) => (x && typeof x === "object" ? String(x.command ?? "").trim() : ""))].filter(Boolean))];
|
|
92
95
|
return {
|
|
93
96
|
requirements: [{ id: "R1", quote, disposition: "deliver" }],
|
|
94
|
-
deliverables: code.map((path, i) => ({ id: `D${i + 1}`, path, change: "light-lane fix (
|
|
97
|
+
deliverables: code.map((path, i) => ({ id: `D${i + 1}`, path, change: "light-lane fix (issue-named file)", requirements: ["R1"] })),
|
|
95
98
|
validate: commands.map((command, i) => ({ id: `V${i + 1}`, command })),
|
|
96
99
|
...(repro ? { repro_test: repro } : { repro_test: "none", repro_test_reason: "the light-lane builder named no repro test" }),
|
|
97
100
|
};
|
|
@@ -109,14 +112,28 @@ export function upgradeNote(reasons) {
|
|
|
109
112
|
/**
|
|
110
113
|
* #4451: the light targeted tests, concurrently through precheck.mjs runChecks (one heavy slot per running command, the
|
|
111
114
|
* worker cap `parallelism` divided among them) with no round budget: every command runs, none is deferred to CI.
|
|
112
|
-
* run({command, workers}) -> {rc, output}. Returns [{command, rc, output}] in input order.
|
|
115
|
+
* run({command, timeoutMs, workers}) -> {rc, output}. Returns [{command, rc, output}] in input order.
|
|
113
116
|
*/
|
|
114
|
-
export async function runLightTests({ commands, parallelism = 1, run, acquire, now = () => Date.now() }) {
|
|
115
|
-
// the per-command
|
|
116
|
-
|
|
117
|
+
export async function runLightTests({ commands, parallelism = 1, run, acquire, now = () => Date.now(), timeoutMs }) {
|
|
118
|
+
// #4502: the configured per-command timeout reaches the runner (else io.runCommand's 300 s default applies); the budget stays far above it,
|
|
119
|
+
// so runChecks never sees a budget cut: a timeout (rc 124) stays a failure
|
|
120
|
+
const per = Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : 2 ** 40;
|
|
121
|
+
const { results } = await runChecks({ commands, parallelism, budgetMs: Number.MAX_SAFE_INTEGER, perCommandMs: per, now, acquire, run: ({ command, timeoutMs: t, workers }) => run({ command, timeoutMs: t, workers }) });
|
|
117
122
|
return results.map(({ command, rc, output }) => ({ command, rc, output }));
|
|
118
123
|
}
|
|
119
124
|
|
|
125
|
+
/** Non-test changed files (engine dir skipped) that the issue does not name. */
|
|
126
|
+
function scopeCreep(files, predicted) {
|
|
127
|
+
const named = new Set(predicted.map((f) => String(f ?? "").trim()));
|
|
128
|
+
return (files ?? []).map((f) => String(f ?? "").trim()).filter((f) => f && f !== LEAN_DIR && !f.startsWith(`${LEAN_DIR}/`) && classifyPath(f) !== "test" && !named.has(f));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** #4502: a typed builder error (budget, idle/timeout deferral, spawn failure, usage limit) belongs to the lane's own catch, not to an upgrade. */
|
|
132
|
+
export function isTypedBuilderError(e) {
|
|
133
|
+
if (!e || typeof e !== "object") return false;
|
|
134
|
+
return Boolean(e.laneCode || e.deferCode || e.spawnDeterministic === true || e.usageLimit || /^CLI_/.test(String(e.code ?? "")) || /usage limit/i.test(String(e.message ?? "")));
|
|
135
|
+
}
|
|
136
|
+
|
|
120
137
|
/**
|
|
121
138
|
* One light lane step. ctx callbacks (all async unless noted): build() runs the one builder session; readResult();
|
|
122
139
|
* checkpoint() commits uncommitted work; changedFiles(); numstat() -> `git diff --numstat` text; runTests(commands) ->
|
|
@@ -124,7 +141,7 @@ export async function runLightTests({ commands, parallelism = 1, run, acquire, n
|
|
|
124
141
|
* or null; apiShape() -> detectApiShape output or null; validate(contract) -> {ok, problems} (sync). Plus title, issueText,
|
|
125
142
|
* riskPaths, cfg. Returns {upgrade:false, contract, files, repro, testFailures} or {upgrade:true, reasons, files, ...}, plus the
|
|
126
143
|
* gate inputs of the FORGE:LITE receipt (#4452): diff (parseNumstat), apiShape (apiShapeReasons), testCommands, testRuns.
|
|
127
|
-
*
|
|
144
|
+
* An internal error is the upgrade reason light_error; a typed builder error (isTypedBuilderError) is rethrown.
|
|
128
145
|
*/
|
|
129
146
|
export async function runLightStep(ctx) {
|
|
130
147
|
const up = (reasons, extra = {}) => ({ upgrade: true, reasons, contract: null, files: [], repro: null, testFailures: 0, diff: null, apiShape: null, testCommands: [], testRuns: null, ...extra });
|
|
@@ -135,8 +152,13 @@ export async function runLightStep(ctx) {
|
|
|
135
152
|
const files = await ctx.changedFiles();
|
|
136
153
|
if (!files.length) return up([{ code: "no_commit", detail: `the light builder made no commit (status ${one(result?.status ?? "none", 40)})` }]);
|
|
137
154
|
if (result?.status !== "fixed") return up([{ code: "not_fixed", detail: `light builder status ${one(result?.status ?? "none", 40)}${result?.summary ? `: ${one(result.summary)}` : ""}` }], { files });
|
|
138
|
-
const
|
|
155
|
+
const predicted = Array.isArray(ctx.predictedFiles) ? ctx.predictedFiles : null;
|
|
156
|
+
const contract = deriveLightContract({ title: ctx.title, issueText: ctx.issueText, result, files, predictedFiles: predicted });
|
|
139
157
|
const vc = ctx.validate(contract);
|
|
158
|
+
if (!vc?.ok && predicted && scopeCreep(files, predicted).length) { // the issue-named deliverables are empty: the diff left the issue's scope
|
|
159
|
+
const t = upgradeTriggers({ diff: parseNumstat(await ctx.numstat()), repro: null, testFailures: 0, riskPaths: ctx.riskPaths ?? [], apiShape: [], cfg: ctx.cfg ?? {} });
|
|
160
|
+
return up([{ code: "scope_creep", detail: `changed file(s) the issue does not name: ${one(scopeCreep(files, predicted).slice(0, 5).join(", "))}` }, ...t.reasons], { files });
|
|
161
|
+
}
|
|
140
162
|
if (!vc?.ok) return up([{ code: "contract_invalid", detail: `derived contract: ${one((vc?.problems ?? [])[0] ?? "invalid")}` }], { files });
|
|
141
163
|
|
|
142
164
|
let testFailures = 0;
|
|
@@ -155,10 +177,13 @@ export async function runLightStep(ctx) {
|
|
|
155
177
|
const [repro, numstat, shape] = await Promise.all([ctx.reproOf(contract), ctx.numstat(), ctx.apiShape()]);
|
|
156
178
|
const diff = parseNumstat(numstat), apiShape = apiShapeReasons(shape);
|
|
157
179
|
const t = upgradeTriggers({ diff, repro, testFailures, riskPaths: ctx.riskPaths ?? [], apiShape, cfg: ctx.cfg ?? {} });
|
|
158
|
-
const
|
|
159
|
-
|
|
180
|
+
const outside = predicted ? scopeCreep(now, predicted) : [];
|
|
181
|
+
const reasons = outside.length ? [...t.reasons, { code: "scope_creep", detail: `changed file(s) the issue does not name: ${one(outside.slice(0, 5).join(", "))}` }] : t.reasons;
|
|
182
|
+
const latest = deriveLightContract({ title: ctx.title, issueText: ctx.issueText, result: (await ctx.readResult()) ?? result, files: now, predictedFiles: predicted });
|
|
183
|
+
return { upgrade: t.upgrade || outside.length > 0, reasons, contract: t.upgrade || outside.length ? null : (ctx.validate(latest)?.ok ? latest : contract), files: now, repro, testFailures,
|
|
160
184
|
diff, apiShape, testCommands: commands, testRuns: runs == null ? null : (runs ?? []).length };
|
|
161
185
|
} catch (e) {
|
|
186
|
+
if (isTypedBuilderError(e)) throw e; // #4502: the lane's own catch ends it as in the full lane, no second builder session
|
|
162
187
|
return up([{ code: "light_error", detail: one(e?.message ?? e) }]);
|
|
163
188
|
}
|
|
164
189
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
4
|
import assert from "node:assert/strict";
|
|
5
|
-
import { deriveLightContract, runLightStep, runLightTests, parseNumstat, apiShapeReasons, upgradeNote, lightBuildPrompt, renderLiteReceipt } from "./lightlane.mjs";
|
|
5
|
+
import { isTypedBuilderError, deriveLightContract, runLightStep, runLightTests, parseNumstat, apiShapeReasons, upgradeNote, lightBuildPrompt, renderLiteReceipt } from "./lightlane.mjs";
|
|
6
6
|
import { validateContract } from "./conformance.mjs";
|
|
7
7
|
|
|
8
8
|
const TITLE = "foo crashes on null input";
|
|
@@ -235,3 +235,37 @@ describe("renderLiteReceipt (#4452)", () => {
|
|
|
235
235
|
for (const k of ["upgrade", "reasons", "contract", "files", "repro", "testFailures"]) assert.ok(k in r, k);
|
|
236
236
|
});
|
|
237
237
|
});
|
|
238
|
+
|
|
239
|
+
describe("#4502 light lane fixes", () => {
|
|
240
|
+
it("runLightStep rethrows typed builder errors; internal errors stay light_error", async () => {
|
|
241
|
+
for (const e of [{ laneCode: "budget.exceeded" }, { deferCode: "cli.idle" }, { code: "CLI_SPAWN_FAILED" }, { code: "CLI_TIMEOUT" }, { spawnDeterministic: true }, { usageLimit: true }]) {
|
|
242
|
+
const { ctx } = step({ ctx: { build: async () => { throw Object.assign(new Error("typed"), e); } } });
|
|
243
|
+
await assert.rejects(runLightStep(ctx), (x) => isTypedBuilderError(x) && x.message === "typed");
|
|
244
|
+
}
|
|
245
|
+
const { ctx } = step({ ctx: { build: async () => { throw new Error("git broke"); } } });
|
|
246
|
+
assert.deepEqual(codes(await runLightStep(ctx)), ["light_error"]);
|
|
247
|
+
assert.equal(isTypedBuilderError(null), false);
|
|
248
|
+
});
|
|
249
|
+
it("the light prompt no longer asks for a check that cannot pass", () => {
|
|
250
|
+
const p = lightBuildPrompt(7);
|
|
251
|
+
assert.doesNotMatch(p, /forgedock[^\n]* check/);
|
|
252
|
+
assert.doesNotMatch(p, /\bcheck\b[^\n]*until it passes/);
|
|
253
|
+
assert.match(p, /git status/);
|
|
254
|
+
});
|
|
255
|
+
it("deliverables come from the issue-named files; a diff file outside them upgrades with scope_creep", () => {
|
|
256
|
+
const c = deriveLightContract({ title: TITLE, issueText: ISSUE, result: RESULT, files: ["src/foo.js", "src/other.js", "test/foo.test.js"], predictedFiles: ["src/foo.js"] });
|
|
257
|
+
assert.deepEqual(c.deliverables.map((d) => d.path), ["src/foo.js"]);
|
|
258
|
+
});
|
|
259
|
+
it("scope_creep: a changed non-test file the issue does not name is an upgrade reason", async () => {
|
|
260
|
+
const { ctx } = step({ files: ["src/foo.js", "src/other.js", "test/foo.test.js"], numstat: "5\t1\tsrc/foo.js\n2\t0\tsrc/other.js\n20\t0\ttest/foo.test.js\n", ctx: { predictedFiles: ["src/foo.js"] } });
|
|
261
|
+
const r = await runLightStep(ctx);
|
|
262
|
+
assert.equal(r.upgrade, true); assert.ok(codes(r).includes("scope_creep"), JSON.stringify(r.reasons));
|
|
263
|
+
const ok = await runLightStep(step({ ctx: { predictedFiles: ["src/foo.js"] } }).ctx);
|
|
264
|
+
assert.equal(ok.upgrade, false, JSON.stringify(ok.reasons));
|
|
265
|
+
});
|
|
266
|
+
it("runLightTests forwards the configured timeout to the runner", async () => {
|
|
267
|
+
const seen = [];
|
|
268
|
+
await runLightTests({ commands: ["a", "b"], timeoutMs: 900000, run: async (x) => { seen.push(x.timeoutMs); return { rc: 0, output: "" }; } });
|
|
269
|
+
assert.deepEqual(seen, [900000, 900000]);
|
|
270
|
+
});
|
|
271
|
+
});
|
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
* A seam calls `onLive()` before it waits on a live lane (#4504); an adopt that resolves first is settled (or requeued on null)
|
|
19
19
|
* before scheduling, so a live lane does not hold up the others.
|
|
20
20
|
* `deferred` is a scheduling fact, not an end: each typed code is requeued with its retry-at (`deferPolicy`), a usage
|
|
21
|
-
* limit holds admissions for that account until its reset, and
|
|
21
|
+
* limit holds admissions for that account until its reset, and deferrals other than a busy resource (`host.busy`,
|
|
22
|
+
* `sandbox.capacity`, #4528) stop after `maxDeferrals`.
|
|
22
23
|
* Usage-limit holds (#4190) are keyed by backend identity in `limits` (the host store, shared by every batch on the
|
|
23
24
|
* host; in memory when absent): admission checks the hold of `identity()`, the account the next lane would use. While held, the
|
|
24
25
|
* batch re-reads the store every `pollMs` (#4446), so `forgedock limits clear` admits on the next poll, not at the reset. An `engine-error` is
|
|
@@ -70,10 +71,12 @@
|
|
|
70
71
|
* In-batch recovery (#4267), on by default: this module recovers only when `recovery: {maxAttempts}` is passed, and the
|
|
71
72
|
* CLI always passes it (forge.yaml `orchestration.recovery.max_attempts`, default 1; 0 disables): a row `classifyNeedsHuman` calls
|
|
72
73
|
* recoverable (needs-human `ci.red` / `base.conflict` / `builder.block_untyped`, `contract.invalid` after a mid-run issue
|
|
73
|
-
* edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts
|
|
74
|
+
* edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts, the non-human `budget-exceeded` turn overrun of #4505,
|
|
75
|
+
* recovered at the next size budget) is not final while its persisted `recoveries` stay
|
|
74
76
|
* under `maxAttempts`: it is requeued as a recovery lane (`ctx.recover = {code, attempt, final}`, `final` when attempt ==
|
|
75
77
|
* `maxAttempts`) under the same admission as any lane, so its dependents stay queued. Only a final lane relabels needs-human
|
|
76
|
-
* itself; a non-final attempt that ends needs-human/ci-pending without a successor is relabelled here, once (#4302)
|
|
78
|
+
* itself; a non-final attempt that ends needs-human/ci-pending/budget-exceeded without a successor is relabelled here, once (#4302),
|
|
79
|
+
* as is a first-run budget-exceeded the batch does not recover (#4505: the lane never labels it). A recovery
|
|
77
80
|
* still queued when the batch exits (breaker, budget, hold, or a resume that does not relaunch it) is relabelled once with the
|
|
78
81
|
* full persisted lane reason (#4315); also when the loop throws, and through `onExit(fn)` (the caller's signal handler
|
|
79
82
|
* awaits `fn(cause)`), each once (#4318). `relabelNeedsHuman` may return `false`/"failed" (not recorded, a resume retries)
|
|
@@ -93,7 +96,7 @@
|
|
|
93
96
|
* the queue order except that decomposed sub-issues go first. A settled row carries `est_minutes` (estimated vs. actual).
|
|
94
97
|
*/
|
|
95
98
|
import { parseDependsOn } from "../engine/phases.mjs";
|
|
96
|
-
import { classifyNeedsHuman, recoveryCodeOf } from "../engine/outcome-codes.mjs";
|
|
99
|
+
import { classifyNeedsHuman, recoveryCodeOf, OUTCOME_CODES } from "../engine/outcome-codes.mjs";
|
|
97
100
|
import { recoveryEvidence } from "./render.mjs";
|
|
98
101
|
import { renderAlerts } from "../stats/render.mjs";
|
|
99
102
|
import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
|
|
@@ -103,11 +106,13 @@ import { laneSize, SIZES } from "./budget.mjs";
|
|
|
103
106
|
const SATISFIED = new Set(["merged", "invalid"]);
|
|
104
107
|
// Times one issue may be requeued behind a dependency before its dependency-open row is final.
|
|
105
108
|
export const MAX_REQUEUES = 2;
|
|
106
|
-
//
|
|
109
|
+
// Deferrals other than a busy resource (UNCAPPED_DEFERRALS) one issue may take before its deferred row is final.
|
|
107
110
|
export const DEFAULT_MAX_DEFERRALS = 3;
|
|
108
111
|
// Relaunches of an issue whose lane ended engine-error.
|
|
109
112
|
export const ENGINE_ERROR_RELAUNCHES = 1;
|
|
110
113
|
export const HOST_BUSY_RETRY_S = 60;
|
|
114
|
+
// #4528: a busy host or a full sandbox pool is a wait, not a failure: requeued after its retry-in, never capped.
|
|
115
|
+
const UNCAPPED_DEFERRALS = new Set(["host.busy", "sandbox.capacity"]);
|
|
111
116
|
export const RATE_LIMIT_FALLBACK_S = 15 * 60;
|
|
112
117
|
export const DEFER_BACKOFF_S = 120;
|
|
113
118
|
export const DEFER_BACKOFF_MAX_S = 30 * 60;
|
|
@@ -156,12 +161,12 @@ export function parseDeps(body, issue) {
|
|
|
156
161
|
const posNum = (v) => { const n = Number(v); return Number.isFinite(n) && n > 0 ? n : null; };
|
|
157
162
|
|
|
158
163
|
/**
|
|
159
|
-
* When to run a `deferred` issue again. `deferrals` counts this issue's earlier
|
|
164
|
+
* When to run a `deferred` issue again. `deferrals` counts this issue's earlier capped deferrals (not UNCAPPED_DEFERRALS).
|
|
160
165
|
* @returns {{delayMs:number, host?:true, hold?:true} | {capped:true}}
|
|
161
166
|
*/
|
|
162
167
|
export function deferPolicy(r, { now, deferrals = 0, maxDeferrals = DEFAULT_MAX_DEFERRALS }) {
|
|
163
168
|
const code = r?.code ?? null;
|
|
164
|
-
if (code
|
|
169
|
+
if (UNCAPPED_DEFERRALS.has(code)) return { delayMs: (posNum(r?.retryInSeconds) ?? HOST_BUSY_RETRY_S) * 1000, host: true };
|
|
165
170
|
const k = deferrals + 1;
|
|
166
171
|
if (k > maxDeferrals) return { capped: true };
|
|
167
172
|
if (code === "rate_limited") {
|
|
@@ -217,6 +222,8 @@ export function pickNext(ready, { classes, share, whenIdle = false, started = 0,
|
|
|
217
222
|
}
|
|
218
223
|
|
|
219
224
|
const toObj = (m) => Object.fromEntries([...m].map(([k, v]) => [String(k), v]));
|
|
225
|
+
/** #4525: how often a parked needs-human PR is re-synced with its base while the batch runs. */
|
|
226
|
+
export const DEFAULT_SYNC_EVERY_MS = 10 * 60 * 1000;
|
|
220
227
|
const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k), v]));
|
|
221
228
|
|
|
222
229
|
/**
|
|
@@ -226,7 +233,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
226
233
|
store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
227
234
|
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
228
235
|
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, stopLane = null,
|
|
229
|
-
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null }) {
|
|
236
|
+
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
|
|
230
237
|
const saved = store?.load?.() ?? null;
|
|
231
238
|
const order = saved ? [...saved.order] : [];
|
|
232
239
|
const set = new Set(order);
|
|
@@ -256,16 +263,21 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
256
263
|
const classes = toMap(saved?.classes); // issue -> {maintenance, priority}; only with a maintenance budget
|
|
257
264
|
const dispatch = { started: saved?.dispatch?.started ?? 0, maint: saved?.dispatch?.maint ?? 0 };
|
|
258
265
|
let budgetHeld = false;
|
|
266
|
+
const parkedBase = new Map(); // #4525: issue -> base sha its parked PR was last synced (or reported conflicting) against
|
|
267
|
+
let lastParkedSync = null;
|
|
259
268
|
const bodies = new Map(); // issue -> body, kept in memory for drift.holds only
|
|
260
269
|
const driftHeld = new Map(); // issue -> flagged package dir holding it this pass
|
|
261
270
|
const reg = sched?.registry ?? null;
|
|
262
271
|
const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
|
|
263
272
|
const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
|
|
273
|
+
let capAt, capAdmits = 0; // #4540: provider capacity sample and the remote admissions made since it
|
|
264
274
|
const remoteRun = new Set(); // remote lanes running now (#4253): admitted against remoteMax, never the host cap
|
|
265
275
|
let budgetLocal = false; // the execution budget refused a remote lane under on_budget local: the rest run on the host
|
|
266
276
|
// #4256: where a lane goes before capacity: `remote.place` (hybrid) or `remote.isRemote` (remote mode).
|
|
267
277
|
const placeOf = (n) => {
|
|
268
278
|
if (!remote) return null;
|
|
279
|
+
// #4542: a lane recovering from a remote capability block runs on the host, which has the capability.
|
|
280
|
+
if (recovering.get(n)?.code === OUTCOME_CODES.ENV_CAPABILITY_REMOTE) return { where: "local", reason: "host capability: recovering env.capability_remote" };
|
|
269
281
|
if (budgetLocal) return { where: "local", reason: `fallback to local: execution budget: ${budgetHit ?? "cap reached"}` };
|
|
270
282
|
try {
|
|
271
283
|
if (typeof remote.place === "function") {
|
|
@@ -441,9 +453,19 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
441
453
|
if (p?.where === "remote") {
|
|
442
454
|
const adm = admitRemote();
|
|
443
455
|
if (adm.ok) {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
456
|
+
// #4540: the provider's free slots (`remote.capacity() -> {free, at}`, sync, null = unknown) cap admissions per sample `at`; the issue queues here instead of a 503.
|
|
457
|
+
let cap = null;
|
|
458
|
+
try { cap = remote.capacity?.() ?? null; } catch { cap = null; }
|
|
459
|
+
// #4547: opt-in (`execution.fallback_local`): every pool agent offline or asleep (sampled online 0) runs the lane on the host.
|
|
460
|
+
if (remote.offlineFallback && cap && cap.online === 0) {
|
|
461
|
+
p = { where: "local", reason: `fallback to local: no pool agent online (${p.reason})` };
|
|
462
|
+
} else if (cap && Number.isInteger(cap.free)) {
|
|
463
|
+
if (cap.at !== capAt) { capAt = cap.at; capAdmits = 0; }
|
|
464
|
+
if (capAdmits >= cap.free && remoteRun.size < remoteMax) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
|
|
465
|
+
}
|
|
466
|
+
if (p.where === "remote" && remoteRun.size < remoteMax) { remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
|
|
467
|
+
if (p.where === "remote" && !remote.fallbackLocal) return false;
|
|
468
|
+
if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax} reached (${p.reason})` }; // #4256: hybrid runs it on the host
|
|
447
469
|
} else {
|
|
448
470
|
noteBudget(adm.reason);
|
|
449
471
|
if (remote.onBudget !== "local" && !remote.fallbackLocal) { budgetStop = adm.reason; return false; }
|
|
@@ -493,16 +515,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
493
515
|
if (r?.sandbox && typeof r.sandbox === "object") {
|
|
494
516
|
const prev = sandboxUse.get(n) ?? { minutes: 0, cost_usd: 0, attempts: 0 };
|
|
495
517
|
const m = Number(r.sandbox.minutes) >= 0 ? Number(r.sandbox.minutes) : 0, c = Number(r.sandbox.cost_usd) >= 0 ? Number(r.sandbox.cost_usd) : 0;
|
|
496
|
-
|
|
518
|
+
const by = { ...(prev.by_agent ?? {}) }; // #4549: minutes per pool agent, over attempts
|
|
519
|
+
if (typeof r.sandbox.agent === "string" && r.sandbox.agent) by[r.sandbox.agent] = Math.round(((by[r.sandbox.agent] ?? 0) + m) * 10) / 10;
|
|
520
|
+
sandboxUse.set(n, { minutes: Math.round((prev.minutes + m) * 10) / 10, cost_usd: Math.round((prev.cost_usd + c) * 1e6) / 1e6, attempts: prev.attempts + 1, ...(Object.keys(by).length ? { by_agent: by } : {}) });
|
|
497
521
|
try { remote?.onUsage?.(n, r.sandbox, rec); } catch { /* the batch total still counts it */ }
|
|
498
522
|
}
|
|
499
|
-
if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n) };
|
|
523
|
+
if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n), ...(sandboxUse.get(n).by_agent ? { by_agent: { ...sandboxUse.get(n).by_agent } } : {}) };
|
|
500
524
|
const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); remoteRun.delete(n); };
|
|
501
525
|
if (row.outcome === "dependency-open" && depList.length) {
|
|
502
526
|
// #4387: a parked dependency is not pulled in again (fail-open on an unreadable label).
|
|
503
527
|
let parked;
|
|
504
528
|
for (const d of depList) {
|
|
505
|
-
if (results.get(d)?.outcome
|
|
529
|
+
if (["needs-human", "budget-exceeded"].includes(results.get(d)?.outcome)) { parked = d; break; } // #4505: an unrecovered overrun parks too
|
|
506
530
|
if (set.has(d) || typeof issueLabels !== "function") continue;
|
|
507
531
|
let names = [];
|
|
508
532
|
try { names = (await issueLabels(d)) ?? []; } catch { names = []; }
|
|
@@ -587,7 +611,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
587
611
|
try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
|
|
588
612
|
}
|
|
589
613
|
}
|
|
590
|
-
if (recoveryMax &&
|
|
614
|
+
if (recoveryMax && ["needs-human", "ci-pending", "budget-exceeded"].includes(row.outcome)) {
|
|
591
615
|
let isConflicting = false;
|
|
592
616
|
if (row.outcome === "ci-pending" && typeof conflicting === "function" && Number.isInteger(row.pr)) {
|
|
593
617
|
try { isConflicting = (await conflicting(n, row.pr)) === true; } catch { isConflicting = false; }
|
|
@@ -607,9 +631,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
607
631
|
}
|
|
608
632
|
}
|
|
609
633
|
// #4302: a non-final recovery attempt left needs-human/ci-pending unlabelled for its successor; none is launched, so relabel once.
|
|
610
|
-
if (prior && prior.attempt < recoveryMax && ["needs-human", "ci-pending"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
|
|
634
|
+
if (prior && prior.attempt < recoveryMax && ["needs-human", "ci-pending", "budget-exceeded"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
|
|
611
635
|
try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
|
|
612
636
|
}
|
|
637
|
+
// #4505: a first-run turn overrun is unlabelled by its lane; no recovery follows (off or cap used), so the runner labels it once.
|
|
638
|
+
if (!prior && row.outcome === "budget-exceeded" && typeof relabelNeedsHuman === "function") {
|
|
639
|
+
try { await relabelNeedsHuman({ issue: n, evidence: `Implement turn budget overrun (\`budget.exceeded\`) not recovered in this batch:\n${String(row.detail ?? "")}` }); } catch { /* best effort */ }
|
|
640
|
+
}
|
|
613
641
|
results.set(n, row);
|
|
614
642
|
if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
|
|
615
643
|
save(); update(row);
|
|
@@ -851,6 +879,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
851
879
|
if (tripped && !running.size) { await drainAdds(); break; } // breaker: let running lanes finish, keep the rest pending for a resume
|
|
852
880
|
if (cancelled && !running.size) { await drainAdds(); break; } // #4376: cancel: the rest end held for a resume
|
|
853
881
|
if (budgetStop && !running.size) { await drainAdds(); break; } // execution budget (stop): the same, the rest end held
|
|
882
|
+
// #4525: a parked needs-human PR is merged with its moved base (seam `syncParked`); non-union conflicts are commented once per base sha.
|
|
883
|
+
const parkedPrs = () => (syncParked ? [...results].filter(([, x]) => x?.outcome === "needs-human" && Number.isInteger(x.pr)) : []);
|
|
884
|
+
if (syncParked && !signal?.aborted && (lastParkedSync === null || t - lastParkedSync >= syncEveryMs)) {
|
|
885
|
+
const rows = parkedPrs();
|
|
886
|
+
if (rows.length) lastParkedSync = t;
|
|
887
|
+
for (const [n, x] of rows) {
|
|
888
|
+
let r = null;
|
|
889
|
+
try { r = await syncParked({ issue: n, pr: x.pr, last: parkedBase.get(n) ?? null }); } catch { r = null; }
|
|
890
|
+
if (!r?.base || r.status === "error" || r.status === "unchanged") continue;
|
|
891
|
+
parkedBase.set(n, r.base);
|
|
892
|
+
if (r.status === "conflict") decided({ issue: n, what: `parked PR #${x.pr} conflicts with its base (${String(r.base).slice(0, 12)}) in ${(r.paths ?? []).join(", ")}`,
|
|
893
|
+
why: "base-sync of a parked needs-human PR merges only clean or pure list-addition conflicts", alternative: "resolve these paths by hand or rerun the issue on the new base" });
|
|
894
|
+
}
|
|
895
|
+
}
|
|
854
896
|
// Wake for a delayed issue, the end of a usage-limit hold, or (with a live K) a poll.
|
|
855
897
|
const waits = pending.map((n) => notBefore.get(n) ?? 0).filter((x) => x > t);
|
|
856
898
|
if (holdUntil) waits.push(holdUntil);
|
|
@@ -859,6 +901,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
859
901
|
if (getParallel && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
|
|
860
902
|
if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
|
|
861
903
|
if (getAdded && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408: an add while only lanes run
|
|
904
|
+
if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
|
|
862
905
|
if (tripped) wake = null;
|
|
863
906
|
if (getCancelled && running.size && !cancelled) wake = Math.min(wake ?? pollMs, pollMs); // #4376: notice a cancel while lanes run
|
|
864
907
|
if (running.size) {
|
|
@@ -916,7 +959,7 @@ export function formatOutcomeLists(rows) {
|
|
|
916
959
|
for (const r of rows ?? []) {
|
|
917
960
|
if (!r || r.recovered) continue;
|
|
918
961
|
const human = r.outcome === "needs-human" && classifyNeedsHuman(r) === "human";
|
|
919
|
-
if (!human && (r.outcome === "needs-human" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
|
|
962
|
+
if (!human && (r.outcome === "needs-human" || r.outcome === "budget-exceeded" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
|
|
920
963
|
}
|
|
921
964
|
const esc = (rows ?? []).filter((r) => r?.outcome === "needs-human" && classifyNeedsHuman(r) === "human");
|
|
922
965
|
const parked = formatParkedLine(rows); // #4389: one line for every parked post-deploy follow-up, never an escalation
|
|
@@ -964,6 +1007,9 @@ export function formatSandboxSummary(rows, { budget = null, day = null } = {}) {
|
|
|
964
1007
|
if (!sb.length) return null;
|
|
965
1008
|
const sum = (f) => sb.reduce((a, r) => a + (Number(f(r)) || 0), 0);
|
|
966
1009
|
const parts = [`sandbox: ${sb.length} lane${sb.length === 1 ? "" : "s"}, ${sum((r) => r.sandbox.minutes).toFixed(1)} min, $${sum((r) => r.sandbox.cost_usd).toFixed(2)} (token cost $${sum((r) => r.cost_usd).toFixed(2)})`];
|
|
1010
|
+
const agents = {}; // #4549: remote lane minutes per pool agent
|
|
1011
|
+
for (const r of sb) for (const [a, m] of Object.entries(r.sandbox.by_agent ?? {})) agents[a] = (agents[a] ?? 0) + (Number(m) || 0);
|
|
1012
|
+
if (Object.keys(agents).length) parts.push(`by agent: ${Object.entries(agents).sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0])).map(([a, m]) => `${a} ${m.toFixed(1)} min`).join(", ")}`);
|
|
967
1013
|
const caps = [];
|
|
968
1014
|
if (budget?.batchMinutes != null) caps.push(`batch ${budget.batchMinutes} min`);
|
|
969
1015
|
if (budget?.batchUsd != null) caps.push(`batch $${budget.batchUsd}`);
|