forgedock 1.32.0 → 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +7 -3
- package/bin/engine/phases.mjs +29 -8
- package/bin/engine-cli.mjs +84 -24
- package/bin/forgedock.mjs +21 -1
- package/bin/lean/basesync.mjs +98 -0
- package/bin/lean/basesync.test.mjs +151 -0
- package/bin/lean/batch.mjs +9 -3
- package/bin/lean/batch.test.mjs +134 -3
- package/bin/lean/budget.mjs +7 -1
- package/bin/lean/budget.test.mjs +9 -1
- package/bin/lean/conformance.mjs +20 -7
- package/bin/lean/conformance.test.mjs +29 -2
- package/bin/lean/depcache.mjs +23 -3
- package/bin/lean/depcache.test.mjs +42 -1
- package/bin/lean/effects.mjs +29 -3
- package/bin/lean/execution.mjs +106 -14
- package/bin/lean/execution.test.mjs +101 -1
- package/bin/lean/hostkill.mjs +8 -0
- package/bin/lean/lean.mjs +65 -24
- package/bin/lean/lean.test.mjs +221 -10
- package/bin/lean/lightlane.mjs +38 -13
- package/bin/lean/lightlane.test.mjs +35 -1
- package/bin/lean/orchestrate.mjs +93 -26
- package/bin/lean/prompts.mjs +17 -7
- package/bin/lean/recovery.test.mjs +26 -0
- package/bin/lean/retry.mjs +15 -1
- package/bin/lean/sandbox-provider.contract.test.mjs +74 -1
- package/bin/lean/sandbox-remote.mjs +40 -7
- package/bin/lean/schedd.mjs +50 -14
- package/bin/lean/schedd.test.mjs +133 -21
- package/bin/lean/seqlease.mjs +49 -2
- package/bin/lean/seqlease.test.mjs +14 -0
- package/bin/lean/sequence.mjs +54 -6
- package/bin/lean/sequence.test.mjs +83 -1
- package/bin/pool/agent.mjs +435 -0
- package/bin/pool/cli.mjs +110 -0
- package/bin/pool/docker.mjs +56 -0
- package/bin/pool/http.mjs +55 -0
- package/bin/pool/router.mjs +299 -0
- package/bin/pool/store.mjs +25 -0
- package/bin/pool/version.mjs +44 -0
- package/bin/runner.mjs +1 -0
- package/bin/stats/aggregate.mjs +6 -0
- package/bin/stats/index.mjs +14 -2
- package/bin/stats/local-source.mjs +1 -1
- package/bin/stats/pool.mjs +51 -0
- package/bin/stats/pool.test.mjs +39 -0
- package/bin/stats/query.mjs +2 -2
- package/bin/stats/stats.test.mjs +15 -1
- package/commands/orchestrate.md +3 -3
- package/docs/CONFIG.md +23 -9
- package/docs/OUTCOME-CODES.md +5 -3
- package/docs/STATS.md +7 -0
- package/package.json +2 -1
- package/pool/README.md +13 -0
- package/pool/image/Dockerfile +46 -0
- package/pool/image/claude-shim +8 -0
- package/pool/image/cs-cred +12 -0
- package/pool/image/entrypoint.sh +61 -0
- package/pool/image/gh-shim +6 -0
- package/pool/install/install-agent-linux.sh +53 -0
- package/pool/install/install-agent.ps1 +225 -0
- package/pool/install/install-agent.sh +101 -0
- package/pool/install/install-router.sh +67 -0
- package/pool/install/keepalive.ps1 +36 -0
- package/pool/install/uninstall-agent.ps1 +20 -0
- package/scripts/hotspots.sh +33 -21
- package/scripts/hotspots.test.sh +22 -3
- package/scripts/refresh-bot-token.test.sh +56 -0
package/bin/lean/orchestrate.mjs
CHANGED
|
@@ -14,9 +14,12 @@
|
|
|
14
14
|
* lane is refused ends `lane-refused` without a run, and a dependency edge between two lanes goes to `onCrossLane`.
|
|
15
15
|
*
|
|
16
16
|
* Durable batch runner (#4121): with a `store` every transition is persisted, so a restarted dispatcher continues the
|
|
17
|
-
* batch; lanes it launched that are still alive are re-attached through `adopt(issue, rec)` instead of relaunched.
|
|
17
|
+
* batch; lanes it launched that are still alive are re-attached through `adopt(issue, rec, {onLive})` instead of relaunched.
|
|
18
|
+
* A seam calls `onLive()` before it waits on a live lane (#4504); an adopt that resolves first is settled (or requeued on null)
|
|
19
|
+
* before scheduling, so a live lane does not hold up the others.
|
|
18
20
|
* `deferred` is a scheduling fact, not an end: each typed code is requeued with its retry-at (`deferPolicy`), a usage
|
|
19
|
-
* limit holds admissions for that account until its reset, and
|
|
21
|
+
* limit holds admissions for that account until its reset, and deferrals other than a busy resource (`host.busy`,
|
|
22
|
+
* `sandbox.capacity`, #4528) stop after `maxDeferrals`.
|
|
20
23
|
* Usage-limit holds (#4190) are keyed by backend identity in `limits` (the host store, shared by every batch on the
|
|
21
24
|
* host; in memory when absent): admission checks the hold of `identity()`, the account the next lane would use. While held, the
|
|
22
25
|
* batch re-reads the store every `pollMs` (#4446), so `forgedock limits clear` admits on the next poll, not at the reset. An `engine-error` is
|
|
@@ -68,10 +71,12 @@
|
|
|
68
71
|
* In-batch recovery (#4267), on by default: this module recovers only when `recovery: {maxAttempts}` is passed, and the
|
|
69
72
|
* CLI always passes it (forge.yaml `orchestration.recovery.max_attempts`, default 1; 0 disables): a row `classifyNeedsHuman` calls
|
|
70
73
|
* recoverable (needs-human `ci.red` / `base.conflict` / `builder.block_untyped`, `contract.invalid` after a mid-run issue
|
|
71
|
-
* edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts
|
|
74
|
+
* edit, `ci-pending` whose PR `conflicting(issue, pr)` says conflicts, the non-human `budget-exceeded` turn overrun of #4505,
|
|
75
|
+
* recovered at the next size budget) is not final while its persisted `recoveries` stay
|
|
72
76
|
* under `maxAttempts`: it is requeued as a recovery lane (`ctx.recover = {code, attempt, final}`, `final` when attempt ==
|
|
73
77
|
* `maxAttempts`) under the same admission as any lane, so its dependents stay queued. Only a final lane relabels needs-human
|
|
74
|
-
* itself; a non-final attempt that ends needs-human/ci-pending without a successor is relabelled here, once (#4302)
|
|
78
|
+
* itself; a non-final attempt that ends needs-human/ci-pending/budget-exceeded without a successor is relabelled here, once (#4302),
|
|
79
|
+
* as is a first-run budget-exceeded the batch does not recover (#4505: the lane never labels it). A recovery
|
|
75
80
|
* still queued when the batch exits (breaker, budget, hold, or a resume that does not relaunch it) is relabelled once with the
|
|
76
81
|
* full persisted lane reason (#4315); also when the loop throws, and through `onExit(fn)` (the caller's signal handler
|
|
77
82
|
* awaits `fn(cause)`), each once (#4318). `relabelNeedsHuman` may return `false`/"failed" (not recorded, a resume retries)
|
|
@@ -91,7 +96,7 @@
|
|
|
91
96
|
* the queue order except that decomposed sub-issues go first. A settled row carries `est_minutes` (estimated vs. actual).
|
|
92
97
|
*/
|
|
93
98
|
import { parseDependsOn } from "../engine/phases.mjs";
|
|
94
|
-
import { classifyNeedsHuman, recoveryCodeOf } from "../engine/outcome-codes.mjs";
|
|
99
|
+
import { classifyNeedsHuman, recoveryCodeOf, OUTCOME_CODES } from "../engine/outcome-codes.mjs";
|
|
95
100
|
import { recoveryEvidence } from "./render.mjs";
|
|
96
101
|
import { renderAlerts } from "../stats/render.mjs";
|
|
97
102
|
import { formatParkedLine, postDeployOnly } from "./prefilter.mjs";
|
|
@@ -101,11 +106,13 @@ import { laneSize, SIZES } from "./budget.mjs";
|
|
|
101
106
|
const SATISFIED = new Set(["merged", "invalid"]);
|
|
102
107
|
// Times one issue may be requeued behind a dependency before its dependency-open row is final.
|
|
103
108
|
export const MAX_REQUEUES = 2;
|
|
104
|
-
//
|
|
109
|
+
// Deferrals other than a busy resource (UNCAPPED_DEFERRALS) one issue may take before its deferred row is final.
|
|
105
110
|
export const DEFAULT_MAX_DEFERRALS = 3;
|
|
106
111
|
// Relaunches of an issue whose lane ended engine-error.
|
|
107
112
|
export const ENGINE_ERROR_RELAUNCHES = 1;
|
|
108
113
|
export const HOST_BUSY_RETRY_S = 60;
|
|
114
|
+
// #4528: a busy host or a full sandbox pool is a wait, not a failure: requeued after its retry-in, never capped.
|
|
115
|
+
const UNCAPPED_DEFERRALS = new Set(["host.busy", "sandbox.capacity"]);
|
|
109
116
|
export const RATE_LIMIT_FALLBACK_S = 15 * 60;
|
|
110
117
|
export const DEFER_BACKOFF_S = 120;
|
|
111
118
|
export const DEFER_BACKOFF_MAX_S = 30 * 60;
|
|
@@ -154,12 +161,12 @@ export function parseDeps(body, issue) {
|
|
|
154
161
|
const posNum = (v) => { const n = Number(v); return Number.isFinite(n) && n > 0 ? n : null; };
|
|
155
162
|
|
|
156
163
|
/**
|
|
157
|
-
* When to run a `deferred` issue again. `deferrals` counts this issue's earlier
|
|
164
|
+
* When to run a `deferred` issue again. `deferrals` counts this issue's earlier capped deferrals (not UNCAPPED_DEFERRALS).
|
|
158
165
|
* @returns {{delayMs:number, host?:true, hold?:true} | {capped:true}}
|
|
159
166
|
*/
|
|
160
167
|
export function deferPolicy(r, { now, deferrals = 0, maxDeferrals = DEFAULT_MAX_DEFERRALS }) {
|
|
161
168
|
const code = r?.code ?? null;
|
|
162
|
-
if (code
|
|
169
|
+
if (UNCAPPED_DEFERRALS.has(code)) return { delayMs: (posNum(r?.retryInSeconds) ?? HOST_BUSY_RETRY_S) * 1000, host: true };
|
|
163
170
|
const k = deferrals + 1;
|
|
164
171
|
if (k > maxDeferrals) return { capped: true };
|
|
165
172
|
if (code === "rate_limited") {
|
|
@@ -215,6 +222,8 @@ export function pickNext(ready, { classes, share, whenIdle = false, started = 0,
|
|
|
215
222
|
}
|
|
216
223
|
|
|
217
224
|
const toObj = (m) => Object.fromEntries([...m].map(([k, v]) => [String(k), v]));
|
|
225
|
+
/** #4525: how often a parked needs-human PR is re-synced with its base while the batch runs. */
|
|
226
|
+
export const DEFAULT_SYNC_EVERY_MS = 10 * 60 * 1000;
|
|
218
227
|
const toMap = (o) => new Map(Object.entries(o ?? {}).map(([k, v]) => [Number(k), v]));
|
|
219
228
|
|
|
220
229
|
/**
|
|
@@ -224,7 +233,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
224
233
|
store = null, adopt = null, getParallel = null, getAdded = null, breaker = null, maxDeferrals = DEFAULT_MAX_DEFERRALS, sleep = cancellableSleep, pollMs = 5000,
|
|
225
234
|
classify = null, maintenance = null, limits = null, identity = () => "default", batchId = null, drift = null, sched = null, priorityOf = null, remote = null, beforeAdmit = null,
|
|
226
235
|
recovery = null, decide = () => {}, conflicting = null, relabelNeedsHuman = null, onExit = null, issueLabels = null, getCancelled = null, stopLane = null,
|
|
227
|
-
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null }) {
|
|
236
|
+
estimate = null, starveMs = DEFAULT_STARVE_MS, signal = null, dispatchCheck = null, syncParked = null, syncEveryMs = DEFAULT_SYNC_EVERY_MS }) {
|
|
228
237
|
const saved = store?.load?.() ?? null;
|
|
229
238
|
const order = saved ? [...saved.order] : [];
|
|
230
239
|
const set = new Set(order);
|
|
@@ -254,16 +263,21 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
254
263
|
const classes = toMap(saved?.classes); // issue -> {maintenance, priority}; only with a maintenance budget
|
|
255
264
|
const dispatch = { started: saved?.dispatch?.started ?? 0, maint: saved?.dispatch?.maint ?? 0 };
|
|
256
265
|
let budgetHeld = false;
|
|
266
|
+
const parkedBase = new Map(); // #4525: issue -> base sha its parked PR was last synced (or reported conflicting) against
|
|
267
|
+
let lastParkedSync = null;
|
|
257
268
|
const bodies = new Map(); // issue -> body, kept in memory for drift.holds only
|
|
258
269
|
const driftHeld = new Map(); // issue -> flagged package dir holding it this pass
|
|
259
270
|
const reg = sched?.registry ?? null;
|
|
260
271
|
const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
|
|
261
272
|
const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
|
|
273
|
+
let capAt, capAdmits = 0; // #4540: provider capacity sample and the remote admissions made since it
|
|
262
274
|
const remoteRun = new Set(); // remote lanes running now (#4253): admitted against remoteMax, never the host cap
|
|
263
275
|
let budgetLocal = false; // the execution budget refused a remote lane under on_budget local: the rest run on the host
|
|
264
276
|
// #4256: where a lane goes before capacity: `remote.place` (hybrid) or `remote.isRemote` (remote mode).
|
|
265
277
|
const placeOf = (n) => {
|
|
266
278
|
if (!remote) return null;
|
|
279
|
+
// #4542: a lane recovering from a remote capability block runs on the host, which has the capability.
|
|
280
|
+
if (recovering.get(n)?.code === OUTCOME_CODES.ENV_CAPABILITY_REMOTE) return { where: "local", reason: "host capability: recovering env.capability_remote" };
|
|
267
281
|
if (budgetLocal) return { where: "local", reason: `fallback to local: execution budget: ${budgetHit ?? "cap reached"}` };
|
|
268
282
|
try {
|
|
269
283
|
if (typeof remote.place === "function") {
|
|
@@ -439,9 +453,19 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
439
453
|
if (p?.where === "remote") {
|
|
440
454
|
const adm = admitRemote();
|
|
441
455
|
if (adm.ok) {
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
456
|
+
// #4540: the provider's free slots (`remote.capacity() -> {free, at}`, sync, null = unknown) cap admissions per sample `at`; the issue queues here instead of a 503.
|
|
457
|
+
let cap = null;
|
|
458
|
+
try { cap = remote.capacity?.() ?? null; } catch { cap = null; }
|
|
459
|
+
// #4547: opt-in (`execution.fallback_local`): every pool agent offline or asleep (sampled online 0) runs the lane on the host.
|
|
460
|
+
if (remote.offlineFallback && cap && cap.online === 0) {
|
|
461
|
+
p = { where: "local", reason: `fallback to local: no pool agent online (${p.reason})` };
|
|
462
|
+
} else if (cap && Number.isInteger(cap.free)) {
|
|
463
|
+
if (cap.at !== capAt) { capAt = cap.at; capAdmits = 0; }
|
|
464
|
+
if (capAdmits >= cap.free && remoteRun.size < remoteMax) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
|
|
465
|
+
}
|
|
466
|
+
if (p.where === "remote" && remoteRun.size < remoteMax) { remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
|
|
467
|
+
if (p.where === "remote" && !remote.fallbackLocal) return false;
|
|
468
|
+
if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax} reached (${p.reason})` }; // #4256: hybrid runs it on the host
|
|
445
469
|
} else {
|
|
446
470
|
noteBudget(adm.reason);
|
|
447
471
|
if (remote.onBudget !== "local" && !remote.fallbackLocal) { budgetStop = adm.reason; return false; }
|
|
@@ -491,16 +515,18 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
491
515
|
if (r?.sandbox && typeof r.sandbox === "object") {
|
|
492
516
|
const prev = sandboxUse.get(n) ?? { minutes: 0, cost_usd: 0, attempts: 0 };
|
|
493
517
|
const m = Number(r.sandbox.minutes) >= 0 ? Number(r.sandbox.minutes) : 0, c = Number(r.sandbox.cost_usd) >= 0 ? Number(r.sandbox.cost_usd) : 0;
|
|
494
|
-
|
|
518
|
+
const by = { ...(prev.by_agent ?? {}) }; // #4549: minutes per pool agent, over attempts
|
|
519
|
+
if (typeof r.sandbox.agent === "string" && r.sandbox.agent) by[r.sandbox.agent] = Math.round(((by[r.sandbox.agent] ?? 0) + m) * 10) / 10;
|
|
520
|
+
sandboxUse.set(n, { minutes: Math.round((prev.minutes + m) * 10) / 10, cost_usd: Math.round((prev.cost_usd + c) * 1e6) / 1e6, attempts: prev.attempts + 1, ...(Object.keys(by).length ? { by_agent: by } : {}) });
|
|
495
521
|
try { remote?.onUsage?.(n, r.sandbox, rec); } catch { /* the batch total still counts it */ }
|
|
496
522
|
}
|
|
497
|
-
if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n) };
|
|
523
|
+
if (sandboxUse.has(n)) row.sandbox = { ...sandboxUse.get(n), ...(sandboxUse.get(n).by_agent ? { by_agent: { ...sandboxUse.get(n).by_agent } } : {}) };
|
|
498
524
|
const done = () => { running.delete(n); runRecs.delete(n); releaseSlot(n); remoteRun.delete(n); };
|
|
499
525
|
if (row.outcome === "dependency-open" && depList.length) {
|
|
500
526
|
// #4387: a parked dependency is not pulled in again (fail-open on an unreadable label).
|
|
501
527
|
let parked;
|
|
502
528
|
for (const d of depList) {
|
|
503
|
-
if (results.get(d)?.outcome
|
|
529
|
+
if (["needs-human", "budget-exceeded"].includes(results.get(d)?.outcome)) { parked = d; break; } // #4505: an unrecovered overrun parks too
|
|
504
530
|
if (set.has(d) || typeof issueLabels !== "function") continue;
|
|
505
531
|
let names = [];
|
|
506
532
|
try { names = (await issueLabels(d)) ?? []; } catch { names = []; }
|
|
@@ -585,7 +611,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
585
611
|
try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
|
|
586
612
|
}
|
|
587
613
|
}
|
|
588
|
-
if (recoveryMax &&
|
|
614
|
+
if (recoveryMax && ["needs-human", "ci-pending", "budget-exceeded"].includes(row.outcome)) {
|
|
589
615
|
let isConflicting = false;
|
|
590
616
|
if (row.outcome === "ci-pending" && typeof conflicting === "function" && Number.isInteger(row.pr)) {
|
|
591
617
|
try { isConflicting = (await conflicting(n, row.pr)) === true; } catch { isConflicting = false; }
|
|
@@ -605,9 +631,13 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
605
631
|
}
|
|
606
632
|
}
|
|
607
633
|
// #4302: a non-final recovery attempt left needs-human/ci-pending unlabelled for its successor; none is launched, so relabel once.
|
|
608
|
-
if (prior && prior.attempt < recoveryMax && ["needs-human", "ci-pending"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
|
|
634
|
+
if (prior && prior.attempt < recoveryMax && ["needs-human", "ci-pending", "budget-exceeded"].includes(row.outcome) && typeof relabelNeedsHuman === "function") {
|
|
609
635
|
try { await relabelNeedsHuman({ issue: n, evidence: evidence() }); } catch { /* best effort */ }
|
|
610
636
|
}
|
|
637
|
+
// #4505: a first-run turn overrun is unlabelled by its lane; no recovery follows (off or cap used), so the runner labels it once.
|
|
638
|
+
if (!prior && row.outcome === "budget-exceeded" && typeof relabelNeedsHuman === "function") {
|
|
639
|
+
try { await relabelNeedsHuman({ issue: n, evidence: `Implement turn budget overrun (\`budget.exceeded\`) not recovered in this batch:\n${String(row.detail ?? "")}` }); } catch { /* best effort */ }
|
|
640
|
+
}
|
|
611
641
|
results.set(n, row);
|
|
612
642
|
if (row.outcome === "merged" && drift?.onMerged && r?.structure && String(rec?.lane ?? r?.lane ?? "").startsWith("milestone/")) {
|
|
613
643
|
save(); update(row);
|
|
@@ -666,15 +696,34 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
666
696
|
};
|
|
667
697
|
|
|
668
698
|
// Lanes launched by an earlier dispatcher: re-attach to the live ones, queue the rest again (no result was recorded).
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
699
|
+
// #4504: every adopt starts at once and races its `onLive` signal, so a live lane never holds up the others: one that
|
|
700
|
+
// already ended is settled from its result (or requeued without one) before scheduling; a live one is awaited in the background.
|
|
701
|
+
const requeueLost = (n) => { runRecs.delete(n); if (!pending.includes(n) && !results.has(n)) { pending.unshift(n); enqueued.set(n, now()); } };
|
|
702
|
+
const adopting = [...runRecs].map(([n, rec]) => {
|
|
703
|
+
if (!adopt || !(rec?.pid || rec?.handle)) return { n, rec, live: false, p: Promise.resolve(null) };
|
|
704
|
+
let onLive;
|
|
705
|
+
const isLive = new Promise((r) => { onLive = () => r(true); });
|
|
706
|
+
const p = Promise.resolve().then(() => adopt(n, rec, { onLive })).catch(() => null);
|
|
707
|
+
return { n, rec, p, first: Promise.race([p.then(() => false), isLive]) };
|
|
708
|
+
});
|
|
709
|
+
for (const a of adopting) a.live = a.first ? await a.first : false;
|
|
710
|
+
for (const { n, rec, p, live } of adopting) {
|
|
711
|
+
if (live) {
|
|
712
|
+
if (rec.remote) remoteRun.add(n);
|
|
713
|
+
else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); }
|
|
714
|
+
track(n, p.then((r) => {
|
|
715
|
+
if (r) return settle(n, r, rec, checked.get(n) ?? null);
|
|
716
|
+
if (halted) return; // #4454: kept for the next resume
|
|
717
|
+
running.delete(n); releaseSlot(n); remoteRun.delete(n); requeueLost(n); save(); // ended without a result after all
|
|
718
|
+
}));
|
|
719
|
+
continue;
|
|
676
720
|
}
|
|
677
|
-
|
|
721
|
+
const r = await p;
|
|
722
|
+
if (!r) { requeueLost(n); continue; }
|
|
723
|
+
if (rec.remote) remoteRun.add(n);
|
|
724
|
+
else if (reg) { reg.adoptSlot(keyOf(n), { ...entryOf(n), lanePid: rec.pid }); slotted.add(n); } // released by the settle
|
|
725
|
+
track(n, Promise.resolve(r).then((x) => settle(n, x, rec, checked.get(n) ?? null)));
|
|
726
|
+
await running.get(n); // settled before scheduling
|
|
678
727
|
}
|
|
679
728
|
// A resume re-derives each unfinished issue's lane from its current milestone (#4153): a re-milestone takes effect.
|
|
680
729
|
if (saved && resolveLane) {
|
|
@@ -830,6 +879,20 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
830
879
|
if (tripped && !running.size) { await drainAdds(); break; } // breaker: let running lanes finish, keep the rest pending for a resume
|
|
831
880
|
if (cancelled && !running.size) { await drainAdds(); break; } // #4376: cancel: the rest end held for a resume
|
|
832
881
|
if (budgetStop && !running.size) { await drainAdds(); break; } // execution budget (stop): the same, the rest end held
|
|
882
|
+
// #4525: a parked needs-human PR is merged with its moved base (seam `syncParked`); non-union conflicts are commented once per base sha.
|
|
883
|
+
const parkedPrs = () => (syncParked ? [...results].filter(([, x]) => x?.outcome === "needs-human" && Number.isInteger(x.pr)) : []);
|
|
884
|
+
if (syncParked && !signal?.aborted && (lastParkedSync === null || t - lastParkedSync >= syncEveryMs)) {
|
|
885
|
+
const rows = parkedPrs();
|
|
886
|
+
if (rows.length) lastParkedSync = t;
|
|
887
|
+
for (const [n, x] of rows) {
|
|
888
|
+
let r = null;
|
|
889
|
+
try { r = await syncParked({ issue: n, pr: x.pr, last: parkedBase.get(n) ?? null }); } catch { r = null; }
|
|
890
|
+
if (!r?.base || r.status === "error" || r.status === "unchanged") continue;
|
|
891
|
+
parkedBase.set(n, r.base);
|
|
892
|
+
if (r.status === "conflict") decided({ issue: n, what: `parked PR #${x.pr} conflicts with its base (${String(r.base).slice(0, 12)}) in ${(r.paths ?? []).join(", ")}`,
|
|
893
|
+
why: "base-sync of a parked needs-human PR merges only clean or pure list-addition conflicts", alternative: "resolve these paths by hand or rerun the issue on the new base" });
|
|
894
|
+
}
|
|
895
|
+
}
|
|
833
896
|
// Wake for a delayed issue, the end of a usage-limit hold, or (with a live K) a poll.
|
|
834
897
|
const waits = pending.map((n) => notBefore.get(n) ?? 0).filter((x) => x > t);
|
|
835
898
|
if (holdUntil) waits.push(holdUntil);
|
|
@@ -838,6 +901,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
|
|
|
838
901
|
if (getParallel && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
|
|
839
902
|
if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
|
|
840
903
|
if (getAdded && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408: an add while only lanes run
|
|
904
|
+
if (syncParked && running.size && parkedPrs().length) wake = Math.min(wake ?? syncEveryMs, Math.max(0, syncEveryMs - (t - (lastParkedSync ?? t)))); // #4525
|
|
841
905
|
if (tripped) wake = null;
|
|
842
906
|
if (getCancelled && running.size && !cancelled) wake = Math.min(wake ?? pollMs, pollMs); // #4376: notice a cancel while lanes run
|
|
843
907
|
if (running.size) {
|
|
@@ -895,7 +959,7 @@ export function formatOutcomeLists(rows) {
|
|
|
895
959
|
for (const r of rows ?? []) {
|
|
896
960
|
if (!r || r.recovered) continue;
|
|
897
961
|
const human = r.outcome === "needs-human" && classifyNeedsHuman(r) === "human";
|
|
898
|
-
if (!human && (r.outcome === "needs-human" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
|
|
962
|
+
if (!human && (r.outcome === "needs-human" || r.outcome === "budget-exceeded" || r.recovery)) out.push(`unrecovered: #${r.issue} (${r.code ?? r.recovery?.code ?? "unclassified"}: ${one(r.recovery ? `recovery ${r.recovery.code} ended ${r.recovery.outcome}: ${r.recovery.detail}` : r.detail)})`);
|
|
899
963
|
}
|
|
900
964
|
const esc = (rows ?? []).filter((r) => r?.outcome === "needs-human" && classifyNeedsHuman(r) === "human");
|
|
901
965
|
const parked = formatParkedLine(rows); // #4389: one line for every parked post-deploy follow-up, never an escalation
|
|
@@ -943,6 +1007,9 @@ export function formatSandboxSummary(rows, { budget = null, day = null } = {}) {
|
|
|
943
1007
|
if (!sb.length) return null;
|
|
944
1008
|
const sum = (f) => sb.reduce((a, r) => a + (Number(f(r)) || 0), 0);
|
|
945
1009
|
const parts = [`sandbox: ${sb.length} lane${sb.length === 1 ? "" : "s"}, ${sum((r) => r.sandbox.minutes).toFixed(1)} min, $${sum((r) => r.sandbox.cost_usd).toFixed(2)} (token cost $${sum((r) => r.cost_usd).toFixed(2)})`];
|
|
1010
|
+
const agents = {}; // #4549: remote lane minutes per pool agent
|
|
1011
|
+
for (const r of sb) for (const [a, m] of Object.entries(r.sandbox.by_agent ?? {})) agents[a] = (agents[a] ?? 0) + (Number(m) || 0);
|
|
1012
|
+
if (Object.keys(agents).length) parts.push(`by agent: ${Object.entries(agents).sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0])).map(([a, m]) => `${a} ${m.toFixed(1)} min`).join(", ")}`);
|
|
946
1013
|
const caps = [];
|
|
947
1014
|
if (budget?.batchMinutes != null) caps.push(`batch ${budget.batchMinutes} min`);
|
|
948
1015
|
if (budget?.batchUsd != null) caps.push(`batch $${budget.batchUsd}`);
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -215,18 +215,27 @@ export function reproPlanNote(on) {
|
|
|
215
215
|
return `\n\nThis issue is a bug. When it is testable, ${CONTRACT_FILE} must also carry "repro_test":{"path":"<test file>","id":"<test name>","command":"<exact command that runs only that test>"}: a test that fails on the base and passes after the fix. The engine proves it with scripts/acceptance-red-green.sh (head's changed test files are overlaid on the base) and fails the run if it is not red on base and green on head. When it is not testable (specs, docs, workflows, environment), write "repro_test":"none","repro_test_reason":"<one line why>" instead.`;
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
+
/** forge#4519: the repo's generated, bot-owned globs (forge.yaml maintenance.generated_paths via hotspots.sh); "" when none. */
|
|
219
|
+
function generatedNote(facts) {
|
|
220
|
+
const globs = Array.isArray(facts?.generated_paths) ? facts.generated_paths.filter((g) => typeof g === "string" && g) : [];
|
|
221
|
+
if (!globs.length) return "";
|
|
222
|
+
return `\n\nGenerated paths (forge.yaml maintenance.generated_paths, owned by the repo's generator): ${globs.join(", ")}. Never list or amend them in as deliverables and never commit changes to them; the engine refuses such amendments.`;
|
|
223
|
+
}
|
|
224
|
+
|
|
218
225
|
/** forge#4195: hotspot facts for the plan prompt: one sentence plus one line per over-threshold file; "" when none. */
|
|
219
226
|
export function hotspotPlanNote(facts) {
|
|
220
|
-
|
|
227
|
+
const gen = generatedNote(facts);
|
|
228
|
+
if (!facts || facts.enabled === false || !Array.isArray(facts.files)) return gen;
|
|
221
229
|
const hot = facts.files.filter((f) => f?.over && !f.excluded);
|
|
222
|
-
if (!hot.length) return
|
|
223
|
-
return `\n\nHotspots (scripts/hotspots.sh): an Affected File below is over ${facts.file_lines} lines, so the contract must put new logic in a new or neighbouring module and give the hotspot only wiring (an import and a call); growing it by more than ${facts.growth_lines} lines needs "hotspot-exempt: <reason>" in that deliverable's change.\n${hot.map((f) => `- ${f.path}: ${f.lines} lines, ${f.commits_90d} commits/90d`).join("\n")}`;
|
|
230
|
+
if (!hot.length) return gen;
|
|
231
|
+
return gen + `\n\nHotspots (scripts/hotspots.sh): an Affected File below is over ${facts.file_lines} lines, so the contract must put new logic in a new or neighbouring module and give the hotspot only wiring (an import and a call); growing it by more than ${facts.growth_lines} lines needs "hotspot-exempt: <reason>" in that deliverable's change.\n${hot.map((f) => `- ${f.path}: ${f.lines} lines, ${f.commits_90d} commits/90d`).join("\n")}`;
|
|
224
232
|
}
|
|
225
233
|
|
|
226
234
|
/** forge#4195: the builder's growth rule with the effective thresholds; "" when the check is off. */
|
|
227
235
|
export function hotspotBuildRule(facts) {
|
|
228
|
-
|
|
229
|
-
|
|
236
|
+
const gen = generatedNote(facts);
|
|
237
|
+
if (!facts || facts.enabled === false || !(facts.file_lines > 0) || !(facts.growth_lines > 0)) return gen;
|
|
238
|
+
return gen + `\n\nDon't grow a file over ${facts.file_lines} lines by more than ${facts.growth_lines} lines. Put new code next to it.`;
|
|
230
239
|
}
|
|
231
240
|
|
|
232
241
|
/** #4322: tests that enumerate the siblings of a module the issue names (bin/lean/enumerating.mjs), for the plan; "" when none. */
|
|
@@ -248,7 +257,7 @@ export function missingEntriesPrompt(missing, mode = "local") {
|
|
|
248
257
|
return `These tests enumerate the siblings of symbols or files this change adds, but do not name them (CI will fail on the stale list):\n${missing.slice(0, 30).map((m) => `- ${m.symbol} (${m.module}) -> ${m.test}:${m.line}`).join("\n")}\n\nFor each, add the symbol to that test's list, ${NOT_MEMBER}. ${rerun(mode)}, commit, and update ${IMPLEMENTATION_FILE} and ${RESULT_FILE}. Do not push or touch GitHub.`;
|
|
249
258
|
}
|
|
250
259
|
|
|
251
|
-
const RENUMBER_RULE = "Renumber the lane's files, never the base's: rename each with git mv to its new number, keep the SQL (or other migration source) byte-identical, change a numbered document (e.g. an ADR) only in its own number (its title), regenerate the snapshot and journal with the repo's own migration generator (never hand- or text-merge the journal), keep journal entries strictly ordered, and update every reference to the old name, including the series index (e.g. docs/adr/README.md) so it lists both the base's entry and yours.";
|
|
260
|
+
const RENUMBER_RULE = "Renumber the lane's files, never the base's: rename each with git mv to its new number, keep the SQL (or other migration source) byte-identical except a first-line header naming its own number/tag, which follows the new one, change a numbered document (e.g. an ADR) only in its own number (its title), regenerate the snapshot and journal with the repo's own migration generator (never hand- or text-merge the journal), keep journal entries strictly ordered, and update every reference to the old name, including the series index (e.g. docs/adr/README.md) so it lists both the base's entry and yours.";
|
|
252
261
|
|
|
253
262
|
/** #4322: lane-added sequence-numbered files whose number the base already uses, at hand-off or after a base-sync. */
|
|
254
263
|
export function sequencePrompt(collisions, lane, mode = "local") {
|
|
@@ -301,7 +310,8 @@ const BLOCK_RULES = `"blocked" needs a typed "block" in ${RESULT_FILE}, one of:
|
|
|
301
310
|
- {"kind":"dependency","issues":[N]}: an open upstream issue must land first. The engine requeues this issue behind it.
|
|
302
311
|
- {"kind":"split-required","subissues":[{"title":"","body":"","depends_on":[0],"files":["<path>"]}],"size":{"files":N,"lines":N}}: too big for one change. The engine files the sub-issues (depends_on = indexes of earlier sub-issues; files = the files or dirs each will edit, also accepted as "paths") only when every sub-issue's files are disjoint from the others' or the whole change's size estimate exceeds ${SPLIT_MIN_FILES} files or ${SPLIT_MIN_LINES} lines; otherwise keep one lane: an ordered checklist in the contract with a validate per item. In an admitted split, when siblings would edit the same hub (a barrel or index export list, a union type, a shared test fixture, an index doc), pre-stage its slots in the earliest sub-issue and make the others depend on it, or give each sibling its own per-feature file; the engine serialises sub-issues that still share a file.
|
|
303
312
|
- {"kind":"human","why":"${HUMAN_BLOCK_REASONS.join("|")}"}: only an external action agents cannot take, or a product decision too critical for agents.
|
|
304
|
-
"An upstream issue must land first", "needs a split" and "needs prod data that read-only monitoring can read" are never "human"
|
|
313
|
+
"An upstream issue must land first", "needs a split" and "needs prod data that read-only monitoring can read" are never "human".
|
|
314
|
+
Remote sandbox lacking a host-only capability (sops, prod_monitor, ops_credentials): name it in the summary; the host reruns the lane.`;
|
|
305
315
|
|
|
306
316
|
export function blockCorrectionPrompt(problem) {
|
|
307
317
|
return `Your ${RESULT_FILE} says "blocked", but ${String(problem).slice(0, 1000)}. That is not a block that needs a human.\n\n${BLOCK_RULES}\n\nEither do the work (fix, commit, update the stage notes and ${RESULT_FILE}) or write a valid typed "block". Do not push or touch GitHub.`;
|
|
@@ -56,6 +56,32 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
56
56
|
assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
|
|
57
57
|
});
|
|
58
58
|
|
|
59
|
+
it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
|
|
60
|
+
const BE = (detail, more = {}) => ({ terminalReason: "budget-exceeded", code: "budget.exceeded", detail, branch: "feat/issue-1-lean", ...more });
|
|
61
|
+
const relabels = [];
|
|
62
|
+
const { rows, calls, decisions } = await batch({ 1: [BE("Implement turn budget spent (size S, 25 turns; 30 used)"), { terminalReason: "merged", pr: 101 }], 2: [{ terminalReason: "merged", pr: 102 }] },
|
|
63
|
+
{ 2: "**Depends on**: #1" }, { relabelNeedsHuman: async (x) => { relabels.push(x); } });
|
|
64
|
+
const r = byIssue(rows);
|
|
65
|
+
assert.deepEqual(calls.filter((c) => c.n === 1).map((c) => c.recover), [null, { code: "budget.exceeded", attempt: 1, final: true }]);
|
|
66
|
+
assert.equal(r[1].outcome, "merged"); assert.equal(r[1].recovered, "budget.exceeded");
|
|
67
|
+
assert.equal(r[2].outcome, "merged", "the dependent stayed queued and ran");
|
|
68
|
+
assert.ok(calls.findIndex((c) => c.n === 2) > calls.findLastIndex((c) => c.n === 1), "the dependent ran after the recovered merge");
|
|
69
|
+
assert.ok(decisions.some((d) => d.issue === 1 && /recovery lane for budget\.exceeded .*dependents stay queued/.test(d.what)));
|
|
70
|
+
assert.equal(relabels.length, 0, "never labelled needs-human");
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("#4505 an unrecovered budget-exceeded (recovery off) is relabelled once by the runner, parks dependents and is listed unrecovered", async () => {
|
|
74
|
+
const relabels = [];
|
|
75
|
+
const { rows, calls } = await batch({ 1: [{ terminalReason: "budget-exceeded", code: "budget.exceeded", detail: "Implement turn budget spent (size S)" }], 2: [] },
|
|
76
|
+
{ 2: "**Depends on**: #1" }, { maxAttempts: 0, relabelNeedsHuman: async (x) => { relabels.push(x); } });
|
|
77
|
+
const r = byIssue(rows);
|
|
78
|
+
assert.equal(calls.filter((c) => c.n === 1).length, 1);
|
|
79
|
+
assert.equal(r[1].outcome, "budget-exceeded");
|
|
80
|
+
assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
|
|
81
|
+
assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
|
|
82
|
+
assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
|
|
83
|
+
});
|
|
84
|
+
|
|
59
85
|
it("human codes (product-decision, spend) are never recovered, still skip dependents and are the only escalations, asked once while other lanes keep running", async () => {
|
|
60
86
|
const { rows, calls, decisions } = await batch({
|
|
61
87
|
1: [NH("builder.blocked", "Builder is blocked (product-decision): pick a pricing model")],
|
package/bin/lean/retry.mjs
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
export const PUSH_TRANSIENT_RE = /remote unpack failed|index-pack failed|RPC failed|unexpected disconnect|early EOF|the remote end hung up|HTTP\/?\s?5\d\d|returned error: 5\d\d|\b50[0234]\b|internal server error|bad gateway|service unavailable|gateway time-?out|connection (?:reset|timed out|refused)|timed out|could not resolve host|broken pipe|ETIMEDOUT|ECONNRESET|temporarily unavailable|\[remote rejected\][^\n]*\(failed\)|unable to write file|unable to create temporary object|unpacker error|failed to write object/i;
|
|
6
6
|
export const PUSH_PERMANENT_RE = /protected branch|GH006|GH013|repository rule violations|permission (?:denied|to .* denied)|denied to|\b403\b|authentication failed|not allowed|hook declined|pre-receive hook declined|\[rejected\]|non-fast-forward|fetch first/i;
|
|
7
7
|
export const GH_TRANSIENT_RE = /HTTP 5\d\d|\b50[0234]\b|secondary rate limit|rate limit exceeded|abuse detection|something went wrong|internal server error|bad gateway|service unavailable|ETIMEDOUT|ECONNRESET|connection reset|timed out|\bEOF\b|temporarily unavailable/i;
|
|
8
|
-
|
|
8
|
+
/** #4507: gh failing to parse its own input: a torn HTTP cache entry (JSON) or a half-written config (YAML). */
|
|
9
|
+
export const GH_PARSE_RE = /invalid character .{0,40}looking for|unexpected end of JSON input|cannot unmarshal|yaml: line \d+|yaml: unmarshal/i;
|
|
10
|
+
export const MERGE_TRANSIENT_RE = new RegExp(`${GH_TRANSIENT_RE.source}|${GH_PARSE_RE.source}|base branch was modified|head branch was modified|mergeability.*unknown|try again`, "i");
|
|
9
11
|
export const MERGE_CONFLICT_RE = /not mergeable|merge conflict|conflict/i;
|
|
10
12
|
|
|
11
13
|
export const errText = (e) => `${e?.stderr ?? ""}\n${e?.message ?? ""}\n${e?.stdout ?? ""}`;
|
|
@@ -29,3 +31,15 @@ export async function retrying(fn, { transient, permanent = null, attempts = 4,
|
|
|
29
31
|
last.transientExhausted = true;
|
|
30
32
|
throw last;
|
|
31
33
|
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* #4507: run `call(env)` with the lane's gh cache env; on a gh parse error (corrupt cache) reset the cache and run it
|
|
37
|
+
* once more. Without a cache it still retries once. Any other error is rethrown unchanged.
|
|
38
|
+
*/
|
|
39
|
+
export async function ghCacheRetry(call, cache) {
|
|
40
|
+
try { return await call(cache?.env?.()); } catch (e) {
|
|
41
|
+
if (!GH_PARSE_RE.test(errText(e))) throw e;
|
|
42
|
+
try { await cache?.reset?.(); } catch { /* retry on the old dir */ }
|
|
43
|
+
return await call(cache?.env?.());
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -54,7 +54,7 @@ function fakeTransport({ launchError = null } = {}) {
|
|
|
54
54
|
function remoteFixture(batch, { launchError = null } = {}) {
|
|
55
55
|
const transport = fakeTransport({ launchError });
|
|
56
56
|
const provider = remoteProvider({ batch, repo: "o/r", settings: { image: "img:1", cacheStore: "s3://cache", tokenEnv: "FD_TOKEN", secrets: ["GH_TOKEN"] }, transport,
|
|
57
|
-
env: { GH_TOKEN: SECRET, FD_TOKEN: TOKEN }, credentials: (n) => ({ GH_TOKEN: SECRET, FD_TOKEN: TOKEN })[n] ?? null });
|
|
57
|
+
env: { GH_TOKEN: SECRET, FD_TOKEN: TOKEN }, credentials: (n) => ({ GH_TOKEN: SECRET, FD_TOKEN: TOKEN })[n] ?? null, gitIdentity: () => ({}) });
|
|
58
58
|
return {
|
|
59
59
|
provider, transport,
|
|
60
60
|
finish: (h, result) => { const b = transport.boxes.get(h.id); b.log += `lane done; token ${SECRET} and ${TOKEN}\n`; b.result = result; b.state = "exited"; b.minutes = 3.5; },
|
|
@@ -147,6 +147,23 @@ describe("remote adapter (#4254)", () => {
|
|
|
147
147
|
assert.equal(transport.specs[0].env.GH_TOKEN, "from-keychain");
|
|
148
148
|
});
|
|
149
149
|
|
|
150
|
+
it("the sandbox commits as the dispatching host's git identity; a lane env identity wins", async () => {
|
|
151
|
+
const batch = openBatch(join(tmpDir("remote-git-"), "batches"), "bt");
|
|
152
|
+
const transport = fakeTransport();
|
|
153
|
+
const p = remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null, gitIdentity: () => ({ name: "bot[bot]", email: "1+bot[bot]@users.noreply.github.com" }) });
|
|
154
|
+
await p.start({ issue: 1, args: [], env: {} });
|
|
155
|
+
const env = transport.specs[0].env;
|
|
156
|
+
for (const role of ["AUTHOR", "COMMITTER"]) {
|
|
157
|
+
assert.equal(env[`GIT_${role}_NAME`], "bot[bot]");
|
|
158
|
+
assert.equal(env[`GIT_${role}_EMAIL`], "1+bot[bot]@users.noreply.github.com");
|
|
159
|
+
}
|
|
160
|
+
await p.start({ issue: 2, args: [], env: { GIT_AUTHOR_NAME: "lane", GIT_AUTHOR_EMAIL: "lane@example.invalid" } });
|
|
161
|
+
assert.equal(transport.specs[1].env.GIT_AUTHOR_EMAIL, "lane@example.invalid");
|
|
162
|
+
const none = remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null, gitIdentity: () => ({ name: "", email: "" }) });
|
|
163
|
+
await none.start({ issue: 3, args: [], env: {} });
|
|
164
|
+
assert.equal("GIT_AUTHOR_EMAIL" in transport.specs[2].env, false);
|
|
165
|
+
});
|
|
166
|
+
|
|
150
167
|
it("streams the lane log into the batch log, redacted; billable minutes on the handle and on the lane result", async () => {
|
|
151
168
|
const { provider, finish, batch } = setup();
|
|
152
169
|
const lanes = providerLanes({ provider: { ...provider, start: async (a) => { const h = await provider.start(a); setTimeout(() => finish(h, { terminalReason: "merged", pr: 2 }), 5); return h; } },
|
|
@@ -255,3 +272,59 @@ describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
|
|
|
255
272
|
assert.equal(existsSync(led.path), false);
|
|
256
273
|
});
|
|
257
274
|
});
|
|
275
|
+
|
|
276
|
+
describe("pool provider (#4546)", () => {
|
|
277
|
+
const setupPool = () => {
|
|
278
|
+
const dir = tmpDir("pool-cfg-");
|
|
279
|
+
const cfg = join(dir, "router.json");
|
|
280
|
+
writeFileSync(cfg, JSON.stringify({ listen: { host: "100.64.0.1", port: 8123 } }));
|
|
281
|
+
writeFileSync(join(dir, "forgedock.token"), "pool-token-789\n");
|
|
282
|
+
return { dir, cfg };
|
|
283
|
+
};
|
|
284
|
+
const launchOnce = async (makeProvider) => {
|
|
285
|
+
const calls = [];
|
|
286
|
+
const real = globalThis.fetch;
|
|
287
|
+
globalThis.fetch = async (url, init) => { calls.push({ url: String(url), init }); return { status: 201, ok: true, headers: { get: () => null }, text: async () => "{}" }; };
|
|
288
|
+
try { await makeProvider().start({ issue: 5, args: ["--lane", "main"], env: {}, secrets: [] }); } finally { globalThis.fetch = real; }
|
|
289
|
+
return calls[0];
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
it("defaults the endpoint and token to the local router and injects the cwd forge.yaml", async () => {
|
|
293
|
+
const { cfg } = setupPool();
|
|
294
|
+
const cwd = tmpDir("pool-cwd-");
|
|
295
|
+
writeFileSync(join(cwd, "forge.yaml"), "verify:\n mode: ci\n");
|
|
296
|
+
const batch = openBatch(tmpDir("pool-b-"), "bt");
|
|
297
|
+
const make = () => providerByName("pool", { batch, repo: "o/r", cwd, exec: parseExecution({ mode: "remote", provider: "pool" }), env: {}, poolConfigPath: cfg });
|
|
298
|
+
assert.equal(make().name, "pool");
|
|
299
|
+
const c = await launchOnce(make);
|
|
300
|
+
assert.equal(c.url, "http://100.64.0.1:8123/sandboxes");
|
|
301
|
+
assert.equal(c.init.headers.authorization, "Bearer pool-token-789");
|
|
302
|
+
const spec = JSON.parse(c.init.body);
|
|
303
|
+
assert.equal(Buffer.from(spec.env.CS_FORGE_YAML_B64, "base64").toString(), "verify:\n mode: ci\n");
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
it("an explicit endpoint wins; no forge.yaml adds no env; a missing router config is misconfigured", async () => {
|
|
307
|
+
const { dir } = setupPool();
|
|
308
|
+
const batch = openBatch(tmpDir("pool-b2-"), "bt");
|
|
309
|
+
const c = await launchOnce(() => providerByName("pool", { batch, repo: "o/r", cwd: dir, exec: parseExecution({ provider: "pool", endpoint: "https://pool.example/v1" }), env: {}, poolConfigPath: join(dir, "router.json") }));
|
|
310
|
+
assert.equal(c.url, "https://pool.example/v1/sandboxes");
|
|
311
|
+
assert.equal(JSON.parse(c.init.body).env.CS_FORGE_YAML_B64, undefined);
|
|
312
|
+
assert.throws(() => providerByName("pool", { batch, repo: "o/r", cwd: dir, exec: parseExecution({ provider: "pool" }), env: {}, poolConfigPath: join(dir, "none.json") }), (e) => e.code === "execution.provider.misconfigured");
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
describe("remote capacity (GET /status, #4540)", () => {
|
|
317
|
+
const mk = (res) => httpTransport({ endpoint: "https://pool.example/v1", fetchFn: async (url) => { if (res instanceof Error) throw res; assert.equal(url, "https://pool.example/v1/status"); return { status: res.status, ok: res.status < 400, text: async () => res.body ?? "", headers: { get: () => null } }; } });
|
|
318
|
+
it("free integer is reported; 404, malformed, negative and transport errors are unknown (null)", async () => {
|
|
319
|
+
assert.deepEqual(await mk({ status: 200, body: '{"free":3}' }).capacity(), { free: 3 });
|
|
320
|
+
assert.deepEqual(await mk({ status: 200, body: '{"free":0}' }).capacity(), { free: 0 });
|
|
321
|
+
for (const r of [{ status: 404 }, { status: 200, body: "nope" }, { status: 200, body: '{"free":"2"}' }, { status: 200, body: '{"free":-1}' }, new Error("down")]) assert.equal(await mk(r).capacity(), null);
|
|
322
|
+
});
|
|
323
|
+
it("remoteProvider.capacity delegates, and is null without a transport capability or on a throw", async () => {
|
|
324
|
+
const batch = openBatch(join(tmpDir("cap-"), "batches"), "bt");
|
|
325
|
+
const p = (transport) => remoteProvider({ batch, repo: "o/r", transport, env: {}, credentials: () => null });
|
|
326
|
+
assert.deepEqual(await p({ capacity: async () => ({ free: 1 }) }).capacity(), { free: 1 });
|
|
327
|
+
assert.equal(await p({}).capacity(), null);
|
|
328
|
+
assert.equal(await p({ capacity: async () => { throw new Error("x"); } }).capacity(), null);
|
|
329
|
+
});
|
|
330
|
+
});
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* `httpTransport` is the one shipped transport: a small JSON protocol (docs/CONFIG.md `execution`) that a runner pool
|
|
16
16
|
* or a session gateway fronts. Tests use an in-memory transport.
|
|
17
17
|
*/
|
|
18
|
+
import { execFileSync } from "node:child_process";
|
|
18
19
|
import { appendFileSync, mkdirSync } from "node:fs";
|
|
19
20
|
import { dirname } from "node:path";
|
|
20
21
|
import { safeSegment } from "./ledger.mjs";
|
|
@@ -41,8 +42,9 @@ export function isActionsEndpoint(endpoint) {
|
|
|
41
42
|
/**
|
|
42
43
|
* Generic HTTP transport: POST {endpoint}/sandboxes (body = launch spec), GET /sandboxes/:id -> {state},
|
|
43
44
|
* GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
|
|
44
|
-
* GET /sandboxes/:id/usage -> {minutes}, DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
|
|
45
|
+
* GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
|
|
45
46
|
* 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
|
|
47
|
+
* Optional `GET /status -> {free: N}` is the pool's free slots (#4540); anything else (404, error, malformed) is unknown (null).
|
|
46
48
|
*/
|
|
47
49
|
export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
|
|
48
50
|
const root = String(endpoint ?? "").replace(/\/+$/, "");
|
|
@@ -70,8 +72,19 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
|
|
|
70
72
|
},
|
|
71
73
|
async logs(id, cursor = 0) { const r = await call("GET", `${p(id)}/logs?cursor=${encodeURIComponent(cursor)}`); return r.ok ? { text: String(r.data?.text ?? ""), cursor: r.data?.cursor ?? cursor } : { text: "", cursor }; },
|
|
72
74
|
async result(id) { const r = await call("GET", `${p(id)}/result`); return r.ok && r.data && typeof r.data === "object" ? r.data : null; },
|
|
73
|
-
async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes) } : null; },
|
|
75
|
+
async usage(id) { const r = await call("GET", `${p(id)}/usage`); return r.ok && Number.isFinite(Number(r.data?.minutes)) ? { minutes: Number(r.data.minutes), ...(typeof r.data.agent === "string" && r.data.agent ? { agent: r.data.agent } : {}) } : null; }, // agent: #4549, optional
|
|
74
76
|
async destroy(id) { const r = await call("DELETE", p(id)); if (r.ok || r.status === 404) return true; throw fail("destroy", r); },
|
|
77
|
+
async capacity() {
|
|
78
|
+
// `free` (#4540) and agent liveness `online`/`total` (#4547) are each optional; none valid is unknown.
|
|
79
|
+
try {
|
|
80
|
+
const r = await call("GET", "/status");
|
|
81
|
+
if (!r.ok) return null;
|
|
82
|
+
const ok = (v) => Number.isInteger(v) && v >= 0;
|
|
83
|
+
const { free, online, total } = r.data ?? {};
|
|
84
|
+
const out = { ...(ok(free) ? { free } : {}), ...(ok(online) ? { online } : {}), ...(ok(total) ? { total } : {}) };
|
|
85
|
+
return Object.keys(out).length ? out : null;
|
|
86
|
+
} catch { return null; }
|
|
87
|
+
},
|
|
75
88
|
};
|
|
76
89
|
}
|
|
77
90
|
|
|
@@ -79,7 +92,23 @@ export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fet
|
|
|
79
92
|
* The remote provider. `settings` is parseExecution's output (image, cacheStore, tokenEnv, secrets); `env` is the
|
|
80
93
|
* dispatcher's env (secret values for redaction on adopt); `credentials(name)` resolves a secret (env, then keychain).
|
|
81
94
|
*/
|
|
82
|
-
|
|
95
|
+
/** The dispatching host's git identity (`git config user.name/email`), so sandbox commits carry the same author as local lanes. */
|
|
96
|
+
export function hostGitIdentity() {
|
|
97
|
+
const get = (k) => { try { return execFileSync("git", ["config", k], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { return ""; } };
|
|
98
|
+
return { name: get("user.name"), email: get("user.email") };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** GIT_AUTHOR_* and GIT_COMMITTER_* for the sandbox, unless the lane env already sets them; empty when the host has no identity. */
|
|
102
|
+
export function gitIdentityEnv(identity, laneEnv = {}) {
|
|
103
|
+
const out = {};
|
|
104
|
+
for (const role of ["AUTHOR", "COMMITTER"]) {
|
|
105
|
+
const n = laneEnv[`GIT_${role}_NAME`] || identity?.name, e = laneEnv[`GIT_${role}_EMAIL`] || identity?.email;
|
|
106
|
+
if (n && e) { out[`GIT_${role}_NAME`] = n; out[`GIT_${role}_EMAIL`] = e; }
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function remoteProvider({ batch, repo, settings = {}, name = "remote", transport, env = process.env, credentials = (name) => resolveCredential(name, { env }), now = () => Date.now(), gitIdentity = hostGitIdentity }) {
|
|
83
112
|
if (!transport) throw new Error("remote provider needs a transport");
|
|
84
113
|
const idFor = (issue) => `fdrsbx-${safeSegment(batch.id)}-${issue}`;
|
|
85
114
|
const errs = new Map(); // sandbox id -> consecutive status errors
|
|
@@ -89,7 +118,7 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
89
118
|
if (settings.tokenEnv) remember(credentials(settings.tokenEnv));
|
|
90
119
|
const minutesOf = async (h) => {
|
|
91
120
|
let m = null;
|
|
92
|
-
try { const u = await transport.usage?.(h.id); if (u && Number.isFinite(Number(u.minutes))) m = Number(u.minutes); } catch { m = null; }
|
|
121
|
+
try { const u = await transport.usage?.(h.id); if (u && Number.isFinite(Number(u.minutes))) m = Number(u.minutes); if (typeof u?.agent === "string" && u.agent) h.agent = u.agent; } catch { m = null; }
|
|
93
122
|
if (m === null && Number.isFinite(h.startedAt)) m = Math.round(((now() - h.startedAt) / 60000) * 10) / 10;
|
|
94
123
|
if (m !== null) h.minutes = Math.max(h.minutes ?? 0, m);
|
|
95
124
|
};
|
|
@@ -102,7 +131,7 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
102
131
|
} catch { /* the next poll retries */ }
|
|
103
132
|
};
|
|
104
133
|
return {
|
|
105
|
-
name
|
|
134
|
+
name,
|
|
106
135
|
idFor,
|
|
107
136
|
async start({ issue, args = [], env: laneEnv = env, secrets = settings.secrets ?? [] }) {
|
|
108
137
|
const id = idFor(issue);
|
|
@@ -115,12 +144,13 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
115
144
|
id, issue, batch: batch.id, image: settings.image ?? null,
|
|
116
145
|
repo: { slug: repo, url: repo ? `https://github.com/${repo}.git` : null, ref: li >= 0 ? args[li + 1] ?? null : null },
|
|
117
146
|
cache: settings.cacheStore ? { store: settings.cacheStore, key: safeSegment(String(repo ?? "repo")) } : null,
|
|
118
|
-
|
|
147
|
+
// The host's git identity: without it the lane image falls back to a placeholder author that fails DCO.
|
|
148
|
+
env: { ...gitIdentityEnv(gitIdentity(), laneEnv ?? {}), ...scopedEnv({ env: resolved, secrets, batchId: batch.id, repo }), ...(settings.forgeYaml ? { CS_FORGE_YAML_B64: Buffer.from(settings.forgeYaml).toString("base64") } : {}) }, // #4546: the dispatcher's forge.yaml (gitignored in the repo)
|
|
119
149
|
command: ["forgedock", "run-issue", String(issue), ...args, "--result-file", resultFile], result_file: resultFile,
|
|
120
150
|
};
|
|
121
151
|
const startedAt = now();
|
|
122
152
|
await transport.launch(spec);
|
|
123
|
-
return { provider:
|
|
153
|
+
return { provider: name, id, issue, startedAt, minutes: 0 };
|
|
124
154
|
},
|
|
125
155
|
async status(h) {
|
|
126
156
|
let s;
|
|
@@ -134,6 +164,9 @@ export function remoteProvider({ batch, repo, settings = {}, transport, env = pr
|
|
|
134
164
|
if (r) { await pullLogs(h); await minutesOf(h); }
|
|
135
165
|
return r ?? null;
|
|
136
166
|
},
|
|
167
|
+
async capacity() {
|
|
168
|
+
try { return (await transport.capacity?.()) ?? null; } catch { return null; }
|
|
169
|
+
},
|
|
137
170
|
async teardown(h) {
|
|
138
171
|
await minutesOf(h);
|
|
139
172
|
try { return (await transport.destroy(h.id)) !== false; } catch { return false; }
|