forgedock 1.49.0 → 1.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1183,6 +1183,7 @@ function flag(argv, name) { const i = argv.indexOf(name); return i >= 0 ? argv[i
1183
1183
  * `--resume <id>` continues after the dispatcher dies, re-adopting live lanes. `deferred` lanes are requeued by code,
1184
1184
  * forge.yaml `orchestration.breaker` ({errors, window_min}, default 3 in 30) stops admissions on an engine-error
1185
1185
  * burst, and `--set-parallel <id> K` changes K of a running batch.
1186
+ * `--set-max-remote <id> N` and `--set-placement <id> remote|hybrid|local` (#5138) change its execution live (live-execution.mjs).
1186
1187
  * Maintenance budget (#4197), only with forge.yaml `maintenance.enabled: true` (or `deps.maintenance`): open issues carrying
1187
1188
  * a `maintenance.labels` label join the batch, each issue is classified from its labels (maintenance; `priority:P<n>`,
1188
1189
  * unlabelled = P2) and admission reserves `maintenance.share` of lane starts for maintenance (`when_idle` drains the rest).
@@ -1248,7 +1249,7 @@ async function envBatch({ argv, root, print = (l) => console.log(l) }) {
1248
1249
  return { ok: true, batch: id };
1249
1250
  }
1250
1251
 
1251
- const ORCH_USAGE = "usage: forgedock orchestrate <issue...> [--parallel K] [--lane L] [--lane-override] [--force <reason>] [--repo owner/repo] [--auto-children off|decomposed|epic|all] [--epic N] | --resume <id> [issue...] | --set-parallel <id> <K> | --clear-breaker <id> | --add <id> <issue...> | --requeue <id> <issue...> [--reason <text>] | --cancel <id> [--stop-remote] | --hold <id> <issue...> | --release <id> <issue...> | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY";
1252
+ const ORCH_USAGE = "usage: forgedock orchestrate <issue...> [--parallel K] [--lane L] [--lane-override] [--force <reason>] [--repo owner/repo] [--auto-children off|decomposed|epic|all] [--epic N] | --resume <id> [issue...] | --set-parallel <id> <K> | --set-max-remote <id> <N> | --set-placement <id> remote|hybrid|local | --clear-breaker <id> | --add <id> <issue...> | --requeue <id> <issue...> [--reason <text>] | --cancel <id> [--stop-remote] | --hold <id> <issue...> | --release <id> <issue...> | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY";
1252
1253
  const ORCH_VALUED = ["--parallel", "--lane", "--repo", "--model", "--prefilter-model", "--reason", "--force", "--auto-children", "--epic"];
1253
1254
  const issuesOf = (argv) => {
1254
1255
  const issues = [];
@@ -1463,6 +1464,17 @@ async function orchestrateClient(argv, deps = {}) {
1463
1464
  console.log(`batch ${id}: parallel set to ${k}`);
1464
1465
  return { ok: true, batch: id, parallel: k };
1465
1466
  }
1467
+ if (argv.includes("--set-max-remote") || argv.includes("--set-placement")) {
1468
+ // #5138: through the daemon when one runs, else the batch's `execution` file directly (nothing hosts the batch).
1469
+ const liveExec = await import("./lean/live-execution.mjs");
1470
+ const { id, patch } = liveExec.parseSetExecution(argv);
1471
+ let sock = null, execution = null;
1472
+ try { sock = await schedd.connectDaemon({ dir, spawnDaemon: null }); } catch { sock = null; }
1473
+ if (sock) execution = (await schedd.request(sock, { op: "set-execution", id, ...patch }, undefined, { timeoutMs: schedd.REQUEST_MS }))?.execution ?? null;
1474
+ else { const batchMod = await import("./lean/batch.mjs"); execution = liveExec.applySetExecution(batchMod.openBatch(batchMod.batchRoot(), id), patch); }
1475
+ console.log(`batch ${id}: ${liveExec.formatOverride(patch)}`);
1476
+ return { ok: true, batch: id, execution };
1477
+ }
1466
1478
  // #4408: --add, and the extra issues of --resume <id> N..., go to the batch (through the daemon when one runs).
1467
1479
  const addVia = (id) => async (issues) => {
1468
1480
  let sock = null;
@@ -1584,6 +1596,14 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
1584
1596
  sink.line(`batch ${b.id}: parallel set to ${k}`);
1585
1597
  return { ok: true, batch: b.id, parallel: k };
1586
1598
  }
1599
+ if (argv.includes("--set-max-remote") || argv.includes("--set-placement")) {
1600
+ // #5138: live max_remote / placement mode of a batch, re-read by its runner on every admission.
1601
+ const liveExec = await import("./lean/live-execution.mjs");
1602
+ const { id, patch } = liveExec.parseSetExecution(argv);
1603
+ const execution = liveExec.applySetExecution(batchMod.openBatch(root, id), patch);
1604
+ sink.line(`batch ${id}: ${liveExec.formatOverride(patch)}`);
1605
+ return { ok: true, batch: id, execution };
1606
+ }
1587
1607
  if (["--show-env", "--set-env", "--unset-env"].some((f) => argv.includes(f))) return envBatch({ argv, root, print: (l) => sink.line(l) });
1588
1608
  if (argv.includes("--cancel")) return cancelBatch({ id: flag(argv, "--cancel"), root, stopRemote: argv.includes("--stop-remote"), print: (l) => sink.line(l) });
1589
1609
  if (argv.includes("--requeue")) {
@@ -1774,13 +1794,14 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
1774
1794
  const relays = Array.isArray(provider.relays) ? provider.relays : [];
1775
1795
  // #5112: the host-busy guard, shared by hybrid placement and the fallback_local capacity fallback.
1776
1796
  const hostHeadroom = deps.hostHeadroom ?? (() => execMod.hostHeadroom({ cfg: readAdmission(cwd), heavyCfg: readHeavy(cwd) }));
1777
- const place = !hybrid ? (n) => execMod.placeLane({ exec, needs: needsOf.get(n) ?? [], relays }) : await (async () => {
1778
- const verifyMode = (deps.verify !== undefined ? deps.verify : readVerify(cwd))?.mode ?? "local";
1779
- let weight = deps.validationWeight ?? null;
1780
- if (!weight) { try { weight = await execMod.validationWeight({ cwd, precheckCommands: readPrecheckCommands(cwd) ?? [] }); } catch { weight = { heavy: false, signals: [] }; } }
1781
- const headroom = hostHeadroom;
1782
- return (n) => execMod.placeLane({ exec, verifyMode, labels: labelsOf.get(n) ?? [], weight, headroom: headroom(), needs: needsOf.get(n) ?? [], relays });
1783
- })();
1797
+ // #5138: placement and max_remote re-read the batch's live `execution` override (--set-placement / --set-max-remote).
1798
+ const liveExec = await import("./lean/live-execution.mjs");
1799
+ const verifyMode = (deps.verify !== undefined ? deps.verify : readVerify(cwd))?.mode ?? "local";
1800
+ const computeWeight = () => deps.validationWeight ?? execMod.validationWeight({ cwd, precheckCommands: readPrecheckCommands(cwd) ?? [] });
1801
+ const weight = liveExec.lazyWeight({ initial: hybrid ? (await Promise.resolve().then(computeWeight).catch(() => null)) ?? { heavy: false, signals: [] } : null, compute: computeWeight });
1802
+ const place = (e, n) => (e.mode !== "hybrid" ? execMod.placeLane({ exec: e, needs: needsOf.get(n) ?? [], relays })
1803
+ : execMod.placeLane({ exec: e, verifyMode, labels: labelsOf.get(n) ?? [], weight: weight.get(), headroom: hostHeadroom(), needs: needsOf.get(n) ?? [], relays }));
1804
+ if (batch.readExecution()?.provider !== true) batch.setExecution({ provider: true });
1784
1805
  // #4540: provider free slots, refreshed in the background (admission is sync); unknown (null) never blocks.
1785
1806
  let capSample = null, onlineSample = null, capAt = 0, capLast = -Infinity, capFailed = false; // #5048: capFailed = last probe rejected // #4547: online = agents alive, null unknown
1786
1807
  const capacity = () => {
@@ -1792,18 +1813,19 @@ export async function runBatch({ argv, cwd: callerCwd = process.cwd(), env: call
1792
1813
  }
1793
1814
  return capSample === null && onlineSample === null ? null : { free: capSample, online: onlineSample, at: capAt };
1794
1815
  };
1795
- remote = {
1796
- capacity, max: exec.maxRemote, isRemote: () => true, place, identity: () => holdId, ...(hybrid ? { fallbackLocal: true } : {}), ...(exec.fallbackLocal ? { offlineFallback: true, headroom: hostHeadroom, ...(exec.capacityFallbackAfter ? { capacityFallbackAfter: exec.capacityFallbackAfter } : {}) } : {}), onBudget: bud.onBudget ?? "stop", costPerMinute: bud.costPerMinute ?? 0,
1816
+ remote = liveExec.liveRemote({
1817
+ capacity, isRemote: () => true, identity: () => holdId, ...(exec.fallbackLocal ? { offlineFallback: true, headroom: hostHeadroom, ...(exec.capacityFallbackAfter ? { capacityFallbackAfter: exec.capacityFallbackAfter } : {}) } : {}), onBudget: bud.onBudget ?? "stop", costPerMinute: bud.costPerMinute ?? 0,
1797
1818
  admit: (u) => execBudget.admit(u),
1798
1819
  onUsage: (n, sb, rec) => {
1799
1820
  const startedAt = rec?.handle?.startedAt ?? rec?.startedAt ?? "?";
1800
1821
  try { execBudget.record(`${batch.id}#${sb.id ?? n}#${startedAt}`, sb); } catch { /* the batch total still counts it */ }
1801
1822
  Promise.resolve(emitSbx({ event: "SANDBOX_USAGE", issue: n, provider: sb.provider, sandbox_id: sb.id, minutes: sb.minutes, cost_usd: sb.cost_usd, batch: batch.id })).catch(() => {});
1802
1823
  },
1803
- };
1824
+ }, { exec, readOverride: batch.readExecution, place, onChange: (e) => sink.line(`execution: live placement ${e.mode}, max_remote ${e.maxRemote}`) });
1804
1825
  sink.line(`execution: ${hybrid ? "hybrid" : "remote"} via ${provider.name}, max_remote ${exec.maxRemote}`);
1805
1826
  } else {
1806
1827
  lanes = withPlacement(batchMod.detachedLanes({ batch, cwd, env: laneEnv, script: laneScript, bin: deps.laneBin, pollMs: deps.pollMs, laneArgs, spawnFn: laneSpawn }));
1828
+ if (batch.readExecution()?.provider === true) batch.setExecution({ provider: false }); // #5138: a resume without a provider refuses live remote/hybrid
1807
1829
  }
1808
1830
  // #4376: a cancel stops detached host lanes by SIGTERM to their group (pid-reuse checked by the result path); sandboxes run on unless `--cancel --stop-remote` (#4612).
1809
1831
  const { killLedgerPid } = await import("./lean/effects.mjs");
package/bin/forgedock.mjs CHANGED
@@ -3381,7 +3381,7 @@ function help() {
3381
3381
  ["npx forgedock run-issue <issue> [--lane <lane>] [--lane-override] [--path core|full] [--retry] [--wait-lease]", "Run the lean core work-on on one issue (default; --path full = legacy multi-phase pipeline; the lane comes from the issue: milestone → milestone/<slug>, else staging; --lane is an assertion, never the default branch) (--retry reopens an engine-error run, or a needs-human run once the label is cleared; --wait-lease waits for a held lease; exit 75 = deferred, lease held)"],
3382
3382
  ["npx forgedock plan validate --live [--repo owner/repo] [--json]", "Re-run the bootstrap dependency-order rules on live GitHub state: each cross-milestone Depends-on edge is ok, held or invalid (exit 1 on invalid)"],
3383
3383
  ["npx forgedock orchestrate <issue...> [--parallel K] [--lane L]", "Run core work-on on several issues with bounded concurrency and Depends-on ordering; each issue on its own lane, cross-lane dependencies reported"],
3384
- ["npx forgedock orchestrate --resume <id> [N...] | --set-parallel <id> K | --add <id> N... | --requeue <id> N... [--reason \"...\"] | --cancel <id> | --hold <id> N... | --release <id> N... | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY", "Reattach to a batch hosted by the schedd daemon (a finished one replays its report, an orphaned one is resumed with live lanes re-adopted), change its concurrency live, add issues to a running (or held) batch under its K and DAG, requeue a terminal issue (and the dependents skipped behind it) after an operator decision (refused while it is labelled needs-human or closed; a finished batch is reopened), or stop one batch without stopping schedd (running lanes torn down with their work saved, the rest held for --resume), or hold/release single issues of a batch (a held issue is not admitted until released or named by --resume), or print (secrets redacted), set or unset a batch's pinned lane env (applies to newly launched lanes)"],
3384
+ ["npx forgedock orchestrate --resume <id> [N...] | --set-parallel <id> K | --set-max-remote <id> N | --set-placement <id> remote|hybrid|local | --add <id> N... | --requeue <id> N... [--reason \"...\"] | --cancel <id> | --hold <id> N... | --release <id> N... | --show-env <id> | --set-env <id> KEY=VAL | --unset-env <id> KEY", "Reattach to a batch hosted by the schedd daemon (a finished one replays its report, an orphaned one is resumed with live lanes re-adopted), change its concurrency live, change its max_remote or placement mode live (remote/hybrid need a batch started with a remote provider; running lanes are not stopped), add issues to a running (or held) batch under its K and DAG, requeue a terminal issue (and the dependents skipped behind it) after an operator decision (refused while it is labelled needs-human or closed; a finished batch is reopened), or stop one batch without stopping schedd (running lanes torn down with their work saved, the rest held for --resume), or hold/release single issues of a batch (a held issue is not admitted until released or named by --resume), or print (secrets redacted), set or unset a batch's pinned lane env (applies to newly launched lanes)"],
3385
3385
  ["npx forgedock triage <issue...> [--lane L] [--dry-run]", "Already-fixed pre-filter: close issues a merged PR or green CI already resolved; print the rest"],
3386
3386
  ["npx forgedock probe [issue...] [--dry-run]", "Run declared acceptance probes for closed issues whose fix is on the default branch; a fail reopens the issue"],
3387
3387
  ["npx forgedock check [--base REF] [--json]", "Run the lane's frozen validate commands and change-scoped pre-PR checks locally in this worktree, as the engine will (no GitHub, no Actions)"],
@@ -19,6 +19,7 @@ import { appendFileSync, chmodSync, closeSync, existsSync, fchmodSync, fsyncSync
19
19
  import { spawn } from "node:child_process";
20
20
  import { homedir } from "node:os";
21
21
  import { dirname, join, resolve, sep } from "node:path";
22
+ import { sanitizeOverride, validateExecutionPatch } from "./live-execution.mjs";
22
23
 
23
24
  export const RAM_FS = new Set(["tmpfs", "ramfs"]);
24
25
  const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
@@ -145,6 +146,14 @@ export function openBatch(root, id) {
145
146
  if (!Number.isInteger(k) || k < 1) throw new Error("parallel must be a positive integer");
146
147
  put(f("parallel"), `${k}\n`);
147
148
  },
149
+ /** Live execution override (#5138): the `execution` file `{maxRemote?, mode?, provider?}`, read on every admission; null when missing or invalid. */
150
+ readExecution: () => sanitizeOverride(readJson(f("execution"))),
151
+ /** Merge a validated patch into the `execution` file. @returns the stored override. */
152
+ setExecution: (patch) => {
153
+ const next = { ...(sanitizeOverride(readJson(f("execution"))) ?? {}), ...validateExecutionPatch(patch) };
154
+ put(f("execution"), JSON.stringify(next) + "\n");
155
+ return next;
156
+ },
148
157
  /** Issues added by `--add` (#4408): the `add` file, sorted unique positive integers; [] when missing or invalid. */
149
158
  readAdded: () => readAdded(f("add")),
150
159
  /**
@@ -1345,3 +1345,71 @@ describe("batch plan secrets (#4787)", () => {
1345
1345
  assert.ok(fast < 10 * LAT, `first lane after ${fast} ms`);
1346
1346
  });
1347
1347
  });
1348
+
1349
+ describe("#5138: live max_remote and placement of a running batch", () => {
1350
+ const tick = () => new Promise((r) => setTimeout(r, 1));
1351
+ it("live max_remote raised from 1 to 2 admits a waiting remote lane while the first still runs", async () => {
1352
+ let m = 1, open;
1353
+ const gate = new Promise((r) => { open = r; });
1354
+ const live = new Set(), seen = [];
1355
+ const done = orchestrate({ issues: [1, 2], parallel: 2, pollMs: 1, readBody: async () => "", sleep: tick,
1356
+ remote: { get max() { return m; }, isRemote: () => true, capacity: () => null },
1357
+ runOne: async (n, _ln, ctx = {}) => { live.add(n); seen.push([n, ctx.placement?.where, [...live].sort()]); if (n === 1) await gate; live.delete(n); return { terminalReason: "merged", pr: 100 + n }; } });
1358
+ await until(() => live.has(1));
1359
+ for (let i = 0; i < 20; i++) await tick();
1360
+ assert.ok(!seen.some(([n]) => n === 2), "#2 waits: max_remote 1 is taken by #1");
1361
+ m = 2;
1362
+ await until(() => seen.some(([n]) => n === 2));
1363
+ assert.deepEqual(seen.find(([n]) => n === 2), [2, "remote", [1, 2]]);
1364
+ open();
1365
+ const rows = await done;
1366
+ assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged"]);
1367
+ });
1368
+ it("live max_remote raised from 0 to 1 admits a waiting remote lane (nothing running)", async () => {
1369
+ let m = 0;
1370
+ const seen = [];
1371
+ const done = orchestrate({ issues: [1], parallel: 2, pollMs: 1, readBody: async () => "", sleep: tick,
1372
+ remote: { get max() { return m; }, isRemote: () => true, capacity: () => null },
1373
+ runOne: async (n, _ln, ctx = {}) => { seen.push([n, ctx.placement?.where]); return { terminalReason: "merged", pr: 100 + n }; } });
1374
+ for (let i = 0; i < 20; i++) await tick();
1375
+ assert.deepEqual(seen, [], "#1 waits under max_remote 0");
1376
+ m = 1;
1377
+ const rows = await done;
1378
+ assert.deepEqual(seen, [[1, "remote"]]);
1379
+ assert.deepEqual(rows.map((r) => r.outcome), ["merged"]);
1380
+ });
1381
+ it("live max_remote lowered from 2 to 1 stops no running lane and holds new remote admissions", async () => {
1382
+ let m = 2;
1383
+ const live = new Set(), gates = new Map(), seen = [];
1384
+ const gate = (n) => { let open; const p = new Promise((r) => { open = r; }); gates.set(n, open); return p; };
1385
+ const done = orchestrate({ issues: [1, 2, 3], parallel: 3, pollMs: 1, readBody: async () => "", sleep: tick,
1386
+ remote: { get max() { return m; }, isRemote: () => true, capacity: () => null },
1387
+ runOne: async (n) => { live.add(n); seen.push([n, [...live].sort()]); if (n !== 3) await gate(n); live.delete(n); return { terminalReason: "merged", pr: 100 + n }; } });
1388
+ await until(() => live.size === 2);
1389
+ m = 1;
1390
+ gates.get(1)();
1391
+ for (let i = 0; i < 20; i++) await tick();
1392
+ assert.ok(!seen.some(([n]) => n === 3), "#3 waits while #2 holds the one remote slot");
1393
+ assert.ok(live.has(2), "#2 keeps running");
1394
+ gates.get(2)();
1395
+ const rows = await done;
1396
+ assert.deepEqual(rows.map((r) => r.outcome), ["merged", "merged", "merged"]);
1397
+ assert.deepEqual(seen.find(([n]) => n === 3)[1], [3]);
1398
+ });
1399
+ it("--set-max-remote / --set-placement write the batch's execution file; bad values and provider-less remote refused", async () => {
1400
+ const root = tmpDir("batch-live-exec-");
1401
+ const b = openBatch(root, "b1");
1402
+ b.writePlan({ argv: ["1"] });
1403
+ await quiet(() => orchestrateFromCli(["--set-max-remote", "b1", "3"], { batchRoot: root }));
1404
+ assert.deepEqual(b.readExecution(), { maxRemote: 3 });
1405
+ await quiet(() => orchestrateFromCli(["--set-placement", "b1", "local"], { batchRoot: root }));
1406
+ assert.deepEqual(b.readExecution(), { maxRemote: 3, mode: "local" });
1407
+ await assert.rejects(orchestrateFromCli(["--set-placement", "b1", "hybrid"], { batchRoot: root }), /no remote provider/);
1408
+ await assert.rejects(orchestrateFromCli(["--set-max-remote", "b1", "-1"], { batchRoot: root }), /non-negative integer/);
1409
+ await assert.rejects(orchestrateFromCli(["--set-placement", "b1", "cloud"], { batchRoot: root }), /placement must be one of/);
1410
+ await assert.rejects(orchestrateFromCli(["--set-max-remote", "nope", "1"], { batchRoot: root }), /no batch nope/);
1411
+ b.setExecution({ provider: true });
1412
+ await quiet(() => orchestrateFromCli(["--set-placement", "b1", "hybrid"], { batchRoot: root }));
1413
+ assert.equal(b.readExecution().mode, "hybrid");
1414
+ });
1415
+ });
@@ -268,7 +268,7 @@ const tmpInputProblem = (id, p) => `${id}: reads ${p}, which the command does no
268
268
 
269
269
  const E2E_TERM = String.raw`(?:e2e|end-to-end|integration|playwright|cypress|live[- ]stack)`;
270
270
  const E2E_WORD_RE = new RegExp(String.raw`\b${E2E_TERM}\b`, "i");
271
- const MOCKED_TERM = String.raw`(?:mock\w*|stub\w*|fake\w*|unit(?:[- ]tests?)?|vitest|jest|component tests?|route tests?)`;
271
+ const MOCKED_TERM = String.raw`(?:mock\w*|stub\w*|fake\w*|unit(?:[- ]?test\w*)?|(?:use[sd]?|using|via|by)\s+units|vitest|jest|component tests?|route tests?)`;
272
272
  const MOCKED_RE = new RegExp(String.raw`\b${MOCKED_TERM}\b`, "i");
273
273
  // #5116: a negated mock ("do not mock", "no stubs") or a fake clock is not a substitute for the e2e check.
274
274
  const NEG_MOCK_RE = /\b(?:do(?:es)? not|don't|never|no|not|without)\s+(?:use\s+|any\s+)?(?:mock\w*|stub\w*|fake\w*)/gi;
@@ -277,7 +277,9 @@ const NEG_DROP_RE = /\b(?:do(?:es)? not|don't|never|not)\s+(?:be\s+|been\s+|bein
277
277
  const FAKE_CLOCK_RE = /\bfake[- ](?:timers?|clocks?)\b/gi;
278
278
  // A weakener governs the e2e term only within a few words and the same clause (no . or ; between them).
279
279
  const GAP = String.raw`(?:[\s,]+\w+){0,5}?[\s,]+`;
280
- const DROP_BEFORE_RE = new RegExp(String.raw`\b(?:no|not|without|never|drop(?:ped|s|ping)?|skip(?:ped|s|ping)?|replac\w+|downgrad\w+|swap\w*)${GAP}${E2E_TERM}\b`, "i");
280
+ // #5145: a bare negator ("no new fixtures needed for e2e") stops at a preposition; the verb weakeners keep the wide GAP.
281
+ const NEG_GAP = String.raw`(?:[\s,]+(?!(?:for|to|of|in|on|at|by|with|from|via)\b)\w+){0,5}?[\s,]+`;
282
+ const DROP_BEFORE_RE = new RegExp(String.raw`\b(?:(?:no|not|without|never)${NEG_GAP}|(?:drop(?:ped|s|ping)?|skip(?:ped|s|ping)?|replac\w+|downgrad\w+|swap\w*)${GAP})${E2E_TERM}\b`, "i");
281
283
  const DROP_AFTER_RE = new RegExp(String.raw`\b${E2E_TERM}${GAP}(?:replaced|dropped|skipped|swapped|downgraded|unnecessary|(?:not|no longer)\s+(?:needed|required|necessary|run\w*|used|executed))\b`, "i");
282
284
  const INSTEAD_OF_RE = new RegExp(String.raw`\b${MOCKED_TERM}[^.;]{0,60}?\b(?:instead of|rather than|in lieu of)\b${GAP}${E2E_TERM}\b`, "i");
283
285
  const PROBE_RE = /forge-probe|acceptance[- ]probe/i;
@@ -1283,12 +1285,37 @@ export function followupItems(reqs, { now = Date.now() } = {}) {
1283
1285
  });
1284
1286
  }
1285
1287
 
1286
- /** #4388: the follow-up script's `key\tissue` lines as {[item.key]: issue}, matched on trimmed keys (the script trims what it echoes); unmatched items omitted. */
1288
+ /**
1289
+ * #5141: the FORGE:FOLLOWUP_PENDING note for follow-ups the lane could not file before or after the merge. Each row keeps the
1290
+ * requirement text and the `FORGE:DEFERRED_FROM: #<n>` marker the follow-up script looks up, so a later pass files it once.
1291
+ */
1292
+ export function renderPendingFollowups(rows, pr) {
1293
+ const flat = (s) => String(s ?? "").replace(/[\t\r\n]+/g, " ").trim();
1294
+ return [`## Follow-up pending`, `Merged by PR #${pr}, but the tracking issue could not be filed, so this issue stays open until it is. Still to file (idempotent by the marker):`, "",
1295
+ ...rows.map((r) => `- ${flat(r.id)} (${flat(r.disposition)}): ${flat(r.quote)} | why: ${flat(r.why)} | \`${r.marker}\``)].join("\n");
1296
+ }
1297
+
1298
+ /**
1299
+ * #4388: the follow-up script's `key\tissue` lines as {[item.key]: issue}; unmatched items omitted. Both sides get every
1300
+ * identity rule of scripts/file-deferred-followups.sh: trimmed (#4388), backticks stripped (#4984) and repeated leading
1301
+ * `R<n>: R<m>:` prefixes collapsed to the first (#5141: a quote starting with its own id looped on followup.unavailable).
1302
+ * An item still unmatched takes the one echoed row whose identity starts with `<item.id>:`; never when several do.
1303
+ */
1287
1304
  export function followupMap(out, items) {
1305
+ const norm = (k) => {
1306
+ let s = String(k).replace(/`/g, "").trim(), m;
1307
+ while ((m = /^(R\d+:)\s*R\d+:\s*(.*)$/s.exec(s))) s = `${m[1]} ${m[2]}`;
1308
+ return s;
1309
+ };
1288
1310
  const echoed = new Map();
1289
- const norm = (k) => String(k).replace(/`/g, "").trim(); // #4984: the script strips backticks from the label it echoes
1290
1311
  for (const l of String(out ?? "").split("\n")) { const [k, n] = l.split("\t"); if (k && /^\d+$/.test(String(n ?? "").trim())) echoed.set(norm(k), Number(n)); }
1291
- return Object.fromEntries(items.filter((i) => echoed.has(norm(i.key))).map((i) => [i.key, echoed.get(norm(i.key))]));
1312
+ const byId = (id) => { const hits = [...echoed].filter(([k]) => k.startsWith(`${id}:`)); return hits.length === 1 ? hits[0][1] : undefined; };
1313
+ const res = {};
1314
+ for (const i of items) {
1315
+ const n = echoed.get(norm(i.key)) ?? (i.id ? byId(String(i.id)) : undefined);
1316
+ if (n !== undefined) res[i.key] = n;
1317
+ }
1318
+ return res;
1292
1319
  }
1293
1320
 
1294
1321
  /**
@@ -3,7 +3,7 @@
3
3
  import { readFileSync } from "node:fs";
4
4
  import { describe, it } from "node:test";
5
5
  import assert from "node:assert/strict";
6
- import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, amendmentKindProblems, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables, applyValidateRecipes, deferChecklessPostDeploy, substituteShellProblem, reproEnvMiss, ORCHESTRATOR_BUCKET, ownerRoute, weakenedE2eDecisions } from "./conformance.mjs";
6
+ import { removedNamedPaths, removedPathsOf, shellSyntaxError, unrunnableValidate, noopCommand, substituteVerdict, describesUnrunnableCommand, vitestQuietFlag, normalizeRunnerCommand, cliUsageError, validateContract, applyAmendments, amendmentKindProblems, precheck, hotspotGrowth, reproFailures, isBugIssue, parseReview, checkCompleteness, decide, closePlan, isInvestigation, formatShadowSummary, followupItems, inIssue, isTestPath, normalizeEvidencePath, mergeReviewRows, reviewSkeleton, missingIds, resolveEvidencePath, movingBaseDiff, scopeText, evidencePathLine, evidencePathOnly, explainFailures, resolveForgedockPaths, missingTool, repoTracks, tmpdirInputs, isScopeCheck, isPureScopeCheck, dropScopeValidates, tagOutOfBand, outOfBandRequirements, isOutOfBand, followupMap, renderPendingFollowups, VERIFY_MODES, OUT_OF_BAND_RE, FOLLOWUP_CLASSES, followupClass, BUCKETS, bucketKind, bucketProblems, bucketTally, isE2eCommand, e2eVerified, impliedDocDeliverables, applyValidateRecipes, deferChecklessPostDeploy, substituteShellProblem, reproEnvMiss, ORCHESTRATOR_BUCKET, ownerRoute, weakenedE2eDecisions } from "./conformance.mjs";
7
7
 
8
8
  const ISSUE = "## Requirements\n1. The scrape route must release the spend reservation on 429.\n2. The search route must release the spend reservation on 429.\n\n- [ ] Billing pricing is unchanged.";
9
9
  const Q1 = "The scrape route must release the spend reservation on 429.";
@@ -903,6 +903,22 @@ describe("#4594 amend-requirement", () => {
903
903
  it("the reviewer prompt tells it to compare amended_from with the amendment (#4996)", () => {
904
904
  assert.match(readFileSync(new URL("./prompts.mjs", import.meta.url), "utf8"), /amended_from\.quote and amended_from\.why with amendment\.text/);
905
905
  });
906
+ it("followupMap matches a key whose quote repeats its own R<n>: prefix, as the script collapses it (#5141)", () => {
907
+ const items = followupItems([{ id: "R6", disposition: "defer", bucket: "host", quote: "R6: Fleet: x in (browser, replay) over 48h.", why: "w" }, { id: "R1", disposition: "defer", bucket: "new-issue", quote: "R3: Named accounts", why: "w" }]);
908
+ assert.deepEqual(followupMap("R6: Fleet: x in (browser, replay) over 48h.\t901\nR1: Named accounts\t902\n", items), { [items[0].key]: 901, [items[1].key]: 902 });
909
+ });
910
+
911
+ it("renderPendingFollowups lists each unfiled follow-up with its text and the FORGE:DEFERRED_FROM marker (#5141)", () => {
912
+ const md = renderPendingFollowups([{ id: "R2", disposition: "defer", quote: "a\tb\nc", why: "w", marker: "FORGE:DEFERRED_FROM: #7" }], 12);
913
+ assert.match(md, /^## Follow-up pending\nMerged by PR #12/);
914
+ assert.match(md, /\n- R2 \(defer\): a b c \| why: w \| `FORGE:DEFERRED_FROM: #7`$/);
915
+ });
916
+ it("followupMap falls back to the single echoed row carrying the item id, never to an ambiguous one (#5141)", () => {
917
+ assert.deepEqual(followupMap("R2: drifted text\t7\n", [{ id: "R2", key: "R2: original text" }]), { "R2: original text": 7 });
918
+ assert.deepEqual(followupMap("R2: a\t7\nR2: b\t8\n", [{ id: "R2", key: "R2: c" }]), {});
919
+ assert.deepEqual(followupMap("R20: a\t7\n", [{ id: "R2", key: "R2: c" }]), {});
920
+ });
921
+
906
922
  it("followupMap matches an item key whose backticks the script stripped (#4984)", () => {
907
923
  assert.deepEqual(followupMap("R2: wire pd.py in\t20\n", [{ key: "R2: wire `pd.py` in" }]), { "R2: wire `pd.py` in": 20 });
908
924
  });
@@ -1343,6 +1359,14 @@ describe("#5087 a plan decision may not weaken an e2e/integration criterion to m
1343
1359
  assert.deepEqual(weakenedE2eDecisions({ ...C, decisions: [{ id: "K1", choice, source: "doc" }] }), ["K1"], choice);
1344
1360
  }
1345
1361
  });
1362
+ it("#5145 unittest/unittests and substitute units are mocks; a negator across a preposition is not a weakener", () => {
1363
+ for (const choice of ["Replace e2e with unittest cases", "Skip e2e, covered by unittests", "e2e dropped, use units", "Use unittest cases instead of the e2e check"]) {
1364
+ assert.deepEqual(weakenedE2eDecisions({ ...C, decisions: [{ id: "K1", choice, source: "doc" }] }), ["K1"], choice);
1365
+ }
1366
+ for (const choice of ["Keep playwright e2e coverage; add unit tests (no new fixtures needed for e2e)"]) {
1367
+ assert.deepEqual(weakenedE2eDecisions({ ...C, decisions: [{ id: "K1", choice, source: "doc" }] }), [], choice);
1368
+ }
1369
+ });
1346
1370
  it("leaves unrelated decisions and contracts without decisions alone", () => {
1347
1371
  assert.deepEqual(weakenedE2eDecisions(C), []);
1348
1372
  assert.deepEqual(weakenedE2eDecisions({ ...C, decisions: [{ id: "K1", choice: "Run the Playwright e2e suite against the live stack", source: "doc" }, { id: "K3", choice: "Use unit tests with mocks for the parser", source: "spec" }] }), []);
@@ -524,8 +524,10 @@ export function remoteTransportFor(exec = {}, { transport = null, env = process.
524
524
  if (isActionsEndpoint(exec.endpoint) && exec.allowActions !== true) {
525
525
  throw typedError("execution.provider.actions_refused", "execution.endpoint is a GitHub Actions API; set execution.allow_actions: true to spend Actions minutes");
526
526
  }
527
- const token = credentials ? (exec.tokenEnv ? credentials(exec.tokenEnv) : null)
528
- : poolCredential({ provider: "remote", ...exec }, { env, router: router !== undefined ? router : poolRouterConfig({ env }) }).token;
527
+ // #5144: a getter, so the bearer is resolved on every request (a token fixed or rotated mid-batch is picked up), not once here.
528
+ const rt = credentials ? null : (router !== undefined ? router : poolRouterConfig({ env }));
529
+ const token = credentials ? () => (exec.tokenEnv ? credentials(exec.tokenEnv) : null)
530
+ : () => poolCredential({ provider: "remote", ...exec }, { env, router: rt }).token;
529
531
  return httpTransport({ endpoint: exec.endpoint, token, ...(fetchFn ? { fetchFn } : {}) });
530
532
  }
531
533
 
@@ -556,11 +558,18 @@ export function providerByName(name, opts) {
556
558
  let forgeYaml = null;
557
559
  if (pool) { try { forgeYaml = opts?.cwd ? readFileSync(join(opts.cwd, "forge.yaml"), "utf8") : null; } catch { forgeYaml = null; } }
558
560
  const endpoint = exec.endpoint ?? (pool ? router?.endpoint : undefined);
559
- const resolve = () => poolCredential({ ...exec, provider: name, ...(endpoint ? { endpoint } : {}) }, { env, router });
560
- const resolved = resolve();
561
+ // #5144: the router's token file is re-read per call (a rotated or newly written token), its endpoint stays as at construction.
562
+ const routerEligible = !(opts?.transport || (pool && exec.endpoint));
563
+ const routerNow = () => {
564
+ if (!routerEligible) return null;
565
+ const c = poolRouterConfig({ env, ...(opts?.poolConfigPath ? { path: opts.poolConfigPath } : {}) });
566
+ return c ? { endpoint: router?.endpoint ?? c.endpoint, token: c.token } : router;
567
+ };
568
+ const resolve = () => poolCredential({ ...exec, provider: name, ...(endpoint ? { endpoint } : {}) }, { env, router: routerNow() });
561
569
  // A token read from a file is resolved under a fixed name so remoteProvider redacts it like any `token_env` credential.
570
+ // #5144: set even when no token exists yet, so one that appears later is still resolved per call (and redacted).
562
571
  const fixed = pool ? "FORGEDOCK_POOL_TOKEN" : "FORGEDOCK_REMOTE_TOKEN";
563
- const tokenEnv = exec.tokenEnv ?? (resolved.token ? fixed : undefined);
572
+ const tokenEnv = exec.tokenEnv ?? fixed;
564
573
  // #4685/#5131: an empty env/keychain lookup of `token_env` falls back to `token_file`, then the (matching) router's token file.
565
574
  const credentials = (n) => (n === tokenEnv ? resolve().token : resolveCredential(n, { env })); // per call, like resolveCredential
566
575
  const settings = { ...exec, ...(endpoint ? { endpoint } : {}), ...(tokenEnv ? { tokenEnv } : {}), ...(forgeYaml ? { forgeYaml } : {}),
@@ -399,6 +399,27 @@ describe("runBatch wiring (#4253)", () => {
399
399
  assert.deepEqual(prov.calls, []);
400
400
  }
401
401
  });
402
+
403
+ it("#5138: a live --set-placement local makes the next lane of a remote batch a detached host lane", async () => {
404
+ const work = tmpDir("exec-cli-live-");
405
+ const stub = join(work, "stub.mjs");
406
+ writeFileSync(stub, `import { writeFileSync } from "node:fs"; const a = process.argv.slice(2); writeFileSync(a[a.indexOf("--result-file") + 1], JSON.stringify({ terminalReason: "merged", pr: 9 }));\n`);
407
+ const root = join(work, "batches");
408
+ let recorded = null;
409
+ const prov = fakeProvider({ finish: (n) => {
410
+ const b = openBatch(root, "blive");
411
+ recorded = b.readExecution();
412
+ if (n === 1) b.setExecution({ mode: "local" }); // the operator's --set-placement before #2 is admitted
413
+ return { terminalReason: "merged", pr: 100 + n, cost_usd: 0 };
414
+ } });
415
+ const res = await quiet(() => orchestrateFromCli(["1", "2", "--repo", "o/r", "--no-prefilter", "--parallel", "1"], base(work, { batchId: "blive", laneScript: stub,
416
+ execution: parseExecution({ mode: "remote", max_remote: "1" }), sandboxProvider: prov,
417
+ sandboxLedger: (n) => makeLedger({ root: join(work, "ledger"), batch: "blive", lane: `sandbox-${n}`, owner: "t", issue: n }) })));
418
+ assert.deepEqual(recorded, { provider: true });
419
+ assert.deepEqual(res.rows.map((r) => [r.issue, r.outcome, r.pr, r.placement?.where]), [[1, "merged", 101, "remote"], [2, "merged", 9, "local"]]);
420
+ assert.match(res.rows[1].placement.reason, /live placement/);
421
+ assert.deepEqual(prov.calls.filter((c) => c[0] === "start").map((c) => c[1]), [1]);
422
+ });
402
423
  });
403
424
 
404
425
  describe("execution budget and sandbox cost (#4255)", () => {
@@ -960,6 +981,50 @@ describe("remote/pool credential resolver (#5131)", () => {
960
981
  assert.equal(hit.seen[0].auth, "Bearer file-tok");
961
982
  });
962
983
 
984
+ it("5144: a token that appears or rotates after the provider is built is the next request's bearer", async () => {
985
+ const dir = tmpDir("router-5144-");
986
+ const tokenFile = join(dir, "forgedock.token");
987
+ writeFileSync(join(dir, "router.json"), JSON.stringify({ listen: { host: "127.0.0.1", port: 7710 } }));
988
+ writeFileSync(tokenFile, "\n"); // router config with no token yet
989
+ const hit = recorder();
990
+ const env = {};
991
+ const mk = (name, exec) => providerByName(name, { batch: { id: "b5144" }, repo: "o/r", exec: parseExecution({ mode: "remote", max_remote: "1", provider: name, ...exec }),
992
+ env, poolConfigPath: join(dir, "router.json"), fetchFn: hit.fetchFn });
993
+ for (const [name, exec] of [["remote", { endpoint: "http://127.0.0.1:7710" }], ["pool", {}]]) {
994
+ writeFileSync(tokenFile, "\n");
995
+ hit.seen.length = 0;
996
+ const p = mk(name, exec);
997
+ await p.capacity();
998
+ assert.equal(hit.seen.at(-1).auth, null, `${name}: no token at construction`);
999
+ writeFileSync(tokenFile, "router-new\n");
1000
+ await p.capacity();
1001
+ assert.equal(hit.seen.at(-1).auth, "Bearer router-new", `${name}: a token written later is sent`);
1002
+ writeFileSync(tokenFile, "router-rotated\n");
1003
+ await p.capacity();
1004
+ assert.equal(hit.seen.at(-1).auth, "Bearer router-rotated", `${name}: a rotated token is sent`);
1005
+ }
1006
+ });
1007
+
1008
+ it("5144: an env token_env that is set after construction is sent; remoteTransportFor default path resolves per call", async () => {
1009
+ const dir = routerDir();
1010
+ const hit = recorder();
1011
+ const penv = {};
1012
+ const q = providerByName("remote", { batch: { id: "b5144e" }, repo: "o/r", exec: parseExecution({ mode: "remote", max_remote: "1", provider: "remote", endpoint: "https://runner.example.com", token_env: "COMPUTESHARE_TOKEN" }),
1013
+ env: penv, poolConfigPath: join(dir, "router.json"), fetchFn: hit.fetchFn });
1014
+ await q.capacity();
1015
+ assert.equal(hit.seen.at(-1).auth, null);
1016
+ penv.COMPUTESHARE_TOKEN = "env-late";
1017
+ await q.capacity();
1018
+ assert.equal(hit.seen.at(-1).auth, "Bearer env-late");
1019
+ const tenv = {};
1020
+ const t = remoteTransportFor({ endpoint: "https://runner.example.com", tokenEnv: "COMPUTESHARE_TOKEN" }, { env: tenv, router: null, fetchFn: hit.fetchFn });
1021
+ await t.capacity();
1022
+ assert.equal(hit.seen.at(-1).auth, null);
1023
+ tenv.COMPUTESHARE_TOKEN = "env-late-2";
1024
+ await t.capacity();
1025
+ assert.equal(hit.seen.at(-1).auth, "Bearer env-late-2");
1026
+ });
1027
+
963
1028
  it("5131: remoteTransportFor's default credentials (gc) fall back to the matching router token", async () => {
964
1029
  const hit = recorder();
965
1030
  const t = remoteTransportFor({ endpoint: "http://127.0.0.1:7710" }, { env: {}, router: { endpoint: "http://127.0.0.1:7710", token: "router-tok" }, fetchFn: hit.fetchFn });
@@ -266,10 +266,16 @@ export function heavyShimScript(name, real, { node = process.execPath, script =
266
266
  return `${SHIM_HEADER}export FORGE_HEAVY_SHIM_DIR="$(cd "$(dirname "$0")" && pwd)"\nexec ${q(node)} ${q(script)} run --name ${q(name)} -- ${q(real)} "$@"\n`;
267
267
  }
268
268
 
269
- /** True when `text` is a shell-style script that looks `name` up on PATH itself (a pass-through wrapper, #4680). */
269
+ /**
270
+ * True when `text` is a shell-style script that looks `name` up on PATH itself (a pass-through wrapper, #4680). The lookup
271
+ * target may be the literal name, or (#5148) a variable/substitution when the script derives that name from `$0`
272
+ * (a `${0##…}` strip or `basename "$0"`), the shape of a wrapper symlinked under several command names.
273
+ */
270
274
  export function isPassThroughWrapper(text, name) {
271
275
  const t = String(text ?? "");
272
- return t.startsWith("#!") && t.includes(name) && /\b(command\s+-v|which|type\s+-p)\b/.test(t);
276
+ if (!t.startsWith("#!") || !/\b(command\s+-v|which|type\s+-p)\b/.test(t)) return false;
277
+ if (t.includes(name)) return true;
278
+ return /(?:command\s+-v|which|type\s+-p)\s+(?:-\s+)?["']?\$/.test(t) && /\$\{0##\*\/\}|basename\s+(?:--\s+)?["']?\$\{?0\b/.test(t);
273
279
  }
274
280
  const head = (p) => { try { return readFileSync(p, "utf8").slice(0, 4096); } catch { return ""; } };
275
281
 
@@ -332,6 +338,8 @@ export function findShimLoops(path, { names = HEAVY_BINS, find = which, read = h
332
338
  }
333
339
 
334
340
  export const SHIM_LOOP_MAX = 4;
341
+ /** #5148: hard bound on shim entries (FORGE_HEAVY_DEPTH) whatever the wrapper shape; far above legitimate `npm run` nesting. */
342
+ export const SHIM_DEPTH_MAX = 32;
335
343
  /**
336
344
  * Re-entry guard (#4680): FORGE_HEAVY_CHAIN lists `name:cmd` per shim run. A repeat of the same name+cmd where cmd is a
337
345
  * pass-through wrapper means it sent the call back to the shim; a repeat of a real binary is a legitimate nested call
@@ -342,15 +350,17 @@ export function guardChain(name, cmd, env, read = head) {
342
350
  const chain = String(env.FORGE_HEAVY_CHAIN ?? "").split("|").filter(Boolean);
343
351
  const dirs = chain.map((e) => dirname(e.slice(e.indexOf(":") + 1)));
344
352
  const loop = (why) => ({ error: `env.shim_loop: ${name} ${why} (PATH dirs: ${[env.FORGE_HEAVY_SHIM_DIR, ...dirs].filter(Boolean).join(", ") || "none"})` });
353
+ const depth = Math.max(0, Number.parseInt(env.FORGE_HEAVY_DEPTH, 10) || 0);
354
+ if (depth >= SHIM_DEPTH_MAX) return loop(`re-entered the heavy shim ${depth} levels deep (max ${SHIM_DEPTH_MAX})`);
345
355
  const repeat = chain.includes(`${name}:${cmd}`);
346
- if (repeat && !isPassThroughWrapper(read(cmd), name)) return { cmd, env };
356
+ if (repeat && !isPassThroughWrapper(read(cmd), name)) return { cmd, env: { ...env, FORGE_HEAVY_DEPTH: String(depth + 1) } };
347
357
  if (chain.length >= SHIM_LOOP_MAX) return loop(`re-entered the heavy shim more than ${SHIM_LOOP_MAX} times`);
348
358
  if (repeat) {
349
359
  const next = which(name, env.PATH ?? "", [env.FORGE_HEAVY_SHIM_DIR, ...dirs].filter(Boolean));
350
360
  if (!next) return loop(`re-entered via ${cmd} and no other ${name} is on PATH`);
351
361
  cmd = next;
352
362
  }
353
- return { cmd, env: { ...env, FORGE_HEAVY_CHAIN: [...chain, `${name}:${cmd}`].join("|") } };
363
+ return { cmd, env: { ...env, FORGE_HEAVY_DEPTH: String(depth + 1), FORGE_HEAVY_CHAIN: [...chain, `${name}:${cmd}`].join("|") } };
354
364
  }
355
365
 
356
366
  /**
@@ -6,7 +6,7 @@ import { spawnSync } from "node:child_process";
6
6
  import { writeFileSync, readFileSync, mkdirSync, chmodSync, existsSync } from "node:fs";
7
7
  import { join } from "node:path";
8
8
  import { tmpDir } from "../tests/_tmp.mjs";
9
- import { guardChain, isPassThroughWrapper, SHIM_LOOP_MAX, classifyStep, heavyConfig, openHeavy, acquireHeavy, writeHeavyShims, heavyShimScript, sumWaits, runWrapped, HEAVY_BINS, isHeavyProcess } from "./heavy.mjs";
9
+ import { guardChain, isPassThroughWrapper, SHIM_LOOP_MAX, SHIM_DEPTH_MAX, classifyStep, heavyConfig, openHeavy, acquireHeavy, writeHeavyShims, heavyShimScript, sumWaits, runWrapped, HEAVY_BINS, isHeavyProcess } from "./heavy.mjs";
10
10
  import { checkAdmission, ADMISSION_DEFAULTS, localLaneCap } from "./admission.mjs";
11
11
  import { formatAlertLines, formatWaitSummary } from "./orchestrate.mjs";
12
12
  import { readHeavy, readAdmission, liveLaneCap } from "../engine-cli.mjs";
@@ -347,3 +347,57 @@ describe("validation steps and the local lane cap (#4777)", () => {
347
347
  assert.equal(liveLaneCap(null, { cfg: readAdmission(d), host: { cores: () => 16, memAvailableMb: () => 64000 } })(), 2);
348
348
  });
349
349
  });
350
+
351
+ describe("#5148 dynamic-name pass-through wrapper", () => {
352
+ const WRAP = (log) => `#!/bin/bash\nname=\${0##*/}\necho w >> "${log}"\n[ "$(wc -l < "${log}")" -gt 12 ] && exit 99\nself_dir=$(cd "\${0%/*}" && pwd)\nreal=$(PATH=$(printf '%s' "$PATH" | tr ':' '\\n' | grep -vxF "$self_dir" | paste -sd:) command -v "$name")\nexec "$real" "$@"\n`;
353
+ const fixture = () => {
354
+ const d = tmpDir("heavy-5148-"), wrap = join(d, "wrap"), real = join(d, "real"), bin = join(d, "bin"), log = join(d, "log");
355
+ for (const x of [wrap, real, bin]) mkdirSync(x, { recursive: true });
356
+ writeFileSync(log, "");
357
+ writeFileSync(join(wrap, "docker"), WRAP(log));
358
+ writeFileSync(join(real, "docker"), `#!/bin/sh\necho r >> ${log}\necho real-docker\n`);
359
+ for (const x of [wrap, real]) chmodSync(join(x, "docker"), 0o755);
360
+ return { d, wrap, real, bin, log };
361
+ };
362
+ const lines = (log) => readFileSync(log, "utf8").split("\n").filter(Boolean);
363
+
364
+ it("#5148: a ${0##*/} pass-through wrapper on PATH does not recurse through the heavy shim", async () => {
365
+ const { wrap, real, bin, log } = fixture();
366
+ assert.ok(isPassThroughWrapper(readFileSync(join(wrap, "docker"), "utf8"), "docker"));
367
+ assert.ok(isPassThroughWrapper('#!/bin/sh\nexec "$(command -v "$(basename "$0")")" "$@"', "docker"));
368
+ assert.ok(!isPassThroughWrapper("#!/bin/sh\necho hi\n", "docker"));
369
+ assert.ok(!isPassThroughWrapper('#!/bin/sh\nx=git\nexec "$(command -v "$x")" "$@"', "docker"), "dynamic lookup without a $0-derived name is not a wrapper");
370
+ assert.deepEqual(await writeHeavyShims(bin, { path: `${wrap}:${real}`, warn: () => {} }), ["docker"]);
371
+ assert.match(readFileSync(join(bin, "docker"), "utf8"), new RegExp(`-- '${real}/docker' "\\$@"`));
372
+ const env = { ...process.env, PATH: `${bin}:${wrap}:${real}:/usr/bin:/bin`, FORGE_HEAVY_DISABLED: "1" };
373
+ const r = spawnSync(join(bin, "docker"), ["ps"], { env, encoding: "utf8", timeout: 20000 });
374
+ assert.equal(r.status, 0, r.stderr);
375
+ assert.match(r.stdout, /real-docker/);
376
+ assert.deepEqual(lines(log), ["r"], "the wrapper never ran; the real binary ran once");
377
+ });
378
+
379
+ it("#5148: with the shim baking the wrapper, the call reaches the real binary within SHIM_LOOP_MAX levels", () => {
380
+ const { wrap, real, bin, log } = fixture();
381
+ writeFileSync(join(bin, "docker"), heavyShimScript("docker", join(wrap, "docker"))); chmodSync(join(bin, "docker"), 0o755);
382
+ const env = { ...process.env, PATH: `${bin}:${wrap}:${real}:/usr/bin:/bin`, FORGE_HEAVY_DISABLED: "1" };
383
+ const r = spawnSync(join(bin, "docker"), ["ps"], { env, encoding: "utf8", timeout: 20000 });
384
+ assert.equal(r.status, 0, r.stderr);
385
+ assert.ok(lines(log).length <= SHIM_LOOP_MAX + 1, lines(log).join(","));
386
+ assert.equal(lines(log).at(-1), "r");
387
+ });
388
+
389
+ it("#5148: guardChain stops at SHIM_DEPTH_MAX even for a repeat it judges legitimate, and counts each entry", () => {
390
+ const env = { PATH: "/s:/r", FORGE_HEAVY_SHIM_DIR: "/s", FORGE_HEAVY_CHAIN: "npm:/r/npm", FORGE_HEAVY_DEPTH: String(SHIM_DEPTH_MAX) };
391
+ assert.match(guardChain("npm", "/r/npm", env, () => "").error, /^env\.shim_loop: npm .*levels deep/);
392
+ const ok = guardChain("npm", "/r/npm", { ...env, FORGE_HEAVY_DEPTH: String(SHIM_DEPTH_MAX - 1) }, () => "");
393
+ assert.equal(ok.error, undefined); assert.equal(ok.env.FORGE_HEAVY_DEPTH, String(SHIM_DEPTH_MAX));
394
+ });
395
+
396
+ it("#5148: under systemd-run TasksMax the shimmed call finishes without hitting the limit", { skip: spawnSync("systemd-run", ["--user", "--scope", "-q", "true"], { timeout: 10000 }).status !== 0 && "no systemd --user manager" }, () => {
397
+ const { wrap, real, bin, log } = fixture();
398
+ writeFileSync(join(bin, "docker"), heavyShimScript("docker", join(wrap, "docker"))); chmodSync(join(bin, "docker"), 0o755);
399
+ const r = spawnSync("systemd-run", ["--user", "--scope", "-q", "-p", "TasksMax=64", "env", `PATH=${bin}:${wrap}:${real}:/usr/bin:/bin`, "FORGE_HEAVY_DISABLED=1", join(bin, "docker"), "ps"], { encoding: "utf8", timeout: 30000 });
400
+ assert.equal(r.status, 0, r.stderr);
401
+ assert.ok(lines(log).length < 13);
402
+ });
403
+ });
package/bin/lean/lean.mjs CHANGED
@@ -15,7 +15,7 @@ import { ciPollDelayMs, ENGINE_VERSION, DEFAULT_MAX_SESSION_LIMIT_PAUSES, DEFAUL
15
15
  import { openDependency, laneMergedPrs, dependencyOnLane, isTrustedComment, decomposedSubissues, isDecomposedRecord, reopenRevokedDecomposition } from "../engine/phases.mjs";
16
16
  import { createHash, randomBytes } from "node:crypto";
17
17
  import { otherLaneOwner } from "./lanelock.mjs";
18
- import { applyValidateRecipes, deferChecklessPostDeploy, reproEnvMiss, resolveForgedockPaths, describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, amendmentKindProblems, precheck, alreadyFixedOnBase, READONLY_PROD_RE, EXTERNAL_NEED_RE, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
18
+ import { applyValidateRecipes, deferChecklessPostDeploy, reproEnvMiss, resolveForgedockPaths, describesUnrunnableCommand, removedPathsOf, TRACKED, normalizeKindModes, blockingFailures, unverifiedRequirements, mutationTargets, mutationFailures, runMutationCheck, MUTATION_SURVIVED, isBugIssue, scopeText, explainFailures, inIssue, validateContract, cliUsageError, normalizeRunnerCommand, applyAmendments, amendmentKindProblems, precheck, alreadyFixedOnBase, READONLY_PROD_RE, EXTERNAL_NEED_RE, repoTracks, dropScopeValidates, tagOutOfBand, parseReview, decide, mergeReviewRows, evidencePathOnly, closePlan, followupItems, renderPendingFollowups, bucketTally, pendingTracking, unnamedEdges, structureNote, impliedDocDeliverables } from "./conformance.mjs";
19
19
  import { checkAdmission, hostParallelism, ADMISSION_RETRY_SECONDS } from "./admission.mjs";
20
20
  import { namedPaths, isTrustedIssueComment } from "./prefilter.mjs";
21
21
  import { checksAt } from "./postdeploy-check.mjs";
@@ -319,7 +319,8 @@ export async function runCore(o) {
319
319
  const terminal = ["merged", "invalid", "needs-human", "engine-error"].includes(reason);
320
320
  // forge#4275: the typed outcome code and failure classes ride on FORGE:STATE, so the next run's context pack reads them typed.
321
321
  const typed = terminal ? { terminalCode: code ?? null, failureClasses: [...failureClasses].sort(), ...(reason === "merged" && mergedTo ? { mergedTo } : {}) } : {}; // #4711: where the change landed
322
- try { await writeLease(null, terminal ? { terminal: true, terminalReason: reason, ...typed } : {}); } catch { /* best effort */ }
322
+ const pending = pendingRecord().length ? { pendingFollowups: pendingRecord() } : {}; // #5141: unfiled follow-ups, filed later by the marker
323
+ try { await writeLease(null, terminal ? { terminal: true, terminalReason: reason, ...typed, ...pending } : pending); } catch { /* best effort */ }
323
324
  }
324
325
 
325
326
  // Pipeline trajectory (posted once at the end): phase rows, decisions, anomalies.
@@ -357,6 +358,8 @@ export async function runCore(o) {
357
358
  let seqLease = null; // #4386: the host sequence lease, held from the merge-gate renumber through the merge
358
359
  const waits = {};
359
360
  const followupsFiled = []; // #5078: {issue, class, count} per tracking issue this lane filed into, for the batch report
361
+ const pendingFollowups = []; // #5141: {req, why} whose filing kept failing in-lane; filed after the merge, else recorded
362
+ const pendingRecord = () => pendingFollowups.filter((p) => !p.req.tracked).map((p) => ({ id: p.req.id, quote: String(p.req.quote ?? ""), disposition: p.req.disposition, why: p.why, marker: `FORGE:DEFERRED_FROM: #${issue}` }));
360
363
  let precheckMs = 0, precheckDeferred = 0, precheckRan = false; // #4331: pre-PR check wall time and deferrals, for the batch report
361
364
  const addWait = (phase, ms, queued = 0) => {
362
365
  if (Number.isFinite(ms) && ms > 0) waits[phase] = (waits[phase] ?? 0) + ms;
@@ -494,7 +497,7 @@ export async function runCore(o) {
494
497
  await releaseLease(reason, code, extra.mergedTo ?? null);
495
498
  done = { terminalReason: reason, detail, branch, worktree: wt, cost_usd: usage.cost_usd, tokens: { input: usage.input, output: usage.output, cache: usage.cache }, wall_ms: now() - t0, model_ms: modelMs, phases: phaseLedger.snapshot(),
496
499
  ...(laneHead && reason !== "merged" ? { laneHead: { ...laneHead } } : {}), light, ...(lightVerdict ? { light_eligibility: lightVerdict } : {}), ...(lightUpgrade ? { light_upgrade: lightUpgrade } : {}), // #4453
497
- ...(Object.keys(waits).length ? { waits: { ...waits } } : {}), ...(followupsFiled.length ? { followups: followupsFiled.map((f) => ({ ...f })) } : {}), ...(heavyQueueMax ? { heavyQueueMax } : {}), ...(precheckRan ? { precheckMs, precheckDeferred } : {}),
500
+ ...(Object.keys(waits).length ? { waits: { ...waits } } : {}), ...(followupsFiled.length ? { followups: followupsFiled.map((f) => ({ ...f })) } : {}), ...(pendingRecord().length ? { pendingFollowups: pendingRecord() } : {}), ...(heavyQueueMax ? { heavyQueueMax } : {}), ...(precheckRan ? { precheckMs, precheckDeferred } : {}),
498
501
  ...(firstPassOf(builderTriggers) != null ? { firstPass: firstPassOf(builderTriggers) } : {}), ...(tb.record() ? { turns: tb.record() } : {}), // #4392
499
502
  ...(route ? { builderRoute: { tier: route.tier, class: route.class, escalated, model: route.model ?? "default" } } : {}),
500
503
  ...(gateRan ? { conformance: { mode: conformance === "enforce" || riskEnforced ? "enforce" : "shadow", would: shadowWould, risk: Boolean(riskEnforced), ...(Object.keys(kinds).length ? { kinds: [...failureClasses] } : {}) } } : {}), ...extra };
@@ -1404,25 +1407,54 @@ export async function runCore(o) {
1404
1407
  }
1405
1408
  return block ? await endBlocked(block, result.summary) : null;
1406
1409
  }
1407
- /** File (or append to) the parent's one batched follow-up issue for these requirements, then verify each tracking issue is open. */
1408
- async function track(reqs, why) {
1409
- const fail = (detail) => Object.assign(new Error(detail), { deferCode: OUTCOME_CODES.FOLLOWUP_UNAVAILABLE, deferDetail: `${detail}; nothing merged, rerun to retry` });
1410
+ /**
1411
+ * File (or append to) the parent's one batched follow-up issue for these requirements, then verify each tracking issue is open.
1412
+ * #5141: the whole step is retried in-lane (3 attempts, 5s then 15s): the script is idempotent (claim marker /
1413
+ * FORGE:DEFERRED_FROM lookup, checklist dedup, duplicate close), so only a persistent failure defers the run.
1414
+ * A malformed TSV (DEFERRED_TSV=INVALID) is deterministic and never retried. #5141: when the retries are spent the run
1415
+ * keeps going (keep): the untracked requirements join pendingFollowups, recorded in FORGE:STATE and filed after the merge.
1416
+ */
1417
+ async function track(reqs, why, { keep = true } = {}) {
1418
+ const fail = (detail, final = true) => Object.assign(new Error(detail), { deferCode: OUTCOME_CODES.FOLLOWUP_UNAVAILABLE, deferDetail: `${detail}; nothing merged, rerun to retry`, final });
1410
1419
  if (!io.fileFollowups) throw fail("no follow-up filing effect is configured");
1411
1420
  const items = followupItems(reqs, { now: now() });
1412
- let filed;
1413
- try { filed = await io.fileFollowups({ parent: issue, repo, items, milestone, defaultMilestone: o.followupsDefaultMilestone ?? null, lane, batchId: o.batchId ?? null, rollupLabel: o.followupsRollupLabel ?? null }); } catch (e) { throw fail(`filing the follow-up issue failed: ${String(e?.message ?? e).slice(0, 200)}`); }
1414
- for (const [i, r] of reqs.entries()) {
1415
- const n = Number(filed?.[items[i].key]);
1416
- if (!Number.isSafeInteger(n) || n <= 0) throw fail(`no tracking issue was returned for ${r.id}`);
1417
- let st = null;
1418
- try { st = JSON.parse(String(await ghc(["issue", "view", String(n), "-R", repo, "--json", "number,state"]))); } catch { st = null; }
1419
- if (String(st?.state).toUpperCase() !== "OPEN") throw fail(`tracking issue #${n} for ${r.id} is not open or unreadable`);
1420
- r.tracked = `#${n}`;
1421
- const f = followupsFiled.find((x) => x.issue === n && x.class === items[i].class); // #5078: grouped per issue and class
1422
- if (f) f.count += 1; else followupsFiled.push({ issue: n, class: items[i].class, count: 1 });
1423
- }
1424
- const buckets = Object.entries(bucketTally(reqs)).map(([k, c]) => `${k}:${c}`).join(","); // #4724
1425
- T.decisions.push(`Tracked ${reqs.map((r) => `${r.id} (${r.disposition})`).join(", ")} in ${[...new Set(reqs.map((r) => r.tracked))].join(", ")} (${why}; buckets ${buckets})`);
1421
+ const attempt = async () => {
1422
+ let filed;
1423
+ try { filed = await io.fileFollowups({ parent: issue, repo, items, milestone, defaultMilestone: o.followupsDefaultMilestone ?? null, lane, batchId: o.batchId ?? null, rollupLabel: o.followupsRollupLabel ?? null }); } catch (e) {
1424
+ throw fail(`filing the follow-up issue failed: ${String(e?.message ?? e).slice(0, 200)}`, /DEFERRED_TSV=INVALID/.test(`${e?.stdout ?? ""}${e?.message ?? ""}`));
1425
+ }
1426
+ for (const [i, r] of reqs.entries()) {
1427
+ if (recorded.has(i)) continue; // tracked in an earlier attempt: keeps its r.tracked, never re-counted
1428
+ const n = Number(filed?.[items[i].key]);
1429
+ if (!Number.isSafeInteger(n) || n <= 0) throw fail(`no tracking issue was returned for ${r.id}`, false);
1430
+ let st = null;
1431
+ try { st = JSON.parse(String(await ghc(["issue", "view", String(n), "-R", repo, "--json", "number,state"]))); } catch { st = null; }
1432
+ if (String(st?.state).toUpperCase() !== "OPEN") throw fail(`tracking issue #${n} for ${r.id} is not open or unreadable`, false);
1433
+ recorded.add(i);
1434
+ r.tracked = `#${n}`;
1435
+ const f = followupsFiled.find((x) => x.issue === n && x.class === items[i].class); // #5078: grouped per issue and class
1436
+ if (f) f.count += 1; else followupsFiled.push({ issue: n, class: items[i].class, count: 1 });
1437
+ }
1438
+ };
1439
+ const recorded = new Set();
1440
+ const waits = [5000, 15000];
1441
+ for (let k = 0; ; k++) {
1442
+ try { await attempt(); break; } catch (e) {
1443
+ if (e.final || (k >= waits.length && !keep)) throw e;
1444
+ if (k >= waits.length) { // #5141: keep the merge-capable work; the close step files these after the merge
1445
+ const left = reqs.filter((_, i) => !recorded.has(i));
1446
+ for (const r of left) if (!pendingFollowups.some((p) => p.req === r)) pendingFollowups.push({ req: r, why });
1447
+ T.anomalies.push(`follow-up filing failed ${k + 1} times (${String(e.message).slice(0, 160)}): ${left.map((r) => r.id).join(", ")} pending, filed after the merge (FORGE:DEFERRED_FROM: #${issue})`);
1448
+ break;
1449
+ }
1450
+ T.decisions.push(`Follow-up filing attempt ${k + 1} failed (${String(e.message).slice(0, 160)}); retrying in-lane in ${waits[k] / 1000}s`);
1451
+ await sleep(waits[k]);
1452
+ }
1453
+ }
1454
+ const done = reqs.filter((r) => r.tracked);
1455
+ if (!done.length) return;
1456
+ const buckets = Object.entries(bucketTally(done)).map(([k, c]) => `${k}:${c}`).join(","); // #4724
1457
+ T.decisions.push(`Tracked ${done.map((r) => `${r.id} (${r.disposition})`).join(", ")} in ${[...new Set(done.map((r) => r.tracked))].join(", ")} (${why}; buckets ${buckets})`);
1426
1458
  }
1427
1459
  /**
1428
1460
  * #4328: probe the host for the tools and modules a contract's validate commands need; #4393: then flag `$NAME`s the lane
@@ -3015,8 +3047,22 @@ export async function runCore(o) {
3015
3047
  }
3016
3048
  try { await recordStructure(pr); } catch (e) { ev({ event: "STRUCTURE_DELTA_FAILED", error: String(e?.message ?? e).slice(0, 300) }); }
3017
3049
  await deleteMergedBranch();
3050
+ let keepOpen = false;
3051
+ if (pendingRecord().length) { // #5141: the close step files what the lane could not track before the PR (idempotent by the marker)
3052
+ const left = pendingFollowups.filter((p) => !p.req.tracked);
3053
+ for (const why of new Set(left.map((p) => p.why))) {
3054
+ try { await track(left.filter((p) => p.why === why).map((p) => p.req), `${why}; filed after the merge`); } catch (e) { T.anomalies.push(`follow-up filing after the merge failed: ${String(e?.message ?? e).slice(0, 160)}`); }
3055
+ }
3056
+ for (const p of left) if (p.req.tracked && !closing.tracked.includes(p.req.tracked)) closing.tracked.push(p.req.tracked);
3057
+ const still = pendingRecord();
3058
+ if (still.length) {
3059
+ keepOpen = true; // release review #5155 (#5156): nothing re-files a pending item, so the issue stays open as the durable record
3060
+ T.decisions.push(`Follow-up still unfiled after the merge: ${still.map((x) => x.id).join(", ")}; issue left open (FORGE:FOLLOWUP_PENDING, FORGE:STATE pendingFollowups) until it is tracked`);
3061
+ try { await post("FORGE:FOLLOWUP_PENDING", renderPendingFollowups(still, pr)); } catch { /* FORGE:STATE and the lane result carry them too */ }
3062
+ }
3063
+ }
3018
3064
  const partial = closing.reason === "partial";
3019
- await ghw(["issue", "close", String(issue), "-R", repo, "--reason", partial ? "not planned" : "completed"]).catch(() => {});
3065
+ if (!keepOpen) await ghw(["issue", "close", String(issue), "-R", repo, "--reason", partial ? "not planned" : "completed"]).catch(() => {});
3020
3066
  if (partial) {
3021
3067
  try { await post("FORGE:TRACKED", `## Closed as partial\nThe investigation's stated question is not answered in this change (PR #${pr}); carried forward in ${closing.tracked.join(", ")}.`); } catch { /* the PR body links them too */ }
3022
3068
  } else if (closing.tracked.length) {
@@ -186,7 +186,9 @@ function world(o = {}) {
186
186
  transitionLabel: async (n, s) => { w.labels.push(s); w.seq.push(`label:${s}`); },
187
187
  ...(o.procs ? { proc: { inDir: async () => o.procs, alive: (pid) => w.alive.has(pid), kill: (pid, sig) => { w.kills.push([pid, sig]); w.seq.push(`kill-${sig}`); if (sig === "SIGKILL" || !o.stubborn) w.alive.delete(pid); } } } : {}),
188
188
  fs: { ...(o.wtExists ? { exists: async (p) => p.startsWith("/wt/lean-7") } : {}), stat: async (p) => (p in w.files ? { mtimeMs: w.mtimes[p] ?? 1 } : null), read: async (p) => { if (!(p in w.files)) throw new Error("ENOENT"); return w.files[p]; }, append: async (p, t) => { w.files[p] = (w.files[p] ?? "") + t; }, write: async (p, t) => { w.files[p] = t; }},
189
- fileFollowups: async ({ parent, items }) => { w.followups.push({ parent, items }); if (o.followupError) throw new Error(o.followupError); return Object.fromEntries(items.map((i) => [i.key, o.followupIssue ?? 900])); },
189
+ fileFollowups: async ({ parent, items }) => { w.followups.push({ parent, items }); if (o.followupError) throw Object.assign(new Error(o.followupError), { stdout: o.followupStdout ?? "" });
190
+ if (o.followupRet) return o.followupRet(w.followups.length, items);
191
+ if (w.followups.length <= (o.followupFailTimes ?? 0)) throw new Error("DEFERRED_LOOKUP=FAILED"); if (w.followups.length <= (o.followupMissingTimes ?? 0)) return {}; return Object.fromEntries(items.map((i) => [i.key, o.followupIssue ?? 900])); },
190
192
  };
191
193
  return w;
192
194
  }
@@ -982,21 +984,82 @@ describe("conformance gate (frozen contract, precheck, structured review)", () =
982
984
  });
983
985
  }
984
986
 
985
- it("a follow-up that cannot be filed defers the run (followup.unavailable): no PR, merge or close; never needs-human", async () => {
987
+ it("#5141 a follow-up that cannot be filed keeps the work: PR and merge proceed, the issue stays open with the unfiled item recorded, never needs-human (#5156)", async () => {
986
988
  const k = two("defer", { why: "w" });
987
989
  const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupError: "DEFERRED_LOOKUP=FAILED" });
990
+ w.leaseWrites = []; w.io.projector = { readState: async () => null, writeState: async (_i, st) => { w.leaseWrites.push(st); } };
988
991
  const r = await run(w);
989
- assert.equal(r.terminalReason, "deferred"); assert.equal(r.code, "followup.unavailable");
990
- assert.ok(!w.comments.some((c) => c.includes("FORGE:REVIEW")), "no verdict is published for a run that cannot link its tracking");
991
- assert.equal(w.prs.length, 0); assert.equal(w.merges.length, 0); assert.equal(w.closed.length, 0);
992
+ assert.equal(r.terminalReason, "merged"); assert.notEqual(r.code, "followup.unavailable");
993
+ assert.equal(w.prs.length, 1); assert.equal(w.merges.length, 1);
994
+ assert.equal(w.closed.length, 0, "release review #5155 (#5156): nothing re-files a pending item, so the issue is not closed over it");
995
+ assert.equal(w.followups.length, 6, "3 in-lane attempts before the PR, 3 more in the close step");
996
+ assert.deepEqual(r.pendingFollowups, [{ id: "R2", quote: "Existing callers of foo keep working.", disposition: "defer", why: "accepted by the conformance review", marker: "FORGE:DEFERRED_FROM: #7" }]);
997
+ assert.deepEqual(w.leaseWrites.at(-1).pendingFollowups, r.pendingFollowups, "the issue's FORGE:STATE carries the unfiled item");
998
+ const note = w.comments.find((c) => c.includes("FORGE:FOLLOWUP_PENDING"));
999
+ assert.match(note, /R2 \(defer\): Existing callers of foo keep working\..*`FORGE:DEFERRED_FROM: #7`/);
992
1000
  assert.ok(!w.edits.some((e) => e.includes("--add-label") && e.includes("needs-human")));
993
1001
  });
994
1002
 
995
- it("a tracking issue that is not open is not a tracking issue (followup.unavailable)", async () => {
1003
+ it("#5141 a follow-up that keeps failing before the PR is filed by the close step after the merge", async () => {
1004
+ const k = two("defer", { why: "w" });
1005
+ const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupMissingTimes: 3 });
1006
+ const r = await run(w);
1007
+ assert.equal(r.terminalReason, "merged"); assert.equal(w.followups.length, 4); assert.equal(r.pendingFollowups, undefined);
1008
+ assert.ok(w.comments.some((c) => c.includes("FORGE:TRACKED") && c.includes("#900")));
1009
+ assert.ok(!w.comments.some((c) => c.includes("FORGE:FOLLOWUP_PENDING")));
1010
+ assert.match(traj(w).join("\n"), /Tracked R2 \(defer\) in #900 \(accepted by the conformance review; filed after the merge/);
1011
+ });
1012
+
1013
+ it("#5141 a follow-up filing that fails once is retried in-lane, not requeued: the run proceeds and tracks once", async () => {
1014
+ const k = two("defer", { why: "w" });
1015
+ const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupFailTimes: 1 });
1016
+ const r = await run(w);
1017
+ assert.notEqual(r.code, "followup.unavailable");
1018
+ assert.equal(w.followups.length, 2, "one retry of the whole filing step");
1019
+ assert.equal(w.prs.length, 1);
1020
+ });
1021
+
1022
+ it("#5141 a follow-up filing that returns no tracking issue is retried in-lane before deferring", async () => {
1023
+ const k = two("defer", { why: "w" });
1024
+ const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupMissingTimes: 2 });
1025
+ const r = await run(w);
1026
+ assert.notEqual(r.code, "followup.unavailable");
1027
+ assert.equal(w.followups.length, 3);
1028
+ });
1029
+
1030
+ it("#5141 a requirement tracked in an earlier attempt keeps its tracking issue; the retry tracks only the rest, once each", async () => {
1031
+ const base = two("defer", { why: "w" });
1032
+ const k = { ...base, requirements: [...base.requirements, { id: "R3", quote: "Old callers log a warning.", disposition: "defer", bucket: "post-deploy", why: "w" }] };
1033
+ let views900 = 0;
1034
+ const w = world({ contract: k, body: `${ISSUE_BODY}\n- Old callers log a warning.`, issueStates: { get 900() { views900 += 1; return "OPEN"; } }, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)],
1035
+ followupRet: (call, items) => (call === 1 ? { [items[0].key]: 900 } : { [items[0].key]: 900, [items[1].key]: 901 }) });
1036
+ const r = await run(w);
1037
+ assert.notEqual(r.code, "followup.unavailable");
1038
+ assert.equal(w.followups.length, 2);
1039
+ const t = traj(w).join("\n");
1040
+ assert.match(t, /Follow-up filing attempt 1 failed \(no tracking issue was returned for R3\); retrying in-lane in 5s/);
1041
+ assert.match(t, /Tracked R2 \(defer\), R3 \(defer\) in #900, #901/);
1042
+ assert.equal(views900, 1, "R2, verified open in attempt 1, is not re-verified or re-counted by the retry");
1043
+ });
1044
+
1045
+ it("#5141 a persistent follow-up failure is retried 3 times in-lane, then the run keeps going; a malformed TSV is not retried and defers", async () => {
1046
+ const k = two("defer", { why: "w" });
1047
+ const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupMissingTimes: 99 });
1048
+ const r = await run(w);
1049
+ assert.equal(r.terminalReason, "merged"); assert.equal(w.followups.length, 6); assert.equal(w.prs.length, 1);
1050
+ assert.equal(r.pendingFollowups[0].id, "R2");
1051
+ const w2 = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], followupError: "Command failed", followupStdout: "DEFERRED_TSV=INVALID\n" });
1052
+ const r2 = await run(w2);
1053
+ assert.equal(r2.code, "followup.unavailable"); assert.equal(w2.followups.length, 1); assert.equal(w2.prs.length, 0);
1054
+ });
1055
+
1056
+ it("a tracking issue that is not open is not a tracking issue: the item stays pending and is recorded", async () => {
996
1057
  const k = two("defer", { why: "w" });
997
1058
  const w = world({ contract: k, claims: [{ id: "R1", status: "implemented" }], reviews: [conf({}, k)], issueStates: { 900: "CLOSED" } });
998
1059
  const r = await run(w);
999
- assert.equal(r.code, "followup.unavailable"); assert.equal(w.merges.length, 0);
1060
+ assert.equal(r.terminalReason, "merged"); assert.equal(r.pendingFollowups[0].id, "R2");
1061
+ assert.ok(w.comments.some((c) => c.includes("FORGE:FOLLOWUP_PENDING")));
1062
+ assert.equal(w.closed.length, 0, "left open while R2 is untracked (#5156)");
1000
1063
  });
1001
1064
 
1002
1065
  it("an existing PR gets the tracking links in its body before merge", async () => {
@@ -0,0 +1,136 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ /**
4
+ * Live execution control of a running batch (#5138): `forgedock orchestrate --set-max-remote <id> N` and
5
+ * `--set-placement <id> remote|hybrid|local`, like `--set-parallel` for K. The override lives in the batch's `execution`
6
+ * file (batch.mjs `readExecution`/`setExecution`, JSON `{maxRemote?, mode?, provider?}`); the runner re-reads it on every
7
+ * admission through the getters `liveRemote` puts on the `remote` object orchestrate gets (`max`, `fallbackLocal`, `place`).
8
+ * Running lanes are never stopped: a lower `max_remote` only holds new remote admissions. A batch that started without a
9
+ * remote provider (`provider` not true: mode local, or `max_remote` 0) refuses `remote`/`hybrid`, since its provider,
10
+ * ledger and execution budget are built once at start; start a new batch for that.
11
+ */
12
+
13
+ export const LIVE_MODES = Object.freeze(["remote", "hybrid", "local"]);
14
+ const LIGHT_WEIGHT = Object.freeze({ heavy: false, signals: [] });
15
+
16
+ /** Validate a patch `{maxRemote?, mode?, provider?}`; throws on a bad value. @returns the cleaned patch. */
17
+ export function validateExecutionPatch(patch = {}) {
18
+ const out = {};
19
+ if (patch.maxRemote !== undefined) {
20
+ const n = typeof patch.maxRemote === "string" && /^\d+$/.test(patch.maxRemote.trim()) ? parseInt(patch.maxRemote, 10) : patch.maxRemote;
21
+ if (!Number.isInteger(n) || n < 0) throw new Error("max_remote must be a non-negative integer");
22
+ out.maxRemote = n;
23
+ }
24
+ if (patch.mode !== undefined) {
25
+ if (!LIVE_MODES.includes(patch.mode)) throw new Error(`placement must be one of ${LIVE_MODES.join("|")}`);
26
+ out.mode = patch.mode;
27
+ }
28
+ if (patch.provider !== undefined) {
29
+ if (typeof patch.provider !== "boolean") throw new Error("provider must be a boolean");
30
+ out.provider = patch.provider;
31
+ }
32
+ return out;
33
+ }
34
+
35
+ /** The fields of a stored override that are valid (anything else is dropped); null when there is none. */
36
+ export function sanitizeOverride(o) {
37
+ if (!o || typeof o !== "object" || Array.isArray(o)) return null;
38
+ const out = {};
39
+ if (Number.isInteger(o.maxRemote) && o.maxRemote >= 0) out.maxRemote = o.maxRemote;
40
+ if (LIVE_MODES.includes(o.mode)) out.mode = o.mode;
41
+ if (typeof o.provider === "boolean") out.provider = o.provider;
42
+ return out;
43
+ }
44
+
45
+ /** The pinned forge.yaml `exec` with the live override's `maxRemote` and `mode` applied. */
46
+ export function effectiveExecution(exec, override) {
47
+ const o = sanitizeOverride(override) ?? {};
48
+ return { ...(exec ?? {}), ...(o.maxRemote !== undefined ? { maxRemote: o.maxRemote } : {}), ...(o.mode !== undefined ? { mode: o.mode } : {}) };
49
+ }
50
+
51
+ /**
52
+ * `--set-max-remote <id> N` / `--set-placement <id> MODE` in argv → `{id, patch}`, or null when neither flag is there.
53
+ * Throws on a missing id or a bad value.
54
+ */
55
+ export function parseSetExecution(argv = []) {
56
+ const at = (flag) => argv.indexOf(flag);
57
+ let id = null;
58
+ const patch = {};
59
+ for (const [flag, key] of [["--set-max-remote", "maxRemote"], ["--set-placement", "mode"]]) {
60
+ const i = at(flag);
61
+ if (i < 0) continue;
62
+ const bid = argv[i + 1], val = argv[i + 2];
63
+ if (!bid || val === undefined) throw new Error(`usage: forgedock orchestrate ${flag} <id> ${key === "mode" ? LIVE_MODES.join("|") : "N"}`);
64
+ if (id !== null && id !== bid) throw new Error("--set-max-remote and --set-placement must name the same batch");
65
+ id = bid;
66
+ patch[key] = key === "maxRemote" ? (/^\d+$/.test(val) ? parseInt(val, 10) : NaN) : val;
67
+ }
68
+ return id === null ? null : { id, patch: validateExecutionPatch(patch) };
69
+ }
70
+
71
+ /**
72
+ * Apply a patch to `batch` (batch.mjs openBatch): refused for a missing batch, and `remote`/`hybrid` for a batch whose
73
+ * `execution` file does not record `provider: true`. @returns the stored override.
74
+ */
75
+ export function applySetExecution(batch, patch) {
76
+ if (!batch.exists()) throw new Error(`no batch ${batch.id}`);
77
+ const p = validateExecutionPatch(patch);
78
+ if ((p.mode === "remote" || p.mode === "hybrid") && batch.readExecution()?.provider !== true) {
79
+ throw new Error(`batch ${batch.id} has no remote provider; placement ${p.mode} needs one: start a new batch with forge.yaml execution`);
80
+ }
81
+ return batch.setExecution(p);
82
+ }
83
+
84
+ /** Text for a stored override: `max_remote 2, placement hybrid`. */
85
+ export function formatOverride(o) {
86
+ const s = sanitizeOverride(o) ?? {};
87
+ return [s.maxRemote !== undefined ? `max_remote ${s.maxRemote}` : null, s.mode ? `placement ${s.mode}` : null].filter(Boolean).join(", ") || "no change";
88
+ }
89
+
90
+ /**
91
+ * Hybrid validation weight (#4256) that is computed only when first needed: `initial` (already computed) or `compute()`
92
+ * started on the first `get()`; until it resolves (or when it fails) a light weight is used.
93
+ */
94
+ export function lazyWeight({ initial = null, compute = null } = {}) {
95
+ let w = initial, started = initial !== null;
96
+ return {
97
+ get: () => {
98
+ if (!started && typeof compute === "function") {
99
+ started = true;
100
+ Promise.resolve().then(compute).then((r) => { w = r ?? LIGHT_WEIGHT; }, () => { w = LIGHT_WEIGHT; });
101
+ }
102
+ return w ?? LIGHT_WEIGHT;
103
+ },
104
+ };
105
+ }
106
+
107
+ /**
108
+ * Put live `max`, `fallbackLocal` and `place` getters on `base` (the `remote` object orchestrate gets; never spread it
109
+ * afterwards, a spread freezes the getters). `place(exec, n)` places one lane under the effective exec (execution.mjs
110
+ * `placeLane` with the remote or hybrid facts); mode `local` places every new lane on the host. `onChange(exec)` is
111
+ * called once each time the effective `{mode, maxRemote}` differs from the last one seen (the pinned one first).
112
+ */
113
+ export function liveRemote(base, { exec, readOverride = () => null, place, onChange = null }) {
114
+ const keyOf = (e) => `${e.mode}|${e.maxRemote}`;
115
+ let last = keyOf(effectiveExecution(exec, null));
116
+ const current = () => {
117
+ let o = null;
118
+ try { o = readOverride() ?? null; } catch { o = null; }
119
+ const e = effectiveExecution(exec, o);
120
+ const k = keyOf(e);
121
+ if (k !== last) { last = k; try { onChange?.(e); } catch { /* advisory */ } }
122
+ return e;
123
+ };
124
+ return Object.defineProperties(base, {
125
+ max: { get: () => current().maxRemote, enumerable: true, configurable: true },
126
+ fallbackLocal: { get: () => { const e = current(); return e.mode === "hybrid" && Number.isInteger(e.maxRemote) && e.maxRemote >= 1; }, enumerable: true, configurable: true },
127
+ place: {
128
+ value: (n) => {
129
+ const e = current();
130
+ if (e.mode === "local") return { where: "local", reason: "live placement: local" };
131
+ return place(e, n);
132
+ },
133
+ enumerable: true, configurable: true, writable: true,
134
+ },
135
+ });
136
+ }
@@ -0,0 +1,83 @@
1
+ // SPDX-FileCopyrightText: Copyright (c) RapierCraft Studios
2
+ // SPDX-License-Identifier: AGPL-3.0-or-later
3
+ import { describe, it } from "node:test";
4
+ import assert from "node:assert/strict";
5
+ import { tmpDir } from "../tests/_tmp.mjs";
6
+ import { openBatch } from "./batch.mjs";
7
+ import { validateExecutionPatch, sanitizeOverride, effectiveExecution, parseSetExecution, applySetExecution, formatOverride, lazyWeight, liveRemote, LIVE_MODES } from "./live-execution.mjs";
8
+
9
+ describe("#5138 live execution control", () => {
10
+ it("validates patches: max_remote a non-negative integer, mode remote|hybrid|local", () => {
11
+ assert.deepEqual(LIVE_MODES, ["remote", "hybrid", "local"]);
12
+ assert.deepEqual(validateExecutionPatch({ maxRemote: 0, mode: "local" }), { maxRemote: 0, mode: "local" });
13
+ assert.deepEqual(validateExecutionPatch({ maxRemote: "3" }), { maxRemote: 3 });
14
+ for (const bad of [{ maxRemote: -1 }, { maxRemote: 1.5 }, { maxRemote: "x" }]) assert.throws(() => validateExecutionPatch(bad), /non-negative integer/);
15
+ assert.throws(() => validateExecutionPatch({ mode: "cloud" }), /placement must be one of/);
16
+ assert.throws(() => validateExecutionPatch({ provider: "yes" }), /boolean/);
17
+ assert.deepEqual(sanitizeOverride({ maxRemote: -2, mode: "x", provider: true }), { provider: true });
18
+ assert.equal(sanitizeOverride(null), null);
19
+ });
20
+
21
+ it("merges the override over the pinned exec", () => {
22
+ const exec = { mode: "remote", maxRemote: 4, provider: "container" };
23
+ assert.deepEqual(effectiveExecution(exec, null), exec);
24
+ assert.deepEqual(effectiveExecution(exec, { maxRemote: 1, mode: "hybrid" }), { mode: "hybrid", maxRemote: 1, provider: "container" });
25
+ assert.equal(formatOverride({ maxRemote: 2, mode: "hybrid" }), "max_remote 2, placement hybrid");
26
+ });
27
+
28
+ it("parses --set-max-remote / --set-placement", () => {
29
+ assert.equal(parseSetExecution(["--set-parallel", "b", "2"]), null);
30
+ assert.deepEqual(parseSetExecution(["--set-max-remote", "b1", "2"]), { id: "b1", patch: { maxRemote: 2 } });
31
+ assert.deepEqual(parseSetExecution(["--set-placement", "b1", "hybrid", "--set-max-remote", "b1", "0"]), { id: "b1", patch: { maxRemote: 0, mode: "hybrid" } });
32
+ assert.throws(() => parseSetExecution(["--set-max-remote", "b1", "-1"]), /non-negative/);
33
+ assert.throws(() => parseSetExecution(["--set-placement", "b1"]), /usage/);
34
+ assert.throws(() => parseSetExecution(["--set-placement", "b1", "x"]), /placement must be/);
35
+ assert.throws(() => parseSetExecution(["--set-placement", "b1", "local", "--set-max-remote", "b2", "1"]), /same batch/);
36
+ });
37
+
38
+ it("applySetExecution: missing batch and a provider-less batch going remote/hybrid are refused", () => {
39
+ const root = tmpDir("live-exec-");
40
+ assert.throws(() => applySetExecution(openBatch(root, "nope"), { mode: "local" }), /no batch nope/);
41
+ const b = openBatch(root, "b1");
42
+ b.writePlan({ argv: ["1"] });
43
+ for (const mode of ["remote", "hybrid"]) assert.throws(() => applySetExecution(b, { mode }), /no remote provider/);
44
+ assert.deepEqual(applySetExecution(b, { mode: "local", maxRemote: 0 }), { mode: "local", maxRemote: 0 });
45
+ b.setExecution({ provider: true });
46
+ assert.deepEqual(applySetExecution(b, { mode: "hybrid" }), { mode: "hybrid", maxRemote: 0, provider: true });
47
+ assert.deepEqual(b.readExecution(), { mode: "hybrid", maxRemote: 0, provider: true });
48
+ });
49
+
50
+ it("liveRemote getters follow the override file; local places on the host; onChange once per change", () => {
51
+ let o = null;
52
+ const changes = [], placed = [];
53
+ const r = liveRemote({ onBudget: "stop" }, { exec: { mode: "remote", maxRemote: 2 }, readOverride: () => o, onChange: (e) => changes.push(`${e.mode}/${e.maxRemote}`),
54
+ place: (e, n) => { placed.push([e.mode, n]); return { where: "remote", reason: `mode ${e.mode}` }; } });
55
+ assert.equal(r.max, 2);
56
+ assert.equal(r.fallbackLocal, false);
57
+ assert.deepEqual(r.place(1), { where: "remote", reason: "mode remote" });
58
+ o = { maxRemote: 5 };
59
+ assert.equal(r.max, 5);
60
+ o = { maxRemote: 5, mode: "hybrid" };
61
+ assert.equal(r.fallbackLocal, true);
62
+ r.place(2);
63
+ o = { maxRemote: 5, mode: "local" };
64
+ assert.deepEqual(r.place(3), { where: "local", reason: "live placement: local" });
65
+ assert.deepEqual(placed, [["remote", 1], ["hybrid", 2]]);
66
+ assert.deepEqual(changes, ["remote/5", "hybrid/5", "local/5"]);
67
+ assert.equal({ ...r }.onBudget, "stop");
68
+ });
69
+
70
+ it("lazyWeight is light until the computed weight resolves, and computes once", async () => {
71
+ let calls = 0;
72
+ const w = lazyWeight({ compute: async () => { calls++; return { heavy: true, signals: ["tsc"] }; } });
73
+ assert.deepEqual(w.get(), { heavy: false, signals: [] });
74
+ await new Promise((r) => setImmediate(r));
75
+ assert.deepEqual(w.get(), { heavy: true, signals: ["tsc"] });
76
+ w.get();
77
+ assert.equal(calls, 1);
78
+ assert.deepEqual(lazyWeight({ initial: { heavy: true, signals: [] } }).get(), { heavy: true, signals: [] });
79
+ const failing = lazyWeight({ compute: () => { throw new Error("scan"); } });
80
+ failing.get(); await new Promise((r) => setImmediate(r));
81
+ assert.deepEqual(failing.get(), { heavy: false, signals: [] });
82
+ });
83
+ });
@@ -31,6 +31,8 @@
31
31
  * runs local in hybrid (`fallbackLocal`, local account not held) or waits for the reset; admission is not held. An `engine-error` is
32
32
  * relaunched once; `breaker` ({errors, window_min}) stops admissions after a burst and lets running lanes finish, the
33
33
  * rest stay pending (outcome `held`) for a resume. `getParallel()` is read on every pass, so K can change live.
34
+ * #5138: `remote.max`, `remote.fallbackLocal` and `remote.place` are read on every grant too (engine-cli puts live getters on
35
+ * them, live-execution.mjs), so `--set-max-remote` / `--set-placement` change a running batch; running lanes are not stopped.
34
36
  * `getChildren()` + `autoChildren {mode, pollS, epics}` (#4965, `orchestration.auto_add_children`): with mode `epic`|`all`, at most once per `pollS`
35
37
  * it reads one `{list, elsewhere}` and adds the open children (`**Parent**: #P`, sub-issues) of every batch issue and `epics` through the same
36
38
  * path as `getAdded` (one decision and one `added` row listing the edges); mode `off` also stops the decomposed-lane join.
@@ -319,7 +321,9 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
319
321
  const integHeld = new Map(); // issue -> reason (red integration streak on its milestone lane, #4667) holding it this pass
320
322
  const reg = sched?.registry ?? null;
321
323
  const priorities = toMap(saved?.priorities); // issue -> priority number; only with the registry
322
- const remoteMax = remote && Number.isInteger(remote.max) && remote.max >= 0 ? remote.max : 0;
324
+ // #5138: read on every grant, so `--set-max-remote` reaches a running batch (a getter on `remote`); running lanes are never stopped.
325
+ const remoteMax = () => { const m = remote?.max; return Number.isInteger(m) && m >= 0 ? m : 0; };
326
+ const remoteCapped = new Set(); // #5138: issues waiting for a max_remote slot; polled so a raised live max_remote admits them
323
327
  let poolOfflineSince = null, poolStallNoted = false; // #4788: when sampled online first read 0, and whether the stall was reported
324
328
  const capWait = makeCapWait({ now }); // #4909: a lane refused for capacity waits for a free slot, recorded as queue time
325
329
  let capAt, capAdmits = 0; // #4540: provider capacity sample and the remote admissions made since it
@@ -601,11 +605,11 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
601
605
  } else if (cap && Number.isInteger(cap.free)) {
602
606
  if (cap.at !== capAt) { capAt = cap.at; capAdmits = 0; }
603
607
  if (capWait.blocked(n, cap)) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
604
- if (capAdmits >= cap.free && remoteRun.size < remoteMax) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
608
+ if (capAdmits >= cap.free && remoteRun.size < remoteMax()) { notBefore.set(n, now() + Math.max(pollMs, 1)); return false; }
605
609
  }
606
- if (p.where === "remote" && remoteRun.size < remoteMax) { capWait.admitted(n); remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
607
- if (p.where === "remote" && !remote.fallbackLocal) return false;
608
- if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax} reached (${p.reason})` }; // #4256: hybrid runs it on the host
610
+ if (p.where === "remote" && remoteRun.size < remoteMax()) { remoteCapped.delete(n); capWait.admitted(n); remoteRun.add(n); capAdmits++; placed.set(n, p); return true; }
611
+ if (p.where === "remote" && !remote.fallbackLocal) { remoteCapped.add(n); return false; }
612
+ if (p.where === "remote") p = { where: "local", reason: `fallback to local: max_remote ${remoteMax()} reached (${p.reason})` }; // #4256: hybrid runs it on the host
609
613
  } else {
610
614
  noteBudget(adm.reason);
611
615
  if (remote.onBudget !== "local" && !remote.fallbackLocal) { budgetStop = adm.reason; return false; }
@@ -1345,6 +1349,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
1345
1349
  if (holdUntil && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4446: a cleared hold (forgedock limits clear) admits on the next poll
1346
1350
  if ((getParallel || getHeld) && pending.length) wake = Math.min(wake ?? pollMs, pollMs);
1347
1351
  if (offered.size && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // a slot another batch frees
1352
+ if (pending.some((n) => remoteCapped.has(n))) wake = Math.min(wake ?? pollMs, pollMs); // #5138: a live --set-max-remote raise
1348
1353
  if ((getAdded || getRequeued || (getChildren && addsEpicChildren(autoChildren?.mode))) && running.size) wake = Math.min(wake ?? pollMs, pollMs); // #4408/#4550: an add or requeue while only lanes run
1349
1354
  if ((pfAll || pfRemote) && pending.length) wake = Math.min(wake ?? pollMs, pollMs); // #4866
1350
1355
  if (baseWait && pending.length && baseHeld.size) wake = Math.min(wake ?? pollMs, pollMs); // #5031: a landed commit releases on the next tick
@@ -1366,6 +1371,7 @@ export async function orchestrate({ issues, parallel = 3, runOne, readBody, pref
1366
1371
  else if (pending.length && budgetHeld) { if (await drainAdds()) continue; break; } // maintenance the budget does not admit now: held for a later batch
1367
1372
  else if (pending.length && opHeldHit.size && !pending.some((n) => !opHeldHit.has(n) && isReady(n))) { if (await drainAdds()) continue; break; } // #4592: held by the operator: pending for a --resume
1368
1373
  else if (pending.length && (driftHeld.size || integHeld.size)) { if (await drainAdds()) continue; break; } // held by a drift flag (block_dispatch) or a red integration streak (#4667): pending for a later batch
1374
+ else if (pending.some((n) => remoteCapped.has(n))) await nap(wake ?? pollMs); // #5138: live max_remote 0: wait for a raise or another placement
1369
1375
  else if (pending.length) { // nothing runnable, nothing running, nothing waiting on time: a dependency cycle
1370
1376
  for (const n of pending.splice(0)) skip(n, "dependency cycle");
1371
1377
  save();
@@ -208,6 +208,27 @@ describe("remote adapter (#4254)", () => {
208
208
  await assert.rejects(r.status("x"), /router auth/);
209
209
  });
210
210
 
211
+ it("5144 httpTransport: a function token is resolved per call", async () => {
212
+ let cur = null;
213
+ const seen = [];
214
+ const t = httpTransport({ endpoint: "https://pool.example", token: () => cur, fetchFn: async (url, init) => { seen.push(init.headers.authorization ?? null); return { status: 200, ok: true, text: async () => JSON.stringify({ state: "running" }), headers: { get: () => null } }; } });
215
+ await t.status("x"); cur = "tok-new"; await t.status("x"); cur = "tok-rotated"; await t.status("x"); cur = null; await t.status("x");
216
+ assert.deepEqual(seen, [null, "Bearer tok-new", "Bearer tok-rotated", null]);
217
+ });
218
+
219
+ it("5144 remoteProvider: a token rotated after construction is redacted from pulled logs", async () => {
220
+ const batch = openBatch(join(tmpDir("remote-5144-"), "batches"), "bt");
221
+ const transport = fakeTransport();
222
+ let tok = null;
223
+ const p = remoteProvider({ batch, repo: "o/r", settings: { tokenEnv: "FD_TOKEN" }, transport, env: {}, credentials: (n) => (n === "FD_TOKEN" ? tok : null), gitIdentity: () => ({}) });
224
+ tok = "rotated-secret-789";
225
+ const h = await p.start({ issue: 3, args: [], env: {}, secrets: [] });
226
+ transport.boxes.get(h.id).log = `leak ${tok}\n`;
227
+ await p.status(h);
228
+ const log = readFileSync(batch.logFile(3), "utf8");
229
+ assert.equal(log.includes(tok), false);
230
+ });
231
+
211
232
  it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
212
233
  const calls = [];
213
234
  const reply = (status, body, headers = {}) => ({ status, ok: status >= 200 && status < 300, text: async () => (body === undefined ? "" : JSON.stringify(body)), headers: { get: (k) => headers[k] ?? null } });
@@ -43,7 +43,8 @@ export function isActionsEndpoint(endpoint) {
43
43
  /**
44
44
  * Generic HTTP transport: POST {endpoint}/sandboxes (body = launch spec), GET /sandboxes/:id -> {state},
45
45
  * GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
46
- * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
46
+ * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`
47
+ * (`token` is a string, or a function called per request, #5144).
47
48
  * 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
48
49
  * Optional `POST /sandboxes/:id/stop` (#4612) asks the pool to SIGTERM the lane so its teardown checkpoints and it reports `deferred run.interrupted`; 2xx = sent, 404/405 = unsupported or gone (never a DELETE fallback).
49
50
  * Optional relay (#4778): `GET /sandboxes/:id/relay -> {requests: [{rid, argv}]}` lists a sandbox's pending read-only capability
@@ -53,7 +54,9 @@ export function isActionsEndpoint(endpoint) {
53
54
  export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
54
55
  const root = String(endpoint ?? "").replace(/\/+$/, "");
55
56
  const call = async (method, path, body) => {
56
- const res = await fetchFn(`${root}${path}`, { method, headers: { accept: "application/json", ...(body ? { "content-type": "application/json" } : {}), ...(token ? { authorization: `Bearer ${token}` } : {}) },
57
+ // #5144: `token` may be a function, resolved on every request so a token fixed or rotated mid-batch is the next bearer.
58
+ const bearer = typeof token === "function" ? token() : token;
59
+ const res = await fetchFn(`${root}${path}`, { method, headers: { accept: "application/json", ...(body ? { "content-type": "application/json" } : {}), ...(bearer ? { authorization: `Bearer ${bearer}` } : {}) },
57
60
  ...(body ? { body: JSON.stringify(body) } : {}) });
58
61
  let data = null;
59
62
  try { const t = await res.text(); data = t ? JSON.parse(t) : null; } catch { data = null; }
@@ -195,6 +198,7 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
195
198
  const id = idFor(issue);
196
199
  // Each allowlisted secret from the env, else the keychain (#4254); nothing else from the host.
197
200
  const resolved = {};
201
+ if (settings.tokenEnv) remember(credentials(settings.tokenEnv)); // #5144: a token rotated since construction is redacted too
198
202
  for (const k of secrets) { const v = laneEnv?.[k] !== undefined && laneEnv[k] !== "" ? String(laneEnv[k]) : credentials(k); if (v) { resolved[k] = v; remember(v); } }
199
203
  const li = args.indexOf("--lane");
200
204
  const resultFile = `/result/${issue}.json`;
@@ -11,6 +11,7 @@
11
11
  * {op:"attach", id, resume?} → (`resume` also clears a pending cancel, #4592) log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
12
12
  * ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
13
13
  * {op:"set-parallel", id, k} → {type:"ok", parallel}
14
+ * {op:"set-execution", id, maxRemote?, mode?} → {type:"ok", execution} (#5138): live max_remote / placement (remote|hybrid|local); remote/hybrid refused for a batch with no provider
14
15
  * {op:"requeue", id, issues, reason, by} → {type:"ok", requeued, refused, hosted, reopened} (#4550): see batch.requeue; a finished batch is hosted again
15
16
  * {op:"add", id, issues} → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
16
17
  * hosted runner reads on its next pass and an unhosted batch on its next resume
@@ -42,6 +43,7 @@ import { fileURLToPath } from "node:url";
42
43
  import { schedDir, pidAlive } from "./sched.mjs";
43
44
  import { HEARTBEAT_MS, writeHeartbeat, clearHeartbeat } from "./schedd-health.mjs";
44
45
  import { batchRoot, listBatches, openBatch, SECRET_ENV } from "./batch.mjs";
46
+ import { applySetExecution } from "./live-execution.mjs";
45
47
 
46
48
  export const IDLE_MS = 10 * 60 * 1000;
47
49
  export const STOP_MS = 10 * 1000;
@@ -250,6 +252,13 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
250
252
  b.setParallel(Number(req.k));
251
253
  return send({ type: "ok", parallel: Number(req.k) }, true);
252
254
  }
255
+ if (req.op === "set-execution") {
256
+ // #5138: live max_remote / placement mode; the hosted runner re-reads the batch's `execution` file on every admission.
257
+ const b = openBatch(root, req.id);
258
+ if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
259
+ const execution = applySetExecution(b, { ...(req.maxRemote !== undefined ? { maxRemote: req.maxRemote } : {}), ...(req.mode !== undefined ? { mode: req.mode } : {}) });
260
+ return send({ type: "ok", execution }, true);
261
+ }
253
262
  if (req.op === "add") {
254
263
  const b = openBatch(root, req.id);
255
264
  if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -100,7 +100,8 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
100
100
  (live lanes are re-adopted, nothing is launched twice). The runner is hosted by the detached `forgedock schedd` daemon,
101
101
  which resumes every daemon-hosted batch that has no report yet when it starts, so a closed session needs no manual
102
102
  `--resume`; a batch started without the daemon still needs `forgedock orchestrate --resume <id>`. Change K of a running batch with
103
- `forgedock orchestrate --set-parallel <id> K`. Issues filed while a batch runs join it with
103
+ `forgedock orchestrate --set-parallel <id> K`. Change its remote lane cap or placement with `forgedock orchestrate --set-max-remote <id> N` and
104
+ `--set-placement <id> remote|hybrid|local` (remote/hybrid only for a batch started with a remote provider; running lanes keep running). Issues filed while a batch runs join it with
104
105
  `forgedock orchestrate --add <id> N...` (or `--resume <id> N...`), under its K and DAG; never start a second batch
105
106
  for them. After a decision (a `FORGE:DECISION` posted and `needs-human` removed), a blocked issue that already ended in the batch
106
107
  (`needs-human`, `invalid`, `skipped`, `dependency-*`, `ci-pending`, `deferred`) returns to it with
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "forgedock",
3
- "version": "1.49.0",
3
+ "version": "1.50.0",
4
4
  "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
5
5
  "bin": {
6
6
  "forgedock": "bin/forgedock.mjs"