forgedock 1.49.0 → 1.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/bin/engine/outcome-codes.mjs +2 -0
  2. package/bin/engine-cli.mjs +123 -20
  3. package/bin/forgedock.mjs +31 -3
  4. package/bin/lean/batch-health.mjs +219 -0
  5. package/bin/lean/batch-health.test.mjs +156 -0
  6. package/bin/lean/batch.mjs +12 -3
  7. package/bin/lean/batch.test.mjs +304 -8
  8. package/bin/lean/conformance.mjs +79 -7
  9. package/bin/lean/conformance.test.mjs +63 -1
  10. package/bin/lean/context.mjs +8 -2
  11. package/bin/lean/context.test.mjs +11 -1
  12. package/bin/lean/credentials.mjs +18 -0
  13. package/bin/lean/execution.mjs +23 -9
  14. package/bin/lean/execution.test.mjs +93 -0
  15. package/bin/lean/fixclosereasons.mjs +52 -0
  16. package/bin/lean/fixclosereasons.test.mjs +64 -0
  17. package/bin/lean/heavy.mjs +14 -4
  18. package/bin/lean/heavy.test.mjs +55 -1
  19. package/bin/lean/lean.mjs +93 -36
  20. package/bin/lean/lean.test.mjs +143 -19
  21. package/bin/lean/live-execution.mjs +136 -0
  22. package/bin/lean/live-execution.test.mjs +83 -0
  23. package/bin/lean/orchestrate-blocked.test.mjs +47 -0
  24. package/bin/lean/orchestrate.mjs +135 -17
  25. package/bin/lean/prefilter.mjs +4 -3
  26. package/bin/lean/prefilter.test.mjs +5 -4
  27. package/bin/lean/prompts.mjs +4 -2
  28. package/bin/lean/recovery.test.mjs +5 -5
  29. package/bin/lean/sandbox-provider.contract.test.mjs +36 -1
  30. package/bin/lean/sandbox-remote.mjs +6 -2
  31. package/bin/lean/schedd.mjs +9 -0
  32. package/bin/pool/agent.mjs +3 -1
  33. package/bin/pool/router.mjs +72 -7
  34. package/bin/review-staging.mjs +2 -0
  35. package/bin/runner.mjs +24 -0
  36. package/commands/orchestrate.md +2 -1
  37. package/commands/review-pr-staging.md +2 -2
  38. package/docs/CONFIG.md +39 -1
  39. package/docs/OUTCOME-CODES.md +2 -1
  40. package/package.json +1 -1
  41. package/pool/README.md +2 -0
  42. package/pool/image/cs-cred +8 -1
  43. package/scripts/review-staging-post.sh +70 -25
  44. package/scripts/review-staging-post.test.sh +40 -8
@@ -73,7 +73,7 @@ describe("pre-filter (#4123)", () => {
73
73
  const call = w.runner[0];
74
74
  assert.equal(call.model, DEFAULT_PREFILTER_MODEL); assert.equal(call.readOnly, true); assert.equal(call.resume, null); assert.equal(call.cwd, "/repo");
75
75
  assert.match(call.prompt, /merged PR #90/); assert.doesNotMatch(call.prompt, /PR #91/);
76
- assert.deepEqual(w.closed, [[12, "not planned"]]);
76
+ assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
77
77
  assert.deepEqual(w.labels, [[12, "invalid"]]);
78
78
  assert.match(w.comments[0][1], /FORGE:PREFILTER/); assert.match(w.comments[0][1], /src\/foo\.js:3 null guard/); assert.match(w.comments[0][1], /PR #90/);
79
79
  });
@@ -120,7 +120,7 @@ describe("pre-filter (#4123)", () => {
120
120
  assert.equal(r.closed, true); assert.equal(r.kind, "ci-green");
121
121
  assert.equal(w.runner.length, 0);
122
122
  assert.match(w.comments[0][1], /actions\/runs\/600/); assert.match(w.comments[0][1], /actions\/runs\/500/);
123
- assert.deepEqual(w.closed, [[34, "not planned"]]);
123
+ assert.deepEqual(w.closed, [[34, "completed"]]); // #5152: CI green again is delivered
124
124
  });
125
125
 
126
126
  it("#4305 the CI-green shortcut is skipped for intermittent, reopened and same-SHA-flaky CI failures", async () => {
@@ -204,7 +204,7 @@ describe("pre-filter (#4123)", () => {
204
204
  assert.equal(r2.closed, true); assert.equal(tiny.comments.length, 1); assert.match(tiny.comments[0][1], /^<!-- FORGE:PREFILTER -->/);
205
205
  assert.match(flakyReason({ comments: [{ body: tiny.comments[0][1], author: { login: "forgedock-app[bot]" }, authorAssociation: "NONE" }] }, ["forgedock-app"])?.reason ?? "", /reopened/);
206
206
  const dead = flaky(99); const r3 = await pf(dead, 34);
207
- assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "not planned"]]); assert.match(r3.detail, /no marker comment/);
207
+ assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "completed"]]); assert.match(r3.detail, /no marker comment/);
208
208
  });
209
209
 
210
210
  it("a CI-failure issue whose workflow is still red (or whose latest run is the failing one) goes to a lane", async () => {
@@ -224,6 +224,7 @@ describe("pre-filter (#4123)", () => {
224
224
  assert.deepEqual(namedPaths("fix lib/a.js <!-- x/hidden.js --!> visible/e.js and <!-- unclosed c/d.js"), ["lib/a.js", "visible/e.js"]);
225
225
  const gone = world({ issues: { 5: { body: "crash in src/old/foo.js:12" } }, verdict: { verdict: "INVALID", evidence: ["src/old/foo.js removed in abc123"] } });
226
226
  assert.equal((await pf(gone, 5)).closed, true);
227
+ assert.deepEqual(gone.closed, [[5, "not planned"]], "#5152: only an INVALID verdict closes not planned");
227
228
  assert.match(gone.runner[0].prompt, /do not exist on origin\/staging: src\/old\/foo\.js/);
228
229
  const kept = world({ issues: { 5: { body: "crash in src/old/foo.js:12 and src/new.js/x.ts" } }, paths: ["src/old/foo.js"] });
229
230
  const r = await pf(kept, 5);
@@ -300,7 +301,7 @@ describe("orchestrate with the pre-filter (#4123)", () => {
300
301
  assert.deepEqual(res.lanes, [13]);
301
302
  assert.equal(logs.at(-1), "lanes: #13");
302
303
  assert.ok(logs.some((s) => /^pre-filter: 2 checked, 1 candidate\(s\), 1 closed/.test(s)));
303
- assert.deepEqual(w.closed, [[12, "not planned"]]);
304
+ assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
304
305
  await assert.rejects(triageFromCli([], {}), /usage/);
305
306
  });
306
307
 
@@ -488,8 +488,10 @@ export function reviewerPrompt({ issue, comments = [], contract, result, files =
488
488
  * #4160: the targeted re-ask after an incomplete or inconsistent reply. Only the missing or invalid ids are asked
489
489
  * for, with the exact problems and the reviewer's previous reply; no diff is re-sent (the worktree still holds the change).
490
490
  */
491
- export function reviewerRepairPrompt({ missing = [], problems = [], rows = null }) {
492
- return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is a backtick-quoted repo-relative path:line copied verbatim (e.g. \`apps/web/app/(console)/layout.tsx:12\`); for an evidence path reported as not existing, give the full repo-relative path of that file from the changed files or the repository.`;
491
+ export function reviewerRepairPrompt({ missing = [], problems = [], rows = null, criteria = [] }) {
492
+ // #5164: a coverage misquote is re-asked with the issue's criterion lines verbatim, so the quote is copied, not retyped
493
+ const crit = (criteria ?? []).length ? `\n\n<issue_criteria>\nCopy every issue_coverage quote exactly (character for character, including #numbers and plurals) from one of these issue lines:\n${criteria.slice(0, 80).map((c) => `- ${String(c).slice(0, 400)}`).join("\n")}\n</issue_criteria>` : "";
494
+ return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>${crit}\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is a backtick-quoted repo-relative path:line copied verbatim (e.g. \`apps/web/app/(console)/layout.tsx:12\`); for an evidence path reported as not existing, give the full repo-relative path of that file from the changed files or the repository.`;
493
495
  }
494
496
 
495
497
  /** Pre-filter (#4123): one read-only, single-turn check of an issue a cheap signal flagged as possibly already fixed. */
@@ -65,9 +65,9 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
65
65
  assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
66
66
  assert.equal(r[1].outcome, "needs-human");
67
67
  assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
68
- assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
68
+ assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
69
69
  const lines = formatOutcomeLists(rows);
70
- assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
70
+ assert.equal(lines.length, 2); assert.match(lines[0], /^held subtree: #1 \(needs-human\) holds #2; release: forgedock orchestrate --requeue <batch> 1$/); assert.match(lines[1], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
71
71
  });
72
72
 
73
73
  it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
@@ -92,7 +92,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
92
92
  assert.equal(calls.filter((c) => c.n === 1).length, 1);
93
93
  assert.equal(r[1].outcome, "budget-exceeded");
94
94
  assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
95
- assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
95
+ assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
96
96
  assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
97
97
  });
98
98
 
@@ -124,7 +124,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
124
124
  assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
125
125
  assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
126
126
  assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
127
- assert.equal(r[2].outcome, "skipped"); assert.equal(r[5].outcome, "merged");
127
+ assert.equal(r[2].outcome, "held"); assert.equal(r[5].outcome, "merged");
128
128
  const lines = formatOutcomeLists(rows);
129
129
  assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
130
130
  const esc = lines.find((l) => l.startsWith("escalations"));
@@ -187,7 +187,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
187
187
  const calls = [];
188
188
  const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
189
189
  runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
190
- assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "skipped"]);
190
+ assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
191
191
  });
192
192
 
193
193
  it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
@@ -9,7 +9,7 @@ import { dirname, join } from "node:path";
9
9
  import { tmpDir } from "../tests/_tmp.mjs";
10
10
  import { containerProvider, parseExecution, providerByName, providerLanes, remoteTransportFor } from "./execution.mjs";
11
11
  import { remoteProvider, httpTransport, isActionsEndpoint, redact } from "./sandbox-remote.mjs";
12
- import { resolveCredential } from "./credentials.mjs";
12
+ import { memoKeychain, resolveCredential } from "./credentials.mjs";
13
13
  import { makeLedger } from "./ledger.mjs";
14
14
  import { openBatch } from "./batch.mjs";
15
15
  import { orchestrate } from "./orchestrate.mjs";
@@ -208,6 +208,27 @@ describe("remote adapter (#4254)", () => {
208
208
  await assert.rejects(r.status("x"), /router auth/);
209
209
  });
210
210
 
211
+ it("5144 httpTransport: a function token is resolved per call", async () => {
212
+ let cur = null;
213
+ const seen = [];
214
+ const t = httpTransport({ endpoint: "https://pool.example", token: () => cur, fetchFn: async (url, init) => { seen.push(init.headers.authorization ?? null); return { status: 200, ok: true, text: async () => JSON.stringify({ state: "running" }), headers: { get: () => null } }; } });
215
+ await t.status("x"); cur = "tok-new"; await t.status("x"); cur = "tok-rotated"; await t.status("x"); cur = null; await t.status("x");
216
+ assert.deepEqual(seen, [null, "Bearer tok-new", "Bearer tok-rotated", null]);
217
+ });
218
+
219
+ it("5144 remoteProvider: a token rotated after construction is redacted from pulled logs", async () => {
220
+ const batch = openBatch(join(tmpDir("remote-5144-"), "batches"), "bt");
221
+ const transport = fakeTransport();
222
+ let tok = null;
223
+ const p = remoteProvider({ batch, repo: "o/r", settings: { tokenEnv: "FD_TOKEN" }, transport, env: {}, credentials: (n) => (n === "FD_TOKEN" ? tok : null), gitIdentity: () => ({}) });
224
+ tok = "rotated-secret-789";
225
+ const h = await p.start({ issue: 3, args: [], env: {}, secrets: [] });
226
+ transport.boxes.get(h.id).log = `leak ${tok}\n`;
227
+ await p.status(h);
228
+ const log = readFileSync(batch.logFile(3), "utf8");
229
+ assert.equal(log.includes(tok), false);
230
+ });
231
+
211
232
  it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
212
233
  const calls = [];
213
234
  const reply = (status, body, headers = {}) => ({ status, ok: status >= 200 && status < 300, text: async () => (body === undefined ? "" : JSON.stringify(body)), headers: { get: (k) => headers[k] ?? null } });
@@ -265,6 +286,20 @@ describe("remote config and the Actions constraint (#4254)", () => {
265
286
  assert.equal(resolveCredential("FD_T", { env: {}, keychain: () => { throw new Error("locked"); } }), null);
266
287
  assert.equal(resolveCredential("bad name", { env: { "bad name": "x" } }), null);
267
288
  });
289
+
290
+ it("5159 memoKeychain: one lookup per name per TTL, misses and failures cached as null", () => {
291
+ let t = 0;
292
+ const calls = [];
293
+ const kc = memoKeychain({ keychain: (n) => { calls.push(n); if (n === "BAD") throw new Error("locked"); return n === "HIT" ? "v" : null; }, ttlMs: 1000, now: () => t });
294
+ assert.equal(kc("HIT"), "v"); assert.equal(kc("HIT"), "v");
295
+ assert.equal(kc("MISS"), null); assert.equal(kc("MISS"), null);
296
+ assert.equal(kc("BAD"), null); assert.equal(kc("BAD"), null);
297
+ assert.deepEqual(calls, ["HIT", "MISS", "BAD"]);
298
+ t = 1000;
299
+ assert.equal(kc("MISS"), null);
300
+ assert.deepEqual(calls, ["HIT", "MISS", "BAD", "MISS"], "re-read after the TTL");
301
+ assert.equal(resolveCredential("MISS", { env: { MISS: "e" }, keychain: kc }), "e", "the env still wins");
302
+ });
268
303
  });
269
304
 
270
305
  describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
@@ -43,7 +43,8 @@ export function isActionsEndpoint(endpoint) {
43
43
  /**
44
44
  * Generic HTTP transport: POST {endpoint}/sandboxes (body = launch spec), GET /sandboxes/:id -> {state},
45
45
  * GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
46
- * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`.
46
+ * GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`
47
+ * (`token` is a string, or a function called per request, #5144).
47
48
  * 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
48
49
  * Optional `POST /sandboxes/:id/stop` (#4612) asks the pool to SIGTERM the lane so its teardown checkpoints and it reports `deferred run.interrupted`; 2xx = sent, 404/405 = unsupported or gone (never a DELETE fallback).
49
50
  * Optional relay (#4778): `GET /sandboxes/:id/relay -> {requests: [{rid, argv}]}` lists a sandbox's pending read-only capability
@@ -53,7 +54,9 @@ export function isActionsEndpoint(endpoint) {
53
54
  export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
54
55
  const root = String(endpoint ?? "").replace(/\/+$/, "");
55
56
  const call = async (method, path, body) => {
56
- const res = await fetchFn(`${root}${path}`, { method, headers: { accept: "application/json", ...(body ? { "content-type": "application/json" } : {}), ...(token ? { authorization: `Bearer ${token}` } : {}) },
57
+ // #5144: `token` may be a function, resolved on every request so a token fixed or rotated mid-batch is the next bearer.
58
+ const bearer = typeof token === "function" ? token() : token;
59
+ const res = await fetchFn(`${root}${path}`, { method, headers: { accept: "application/json", ...(body ? { "content-type": "application/json" } : {}), ...(bearer ? { authorization: `Bearer ${bearer}` } : {}) },
57
60
  ...(body ? { body: JSON.stringify(body) } : {}) });
58
61
  let data = null;
59
62
  try { const t = await res.text(); data = t ? JSON.parse(t) : null; } catch { data = null; }
@@ -195,6 +198,7 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
195
198
  const id = idFor(issue);
196
199
  // Each allowlisted secret from the env, else the keychain (#4254); nothing else from the host.
197
200
  const resolved = {};
201
+ if (settings.tokenEnv) remember(credentials(settings.tokenEnv)); // #5144: a token rotated since construction is redacted too
198
202
  for (const k of secrets) { const v = laneEnv?.[k] !== undefined && laneEnv[k] !== "" ? String(laneEnv[k]) : credentials(k); if (v) { resolved[k] = v; remember(v); } }
199
203
  const li = args.indexOf("--lane");
200
204
  const resultFile = `/result/${issue}.json`;
@@ -11,6 +11,7 @@
11
11
  * {op:"attach", id, resume?} → (`resume` also clears a pending cancel, #4592) log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
12
12
  * ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
13
13
  * {op:"set-parallel", id, k} → {type:"ok", parallel}
14
+ * {op:"set-execution", id, maxRemote?, mode?} → {type:"ok", execution} (#5138): live max_remote / placement (remote|hybrid|local); remote/hybrid refused for a batch with no provider
14
15
  * {op:"requeue", id, issues, reason, by} → {type:"ok", requeued, refused, hosted, reopened} (#4550): see batch.requeue; a finished batch is hosted again
15
16
  * {op:"add", id, issues} → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
16
17
  * hosted runner reads on its next pass and an unhosted batch on its next resume
@@ -42,6 +43,7 @@ import { fileURLToPath } from "node:url";
42
43
  import { schedDir, pidAlive } from "./sched.mjs";
43
44
  import { HEARTBEAT_MS, writeHeartbeat, clearHeartbeat } from "./schedd-health.mjs";
44
45
  import { batchRoot, listBatches, openBatch, SECRET_ENV } from "./batch.mjs";
46
+ import { applySetExecution } from "./live-execution.mjs";
45
47
 
46
48
  export const IDLE_MS = 10 * 60 * 1000;
47
49
  export const STOP_MS = 10 * 1000;
@@ -250,6 +252,13 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
250
252
  b.setParallel(Number(req.k));
251
253
  return send({ type: "ok", parallel: Number(req.k) }, true);
252
254
  }
255
+ if (req.op === "set-execution") {
256
+ // #5138: live max_remote / placement mode; the hosted runner re-reads the batch's `execution` file on every admission.
257
+ const b = openBatch(root, req.id);
258
+ if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
259
+ const execution = applySetExecution(b, { ...(req.maxRemote !== undefined ? { maxRemote: req.maxRemote } : {}), ...(req.mode !== undefined ? { mode: req.mode } : {}) });
260
+ return send({ type: "ok", execution }, true);
261
+ }
253
262
  if (req.op === "add") {
254
263
  const b = openBatch(root, req.id);
255
264
  if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
@@ -245,6 +245,8 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
245
245
  const c = credCache[kind];
246
246
  if (!c || now() - c.at > CRED_TTL[kind]) {
247
247
  const x = await call(cfg.router_url, "GET", `/agent/creds/${kind}`, { token: cfg.token, timeoutMs: 20000 });
248
+ // #5165: every router login at its usage ceiling -> pass the typed 429 (reason, reset) through, uncached, so the lane defers.
249
+ if (x.status === 429 && x.data && typeof x.data === "object") return { status: 429, body: x.data, retryAfter: x.retryAfter };
248
250
  if (!x.ok || !x.data?.token) return { status: 502, body: { error: `router credentials (${kind}): HTTP ${x.status}` } };
249
251
  credCache[kind] = { token: x.data.token, at: now() };
250
252
  }
@@ -397,7 +399,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
397
399
  const url = new URL(req.url, "http://agent");
398
400
  try {
399
401
  const cred = req.method === "GET" && /^\/sandboxes\/([^/]+)\/creds\/([a-z]+)$/.exec(url.pathname);
400
- if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body); }
402
+ if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body, r.retryAfter ? { "retry-after": String(r.retryAfter) } : {}); }
401
403
  const relay = (req.method === "POST" && /^\/sandboxes\/([^/]+)\/relay$/.exec(url.pathname)) || (req.method === "GET" && /^\/sandboxes\/([^/]+)\/relay\/([^/]+)$/.exec(url.pathname));
402
404
  if (relay) { const r = await sandboxRelay(decodeURIComponent(relay[1]), relay[2] ? decodeURIComponent(relay[2]) : null, req); return send(res, r.status, r.body ?? {}); }
403
405
  if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized", hop: "agent", reason: "agent auth: router token rejected by agent" });
@@ -93,19 +93,28 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
93
93
  }
94
94
  return gh.token;
95
95
  }
96
- // Claude access token for sandboxes: the logged-in account on this host with the most validity left. Only the
97
- // short-lived access token is shared; refresh tokens stay here, so this host keeps refreshing its own sessions.
96
+ // Claude access token for sandboxes (#5165): among the logins with 30+ minutes left, the one with the lowest tokenbaton
97
+ // usage (validity breaks ties). Only the short-lived access token is shared; refresh tokens stay here, so this host
98
+ // keeps refreshing its own sessions.
98
99
  function claudeToken() {
99
100
  const dirs = cfg.claude_config_dirs ?? readdirSync(homedir()).filter((d) => /^\.claude(-\d+)?$/.test(d)).map((d) => join(homedir(), d));
100
- let best = null;
101
+ const logins = [];
101
102
  for (const d of dirs) {
102
103
  try {
103
104
  const o = JSON.parse(readFileSync(join(d, ".credentials.json"), "utf8")).claudeAiOauth;
104
- if (o?.accessToken && o.expiresAt > now() + 30 * 60000 && (!best || o.expiresAt > best.expiresAt)) best = o;
105
+ const n = /\.claude-(\d+)$/.exec(d)?.[1];
106
+ if (o?.accessToken) logins.push({ n: n ? Number(n) : null, accessToken: o.accessToken, expiresAt: o.expiresAt });
105
107
  } catch { /* no login in this dir */ }
106
108
  }
107
- if (!best) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
108
- return { token: best.accessToken, expires_at: new Date(best.expiresAt).toISOString() };
109
+ const src = usageSource(cfg);
110
+ const usage = readUsage(src, logins.map((l) => l.n).filter((n) => n !== null));
111
+ const pick = pickClaudeLogin({ logins, usage, ceilingPct: cfg.usage_ceiling_pct ?? 95, now: now() });
112
+ if (pick.status === 503) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
113
+ if (pick.status === 429) { log.warn?.(`claude login: every account at or above the usage ceiling; ${pick.body.reason}`); return pick; }
114
+ if (pick.fallback) log.warn?.(`claude login: no usage source (${src}), falling back to longest validity`);
115
+ const { login, pct } = pick;
116
+ log.info?.(`claude login selected: account=${login.n ?? "default"} usage=${pct ?? "unknown"}% expires_at=${new Date(login.expiresAt).toISOString()}`);
117
+ return { status: 200, body: { token: login.accessToken, expires_at: new Date(login.expiresAt).toISOString() } };
109
118
  }
110
119
  const agentCall = (a, method, path, body, timeoutMs) => call(a.url, method, path, { body, token: a.token, timeoutMs, fetchFn });
111
120
 
@@ -285,7 +294,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
285
294
  [route("GET", "/agents"), fd(async () => ({ status: 200, body: { agents: await capacities(), sandboxes: sbx() } }))],
286
295
  [route("POST", "/codes"), fd(async () => ({ status: 200, body: issueCode() }))],
287
296
  [route("GET", "/agent/creds/github"), { fn: async () => ({ status: 200, body: { token: await githubToken() } }), auth: "agent" }],
288
- [route("GET", "/agent/creds/claude"), { fn: async () => ({ status: 200, body: claudeToken() }), auth: "agent" }],
297
+ [route("GET", "/agent/creds/claude"), { fn: async () => claudeToken(), auth: "agent" }],
289
298
  [route("POST", "/agent/relay/:id"), { fn: async (p, body, _u, req) => relayPost(req, p, body), auth: "agent" }], // #4778
290
299
  [route("GET", "/agent/relay/:id/:rid"), { fn: async (p, _b, _u, req) => relayPoll(req, p), auth: "agent" }],
291
300
  [route("GET", "/agent/bundle"), { fn: async () => ({ bundle: true }), auth: "agent" }],
@@ -355,6 +364,62 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
355
364
  return { handle, launch, claim, tick, enroll, issueCode, capacities, status, store };
356
365
  }
357
366
 
367
+ /** tokenbaton's usage dir (#5165): router.json `usage_source`, else `$TOKENBATON_HOME`, else `~/.tokenbaton`. */
368
+ export function usageSource(cfg, env = process.env) {
369
+ return cfg.usage_source ?? env.TOKENBATON_HOME ?? join(homedir(), ".tokenbaton");
370
+ }
371
+
372
+ /** One tokenbaton `live.<n>` line (`<5h %> <5h reset s> <weekly %> <weekly reset s> [updated s]`), or null; the format of bin/lean/rotator.mjs parseLive. */
373
+ export function parseLiveUsage(text) {
374
+ const f = String(text ?? "").trim().split(/\s+/).map(Number);
375
+ if (f.length < 4 || !f.slice(0, 4).every(Number.isFinite)) return null;
376
+ return { fiveHourPct: f[0], fiveHourResetMs: f[1] * 1000, weeklyPct: f[2], weeklyResetMs: f[3] * 1000 };
377
+ }
378
+
379
+ /** Map account n -> parsed `live.<n>` for the given accounts; unreadable or malformed files are left out. */
380
+ export function readUsage(dir, ns, readFile = (p) => readFileSync(p, "utf8")) {
381
+ const out = new Map();
382
+ for (const n of ns) { try { const u = parseLiveUsage(readFile(join(dir, `live.${n}`))); if (u) out.set(n, u); } catch { /* no usage for n */ } }
383
+ return out;
384
+ }
385
+
386
+ /**
387
+ * Pick the Claude login to hand out (#5165). Candidates have 30+ minutes of validity. A login with known usage is
388
+ * excluded at or above `ceilingPct` 5-hour use or 100% weekly use (a passed reset counts as 0, like the local rotator);
389
+ * the rest rank by 5-hour use, then validity. Logins with unknown usage follow. No usage at all: longest validity,
390
+ * `fallback: true`. Every candidate excluded: a typed 429 carrying the earliest reset.
391
+ * @returns {{status: 200, login, pct, fallback} | {status: 429, retryAfter, body} | {status: 503}}
392
+ */
393
+ export function pickClaudeLogin({ logins, usage, ceilingPct = 95, now }) {
394
+ const valid = logins.filter((l) => l.accessToken && l.expiresAt > now + 30 * 60000);
395
+ if (!valid.length) return { status: 503 };
396
+ const used = (pct, resetMs) => (resetMs <= now ? 0 : pct);
397
+ const byValidity = (a, b) => b.expiresAt - a.expiresAt;
398
+ const known = [], unknown = [], blocked = [];
399
+ for (const l of valid) {
400
+ const u = l.n !== null && l.n !== undefined ? usage.get(l.n) : undefined;
401
+ if (!u) { unknown.push(l); continue; }
402
+ const five = used(u.fiveHourPct, u.fiveHourResetMs), week = used(u.weeklyPct, u.weeklyResetMs);
403
+ if (five >= ceilingPct || week >= 100) {
404
+ const r = [];
405
+ if (five >= ceilingPct) r.push(u.fiveHourResetMs);
406
+ if (week >= 100) r.push(u.weeklyResetMs);
407
+ blocked.push(Math.max(...r));
408
+ } else known.push({ l, pct: five });
409
+ }
410
+ if (known.length) {
411
+ known.sort((a, b) => a.pct - b.pct || byValidity(a.l, b.l));
412
+ return { status: 200, login: known[0].l, pct: known[0].pct, fallback: false };
413
+ }
414
+ if (unknown.length) {
415
+ unknown.sort(byValidity);
416
+ return { status: 200, login: unknown[0], pct: null, fallback: !blocked.length };
417
+ }
418
+ const resetMs = Math.min(...blocked), at = new Date(resetMs).toISOString();
419
+ return { status: 429, retryAfter: Math.max(1, Math.ceil((resetMs - now) / 1000)),
420
+ body: { error: "usage limit", code: "rate_limited", reason: `usage limit hit; resets at ${at}`, reset_at: at } };
421
+ }
422
+
358
423
  export function startRouter(cfg, deps) {
359
424
  const router = createRouter(cfg, deps);
360
425
  const server = createServer(router.handle);
@@ -385,6 +385,8 @@ async function finalize(a, io) {
385
385
  // Incomplete-panel guard + 6.75 delivery
386
386
  const d = await bash(io, "review-staging-post.sh", ["delivery", ...common], env);
387
387
  relay(io, d);
388
+ // Exit 2 = the comments could not be listed on any attempt (forge#5166): rerun, never re-dispatch reviewers that may have posted.
389
+ if (d.code === 2) throw new Stop(1, ["REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery could not be verified (GitHub comment listing failed); rerun --finalize, do not re-dispatch reviewers", BLOCK_DEPLOY]);
388
390
  if (d.code !== 0) throw new Stop(1, [`REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery incomplete (exit ${d.code}); re-dispatch each missing reviewer once with the note "structured findings block required", else the panel is degraded`, BLOCK_DEPLOY]);
389
391
 
390
392
  // 6.5 verdict handling (override, posture, test-gate markers)
package/bin/runner.mjs CHANGED
@@ -1352,6 +1352,27 @@ export function detectUsageLimit(stdout, output = "") {
1352
1352
  return hasUsageLimitResetClause(output || "");
1353
1353
  }
1354
1354
 
1355
+ // forge#5168: the claude CLI's own login/authentication failure texts. Literal
1356
+ // alternatives only (linear on untrusted input).
1357
+ const AUTH_FAILURE_RE = /Failed to authenticate|authentication_error|OAuth token (?:has )?expired|reached its time limit and cannot be renewed|Invalid API key|Please run \/login/i;
1358
+
1359
+ /**
1360
+ * Detects whether a failed CLI run was an authentication failure of the claude
1361
+ * login (forge#5168): an expired or invalid login is an account condition, not
1362
+ * a transient engine error. With an envelope on stdout, it must be
1363
+ * `is_error: true` with a `result` naming the failure; without one, the plain
1364
+ * combined output must. `stdout` is parsed ALONE (forge#2422); never throws.
1365
+ *
1366
+ * @param {string} stdout - raw stdout of the CLI run
1367
+ * @param {string} [output] - combined stdout+stderr, for the plain-text fallback
1368
+ * @returns {boolean}
1369
+ */
1370
+ export function detectAuthFailure(stdout, output = "") {
1371
+ const env = parseCliEnvelope(stdout);
1372
+ if (env) return env.is_error === true && typeof env.result === "string" && AUTH_FAILURE_RE.test(env.result);
1373
+ return typeof output === "string" && AUTH_FAILURE_RE.test(output);
1374
+ }
1375
+
1355
1376
  /**
1356
1377
  * Best-effort extraction of a Claude CLI session-limit reset time from
1357
1378
  * captured stdout/stderr (forge#2241). The CLI's own session-limit message
@@ -1844,6 +1865,9 @@ export function runCliBackend({
1844
1865
  // default when the reset time itself is missing or unparseable. The
1845
1866
  // engine, not the runner, owns that default — no epoch is fabricated here.
1846
1867
  if (isUsageLimit) err.usageLimit = true;
1868
+ // forge#5168: a failed login (never when the usage limit already matched,
1869
+ // which keeps precedence) lets the lane defer as `backend.auth`.
1870
+ else if (detectAuthFailure(stdout, output)) err.authFailure = true;
1847
1871
  if (resetAt) {
1848
1872
  err.resetAt = resetAt;
1849
1873
  // forge#2524: also attach a machine-usable epoch-ms timestamp so
@@ -100,7 +100,8 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
100
100
  (live lanes are re-adopted, nothing is launched twice). The runner is hosted by the detached `forgedock schedd` daemon,
101
101
  which resumes every daemon-hosted batch that has no report yet when it starts, so a closed session needs no manual
102
102
  `--resume`; a batch started without the daemon still needs `forgedock orchestrate --resume <id>`. Change K of a running batch with
103
- `forgedock orchestrate --set-parallel <id> K`. Issues filed while a batch runs join it with
103
+ `forgedock orchestrate --set-parallel <id> K`. Change its remote lane cap or placement with `forgedock orchestrate --set-max-remote <id> N` and
104
+ `--set-placement <id> remote|hybrid|local` (remote/hybrid only for a batch started with a remote provider; running lanes keep running). Issues filed while a batch runs join it with
104
105
  `forgedock orchestrate --add <id> N...` (or `--resume <id> N...`), under its K and DAG; never start a second batch
105
106
  for them. After a decision (a `FORGE:DECISION` posted and `needs-human` removed), a blocked issue that already ended in the batch
106
107
  (`needs-human`, `invalid`, `skipped`, `dependency-*`, `ci-pending`, `deferred`) returns to it with
@@ -401,14 +401,14 @@ node "$FD_ROOT/bin/forgedock.mjs" review-staging "$PR_NUMBER" --finalize --test-
401
401
  ```
402
402
 
403
403
  The call runs these steps in order:
404
- 1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying.
404
+ 1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying. The PR comments are listed once per attempt. A missing reviewer or a GitHub API error is re-listed with backoff (5 s, 15 s, 45 s) before any verdict, and each domain is reported with the attempt it was found on.
405
405
  2. The Phase 6.5 verdict handling.
406
406
  3. `scripts/review-finalize.sh`:
407
407
  - `extract`: trusted comments only, dedup, and the `commands/review-pr.md` §6B.4 provenance and concreteness gates. **CRITICAL/HIGH are never routed as pre-existing**, whatever their confidence.
408
408
  - `classify --dry-run`, then `classify` for real. This passes `--scope-files` (the 0B.1 scoped set when `SCOPE_MODE=scoped`) and `--prior-prs` (the bundle members), and posts `FORGE:NOTE_DISPOSITION`.
409
409
  - `file-prepare`.
410
410
 
411
- Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A.
411
+ Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A. `REVIEW DELIVERY UNVERIFIED` means GitHub could not list the comments on the last attempt. It stops the run, but nothing is labelled or posted. Do not re-dispatch anyone: rerun 7A once GitHub answers.
412
412
 
413
413
  Exit 0 prints the following:
414
414
  - `TEST_GATE:`, `SCOPE_MODE=`.
package/docs/CONFIG.md CHANGED
@@ -438,6 +438,7 @@ orchestration:
438
438
  | `max_concurrent` | integer | No | Max concurrent top-level `/work-on` dispatches. It does not bound cascade growth; under `policy: all` it raises drain and spawn throughput together. Default: 12. |
439
439
  | `breaker.errors` / `breaker.window_min` | integer | No | `forgedock orchestrate` circuit breaker (#4121): after `errors` engine-errors within `window_min` minutes the batch stops admitting lanes, lets running lanes finish, and keeps the rest pending for `--resume`. `errors: 0` turns it off. Default: 3 in 30 |
440
440
  | `max_deferrals` | integer | No | `forgedock orchestrate`: requeues of one issue after a non-host `deferred` (usage limit, merge/push transient, CLI idle, lease held, gh unavailable) before its row is final. `host.busy` and `sandbox.capacity` (sandbox pool full) deferrals are not capped. Default: 3 |
441
+ | `transient_retries` | integer | No | `forgedock orchestrate` (#5161): backoff retries of an issue whose lane ended `engine-error` with a transport or gateway failure in its detail (curl `returned error: 5xx`, `HTTP/1.1 5xx`, `Bad Gateway`, `Service Unavailable`, `overloaded`, connection reset/refused/timeout). Delays 30s, 2m, 8m, then 15m each; dependents stay pending meanwhile. Not counted against the single engine-error relaunch; once spent, the row ends `engine-error`. Every such error still counts toward the breaker. `0` turns transient retries off. Default: 4 |
441
442
  | `blocked_label` | string | No | `forgedock orchestrate` (#4773): an open issue carrying this label is admitted to the batch but not dispatched; removing the label releases it on the next scheduler pass. A `Blocked by #N` / `Depends on #N` line in the body of a trusted author (OWNER, MEMBER, COLLABORATOR or a bot) is a DAG edge to #N, and `--add` prints the edges it created. Empty turns the label off. Default: `blocked` |
442
443
  | `auto_add_children` | string | No | `forgedock orchestrate` (#4965): which children a running batch adds by itself. `off` (no auto-adds; a decomposed lane's sub-issues stay out too), `decomposed` (a decomposed lane's sub-issues join, as before), `epic` (also the open children of every batch issue and of each `--epic N`, found by a `**Parent**: #P` line, the `parent` field or GitHub sub-issues, including children filed after the batch started) or `all`. Only children whose author is trusted (an owner, member or collaborator, or a bot, the `Blocked by` rule) are auto-added; any other is skipped with a logged row. Children labelled `needs-human` and children another live batch of the repo still holds are not added. Each pass that adds posts one batch decision and one `added` row listing the children and edges. `forgedock orchestrate --auto-children <mode>` overrides it per batch and `--resume` keeps it. Default: `decomposed` |
443
444
  | `auto_add_children_poll_s` | integer | No | Seconds between child scans for `epic`/`all` (one `gh issue list` per scan). Default: `120` |
@@ -470,6 +471,43 @@ orchestration:
470
471
 
471
472
  ---
472
473
 
474
+ ## `orchestrate.health` and `notify` (OPTIONAL)
475
+
476
+ Early batch health (#5167). `forgedock orchestrate` checks each batch at T+15 and T+60 minutes after it was created (`plan.json` `created`), and once more when the batch ends. When the batch starts, it prints the schedule: `health checks: T+15 14:05Z, T+60 14:50Z, and at batch end; forgedock status --batch <id>`. The first unhealthy check raises one `BATCH UNHEALTHY <id>` alert, which lists the top causes and the exact recovery commands (`--requeue`, a relaunch, `--resume`, `--cancel`). `health.json` in the batch dir records the checkpoints already done and the alert, so a daemon restart arms only the remaining checks and a batch never alerts twice. Checks and alerts never fail, delay or change the batch.
477
+
478
+ Causes: `root-failed` (a root issue, meaning one with no dependency inside the batch, ended non-merged); `terminal-share` (the share of rows ended `skipped`/`held`/`engine-error`/`needs-human` or another non-merged outcome except `invalid`/`decomposed` is at or above the threshold, with nothing merged); `stalled` (0 lanes running while issues are pending, operator-held issues excepted); `refused` (the launch was refused).
479
+
480
+ ```yaml
481
+ orchestrate:
482
+ health:
483
+ root_failed: true # any root ended non-merged is unhealthy (default true)
484
+ terminal_share: 0.5 # 0..1, terminal-non-merged share with 0 merged (default 0.5)
485
+ stalled: true # 0 running while pending > 0 (default true)
486
+ issue: 123 # comment target; default: the batch's --epic issue(s); a milestone gets no comment
487
+ checks_min: [15, 60] # minutes after launch (default [15, 60])
488
+ notify:
489
+ command: "curl -fsS -H 'content-type: application/json' -d @- https://hooks.example/forge"
490
+ ```
491
+
492
+ | Key | Default | Meaning |
493
+ |---|---|---|
494
+ | `orchestrate.health.root_failed` | `true` | Raise `root-failed`. |
495
+ | `orchestrate.health.terminal_share` | `0.5` | Threshold for `terminal-share`. |
496
+ | `orchestrate.health.stalled` | `true` | Raise `stalled`. |
497
+ | `orchestrate.health.issue` | none | Issue to comment on. When unset, the alert comments on each `--epic` issue. |
498
+ | `orchestrate.health.checks_min` | `[15, 60]` | Timed checkpoints, in minutes after launch. |
499
+ | `notify.command` | none | Shell command run once per alert. It gets the alert JSON (`{alert, batch, repo, text, causes, recovery}`) on stdin and `FORGEDOCK_ALERT=BATCH_UNHEALTHY`, `FORGEDOCK_ALERT_BATCH` and `FORGEDOCK_ALERT_TEXT` in its env. It is killed after 30 s. A failure only prints a `health: notify.command failed` line. |
500
+
501
+ The keys are flattened one level by the parser, the same as `orchestrate.drift.*`. Health keys alone do not turn drift on.
502
+
503
+ Every channel is best effort: the runner line, the GitHub comment (marker `FORGE:BATCH-HEALTH`), `notify.command`, and the `forgedock status` line. `forgedock status` lists the unhealthy batches that are unresolved (no report yet, or alerted within the last day) and the launch refusals from the last day. `forgedock status --batch <id>` shows that batch's health (healthy, or unhealthy with causes and recovery) and its pending checkpoints.
504
+
505
+ A refused milestone start (#4659, `parallel milestone start refused: …`) still exits 1 with the error on stderr and creates no batch dir. It is also recorded in `<batch root>/refusals.json` (the newest 20 are kept) and sends the same alert, with the merge-the-dependency or `--force "<reason>"` relaunch.
506
+
507
+ **Commands that use this section**: `forgedock orchestrate`, `forgedock status`
508
+
509
+ ---
510
+
473
511
  ## `decisions` (OPTIONAL)
474
512
 
475
513
  Decision engine (#4766, see [DECISIONS.md](DECISIONS.md)). The issue-watch stage (#4793) calls `settle` for the open `needs-human` issues a watch selects: it runs when `mode: live` (a repo setting: every watch of the repo runs the stage), or when the watch sets `sweep_needs_human: true`; `policy` and `human_only` are also read by prodwatch (`loadPolicy`). The parser has one nesting level and no arrays, so lists are comma-separated.
@@ -1185,7 +1223,7 @@ decomposition:
1185
1223
 
1186
1224
  **With `close_parent: false`** (default): unchanged. The parent stays open and is closed by C3 when the last child closes.
1187
1225
 
1188
- The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `not planned`, keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
1226
+ The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `completed` (#5152; scope moved to the sub-issues, not dropped), keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
1189
1227
 
1190
1228
  **Limitation**: `--retry` and engine resume refuse closed issues, so a closed decomposed parent cannot be re-run. It has nothing left to run; work on its open children instead.
1191
1229
 
@@ -44,6 +44,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
44
44
  | `gh.unavailable` | `run-issue --repo` could not verify the cwd repo because `gh repo view` kept failing after 3 attempts with backoff (typically a timeout under host load); the core or full run is `deferred` (exit 75, retry in 60s) before any lease, nothing posted | engine CLI | `bin/engine-cli.mjs` |
45
45
  | `gh.auth` | A `gh` call in a core run answered HTTP 401 (expired or revoked bot token) and one run of `scripts/refresh-bot-token.sh` plus a retry did not fix it; the run is `deferred` (retry in 120s), never `engine-error`. See `docs/CONFIG.md` "Bot token refresh" | lean | `bin/lean/retry.mjs` |
46
46
  | `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
47
+ | `backend.auth` | The claude CLI login failed ("Failed to authenticate", expired or invalid login); the lane is `deferred` (retry in 900s) under a per-account hold like a usage limit, never `engine-error`, never relaunched or counted by the breaker (#5168) | lean | `bin/runner.mjs` `err.authFailure` |
47
48
  | `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
48
49
  | `engine.dco_rewrite` | Core work-on (forge#4997): the DCO sign-off rewrite (`git filter-branch`) of the lane's unpushed commits failed, or the lane's uncommitted changes could not be restored after it (kept on `refs/forgedock/dco-stash`). A mechanical engine step, not a human block: the lane ends `engine-error` (never `needs-human`), which `forgedock orchestrate` relaunches like any engine-error | lean | `bin/lean/signoff.mjs` `ensureSignedOff` |
49
50
  | `engine.validate_result_lost` | Core work-on (forge#4708): a validate result recorded before a merge-time CI fix round is missing from the fix session's result file and could not be carried forward; an engine bookkeeping error, not an unimplemented requirement | lean | `bin/lean/lean.mjs` `mergeFixRound` |
@@ -70,7 +71,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
70
71
  | `lane.branch_missing` | Core work-on: the lane branch (e.g. `milestone/<slug>`) does not exist on origin; fail closed | lean | `git ls-remote --heads origin` |
71
72
  | `lane.unresolved` | Core work-on: the lane could not be worked out (milestone or default branch unreadable, empty milestone slug, bad `feature_pattern`); fail closed | lean | Failed `gh` read or config |
72
73
  | `invalid` | Issue is invalid or in a divergent closed state | investigate, engine | `INVESTIGATION:INVALID` marker, or live snapshot |
73
- | `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `not planned` (label kept) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
74
+ | `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `completed` (label kept; #5152: its scope moved, it was not dropped) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
74
75
  | `builder.blocked` | Core work-on: the builder reported an external action agents cannot take, or a product decision too critical for agents: `block.kind: "human"` with `why` one of `credentials`, `spend`, `promotion`, `customer`, `product-decision`. An upstream issue, a needed split, or prod data that monitoring access can read never count | lean | `result.json` status `blocked` + typed `block` |
75
76
  | `builder.block_untyped` | Core work-on: the builder stayed `blocked` without a valid typed reason (missing `block`, a non-human `why` such as prod data, or a dependency that is not open) after one corrective resume | lean | `result.json` status `blocked`, twice |
76
77
  | `builder.no_evidence` | Core work-on: the builder reported invalid/already-fixed without file:line evidence, so the issue was not closed | lean | `result.json` with empty `evidence` |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "forgedock",
3
- "version": "1.49.0",
3
+ "version": "1.51.0",
4
4
  "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
5
5
  "bin": {
6
6
  "forgedock": "bin/forgedock.mjs"
package/pool/README.md CHANGED
@@ -10,4 +10,6 @@ ForgeDock's built-in remote compute pool (imported from computeshare; AGPL-3.0-o
10
10
 
11
11
  The module boundary is the ForgeDock HTTP sandbox protocol: the router speaks it to the dispatcher and relays to agents over HTTP. Sandboxes receive only short-lived GitHub and Claude access tokens; refresh tokens stay on the router host.
12
12
 
13
+ Claude login choice (#5165): among the router host's logins with 30+ minutes left, the router hands out the one with the lowest 5-hour use in tokenbaton's `live.<n>` (account `n` = `~/.claude-<n>`), read from router.json `usage_source` (default `$TOKENBATON_HOME`, else `~/.tokenbaton`); validity only breaks ties. An account at or above `usage_ceiling_pct` (default 95) or at 100% weekly use is skipped; when every login is, `/agent/creds/claude` answers a typed 429 (`code: rate_limited`, `reset_at` = the earliest reset, `retry-after`), which the agent passes through and `cs-cred` prints as `usage limit hit; resets at <ISO>` so the lane defers. Without readable usage it falls back to the longest validity and logs a warning. The selected account index and usage are logged, never the token.
14
+
13
15
  The `forgedock pool` CLI wiring is tracked separately (#4544).
@@ -7,6 +7,13 @@ kind="${1:?kind}"; ttl=240; [ "$kind" = claude ] && ttl=600
7
7
  cache="/tmp/.cs-cred-$kind"
8
8
  if [ -s "$cache" ] && [ $(( $(date +%s) - $(stat -c %Y "$cache") )) -lt $ttl ]; then cat "$cache"; exit 0; fi
9
9
  [ -n "${CS_CRED_BASE:-}" ] || { echo "no credential relay (CS_CRED_BASE)" >&2; exit 1; }
10
- tok=$(curl -fsS --max-time 30 -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind" | jq -r .token)
10
+ body=$(mktemp); trap 'rm -f "$body"' EXIT
11
+ code=$(curl -sS --max-time 30 -o "$body" -w '%{http_code}' -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind") || code=000
12
+ if [ "$code" = 429 ]; then
13
+ # #5165: every router login is at its usage ceiling; this line is what the lane's log tail defers on.
14
+ why=$(jq -r '.reason // empty' "$body" 2>/dev/null || true)
15
+ echo "${why:-usage limit hit; resets at unknown}" >&2; exit 1
16
+ fi
17
+ tok=""; [ "$code" = 200 ] && tok=$(jq -r .token "$body" 2>/dev/null || true)
11
18
  [ -n "$tok" ] && [ "$tok" != null ] || { echo "credential fetch failed ($kind)" >&2; exit 1; }
12
19
  umask 077; printf '%s' "$tok" > "$cache"; printf '%s' "$tok"