forgedock 1.50.0 → 1.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -65,9 +65,9 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
65
65
  assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
66
66
  assert.equal(r[1].outcome, "needs-human");
67
67
  assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
68
- assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
68
+ assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
69
69
  const lines = formatOutcomeLists(rows);
70
- assert.equal(lines.length, 1); assert.match(lines[0], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
70
+ assert.equal(lines.length, 2); assert.match(lines[0], /^held subtree: #1 \(needs-human\) holds #2; release: forgedock orchestrate --requeue <batch> 1$/); assert.match(lines[1], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
71
71
  });
72
72
 
73
73
  it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
@@ -92,7 +92,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
92
92
  assert.equal(calls.filter((c) => c.n === 1).length, 1);
93
93
  assert.equal(r[1].outcome, "budget-exceeded");
94
94
  assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
95
- assert.equal(r[2].outcome, "skipped"); assert.ok(!calls.some((c) => c.n === 2));
95
+ assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
96
96
  assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
97
97
  });
98
98
 
@@ -124,7 +124,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
124
124
  assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
125
125
  assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
126
126
  assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
127
- assert.equal(r[2].outcome, "skipped"); assert.equal(r[5].outcome, "merged");
127
+ assert.equal(r[2].outcome, "held"); assert.equal(r[5].outcome, "merged");
128
128
  const lines = formatOutcomeLists(rows);
129
129
  assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
130
130
  const esc = lines.find((l) => l.startsWith("escalations"));
@@ -187,7 +187,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
187
187
  const calls = [];
188
188
  const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
189
189
  runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
190
- assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "skipped"]);
190
+ assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
191
191
  });
192
192
 
193
193
  it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
@@ -9,7 +9,7 @@ import { dirname, join } from "node:path";
9
9
  import { tmpDir } from "../tests/_tmp.mjs";
10
10
  import { containerProvider, parseExecution, providerByName, providerLanes, remoteTransportFor } from "./execution.mjs";
11
11
  import { remoteProvider, httpTransport, isActionsEndpoint, redact } from "./sandbox-remote.mjs";
12
- import { resolveCredential } from "./credentials.mjs";
12
+ import { memoKeychain, resolveCredential } from "./credentials.mjs";
13
13
  import { makeLedger } from "./ledger.mjs";
14
14
  import { openBatch } from "./batch.mjs";
15
15
  import { orchestrate } from "./orchestrate.mjs";
@@ -286,6 +286,20 @@ describe("remote config and the Actions constraint (#4254)", () => {
286
286
  assert.equal(resolveCredential("FD_T", { env: {}, keychain: () => { throw new Error("locked"); } }), null);
287
287
  assert.equal(resolveCredential("bad name", { env: { "bad name": "x" } }), null);
288
288
  });
289
+
290
+ it("5159 memoKeychain: one lookup per name per TTL, misses and failures cached as null", () => {
291
+ let t = 0;
292
+ const calls = [];
293
+ const kc = memoKeychain({ keychain: (n) => { calls.push(n); if (n === "BAD") throw new Error("locked"); return n === "HIT" ? "v" : null; }, ttlMs: 1000, now: () => t });
294
+ assert.equal(kc("HIT"), "v"); assert.equal(kc("HIT"), "v");
295
+ assert.equal(kc("MISS"), null); assert.equal(kc("MISS"), null);
296
+ assert.equal(kc("BAD"), null); assert.equal(kc("BAD"), null);
297
+ assert.deepEqual(calls, ["HIT", "MISS", "BAD"]);
298
+ t = 1000;
299
+ assert.equal(kc("MISS"), null);
300
+ assert.deepEqual(calls, ["HIT", "MISS", "BAD", "MISS"], "re-read after the TTL");
301
+ assert.equal(resolveCredential("MISS", { env: { MISS: "e" }, keychain: kc }), "e", "the env still wins");
302
+ });
289
303
  });
290
304
 
291
305
  describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
@@ -245,6 +245,8 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
245
245
  const c = credCache[kind];
246
246
  if (!c || now() - c.at > CRED_TTL[kind]) {
247
247
  const x = await call(cfg.router_url, "GET", `/agent/creds/${kind}`, { token: cfg.token, timeoutMs: 20000 });
248
+ // #5165: every router login at its usage ceiling -> pass the typed 429 (reason, reset) through, uncached, so the lane defers.
249
+ if (x.status === 429 && x.data && typeof x.data === "object") return { status: 429, body: x.data, retryAfter: x.retryAfter };
248
250
  if (!x.ok || !x.data?.token) return { status: 502, body: { error: `router credentials (${kind}): HTTP ${x.status}` } };
249
251
  credCache[kind] = { token: x.data.token, at: now() };
250
252
  }
@@ -397,7 +399,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
397
399
  const url = new URL(req.url, "http://agent");
398
400
  try {
399
401
  const cred = req.method === "GET" && /^\/sandboxes\/([^/]+)\/creds\/([a-z]+)$/.exec(url.pathname);
400
- if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body); }
402
+ if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body, r.retryAfter ? { "retry-after": String(r.retryAfter) } : {}); }
401
403
  const relay = (req.method === "POST" && /^\/sandboxes\/([^/]+)\/relay$/.exec(url.pathname)) || (req.method === "GET" && /^\/sandboxes\/([^/]+)\/relay\/([^/]+)$/.exec(url.pathname));
402
404
  if (relay) { const r = await sandboxRelay(decodeURIComponent(relay[1]), relay[2] ? decodeURIComponent(relay[2]) : null, req); return send(res, r.status, r.body ?? {}); }
403
405
  if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized", hop: "agent", reason: "agent auth: router token rejected by agent" });
@@ -93,19 +93,28 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
93
93
  }
94
94
  return gh.token;
95
95
  }
96
- // Claude access token for sandboxes: the logged-in account on this host with the most validity left. Only the
97
- // short-lived access token is shared; refresh tokens stay here, so this host keeps refreshing its own sessions.
96
+ // Claude access token for sandboxes (#5165): among the logins with 30+ minutes left, the one with the lowest tokenbaton
97
+ // usage (validity breaks ties). Only the short-lived access token is shared; refresh tokens stay here, so this host
98
+ // keeps refreshing its own sessions.
98
99
  function claudeToken() {
99
100
  const dirs = cfg.claude_config_dirs ?? readdirSync(homedir()).filter((d) => /^\.claude(-\d+)?$/.test(d)).map((d) => join(homedir(), d));
100
- let best = null;
101
+ const logins = [];
101
102
  for (const d of dirs) {
102
103
  try {
103
104
  const o = JSON.parse(readFileSync(join(d, ".credentials.json"), "utf8")).claudeAiOauth;
104
- if (o?.accessToken && o.expiresAt > now() + 30 * 60000 && (!best || o.expiresAt > best.expiresAt)) best = o;
105
+ const n = /\.claude-(\d+)$/.exec(d)?.[1];
106
+ if (o?.accessToken) logins.push({ n: n ? Number(n) : null, accessToken: o.accessToken, expiresAt: o.expiresAt });
105
107
  } catch { /* no login in this dir */ }
106
108
  }
107
- if (!best) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
108
- return { token: best.accessToken, expires_at: new Date(best.expiresAt).toISOString() };
109
+ const src = usageSource(cfg);
110
+ const usage = readUsage(src, logins.map((l) => l.n).filter((n) => n !== null));
111
+ const pick = pickClaudeLogin({ logins, usage, ceilingPct: cfg.usage_ceiling_pct ?? 95, now: now() });
112
+ if (pick.status === 503) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
113
+ if (pick.status === 429) { log.warn?.(`claude login: every account at or above the usage ceiling; ${pick.body.reason}`); return pick; }
114
+ if (pick.fallback) log.warn?.(`claude login: no usage source (${src}), falling back to longest validity`);
115
+ const { login, pct } = pick;
116
+ log.info?.(`claude login selected: account=${login.n ?? "default"} usage=${pct ?? "unknown"}% expires_at=${new Date(login.expiresAt).toISOString()}`);
117
+ return { status: 200, body: { token: login.accessToken, expires_at: new Date(login.expiresAt).toISOString() } };
109
118
  }
110
119
  const agentCall = (a, method, path, body, timeoutMs) => call(a.url, method, path, { body, token: a.token, timeoutMs, fetchFn });
111
120
 
@@ -285,7 +294,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
285
294
  [route("GET", "/agents"), fd(async () => ({ status: 200, body: { agents: await capacities(), sandboxes: sbx() } }))],
286
295
  [route("POST", "/codes"), fd(async () => ({ status: 200, body: issueCode() }))],
287
296
  [route("GET", "/agent/creds/github"), { fn: async () => ({ status: 200, body: { token: await githubToken() } }), auth: "agent" }],
288
- [route("GET", "/agent/creds/claude"), { fn: async () => ({ status: 200, body: claudeToken() }), auth: "agent" }],
297
+ [route("GET", "/agent/creds/claude"), { fn: async () => claudeToken(), auth: "agent" }],
289
298
  [route("POST", "/agent/relay/:id"), { fn: async (p, body, _u, req) => relayPost(req, p, body), auth: "agent" }], // #4778
290
299
  [route("GET", "/agent/relay/:id/:rid"), { fn: async (p, _b, _u, req) => relayPoll(req, p), auth: "agent" }],
291
300
  [route("GET", "/agent/bundle"), { fn: async () => ({ bundle: true }), auth: "agent" }],
@@ -355,6 +364,62 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
355
364
  return { handle, launch, claim, tick, enroll, issueCode, capacities, status, store };
356
365
  }
357
366
 
367
+ /** tokenbaton's usage dir (#5165): router.json `usage_source`, else `$TOKENBATON_HOME`, else `~/.tokenbaton`. */
368
+ export function usageSource(cfg, env = process.env) {
369
+ return cfg.usage_source ?? env.TOKENBATON_HOME ?? join(homedir(), ".tokenbaton");
370
+ }
371
+
372
+ /** One tokenbaton `live.<n>` line (`<5h %> <5h reset s> <weekly %> <weekly reset s> [updated s]`), or null; the format of bin/lean/rotator.mjs parseLive. */
373
+ export function parseLiveUsage(text) {
374
+ const f = String(text ?? "").trim().split(/\s+/).map(Number);
375
+ if (f.length < 4 || !f.slice(0, 4).every(Number.isFinite)) return null;
376
+ return { fiveHourPct: f[0], fiveHourResetMs: f[1] * 1000, weeklyPct: f[2], weeklyResetMs: f[3] * 1000 };
377
+ }
378
+
379
+ /** Map account n -> parsed `live.<n>` for the given accounts; unreadable or malformed files are left out. */
380
+ export function readUsage(dir, ns, readFile = (p) => readFileSync(p, "utf8")) {
381
+ const out = new Map();
382
+ for (const n of ns) { try { const u = parseLiveUsage(readFile(join(dir, `live.${n}`))); if (u) out.set(n, u); } catch { /* no usage for n */ } }
383
+ return out;
384
+ }
385
+
386
+ /**
387
+ * Pick the Claude login to hand out (#5165). Candidates have 30+ minutes of validity. A login with known usage is
388
+ * excluded at or above `ceilingPct` 5-hour use or 100% weekly use (a passed reset counts as 0, like the local rotator);
389
+ * the rest rank by 5-hour use, then validity. Logins with unknown usage follow. No usage at all: longest validity,
390
+ * `fallback: true`. Every candidate excluded: a typed 429 carrying the earliest reset.
391
+ * @returns {{status: 200, login, pct, fallback} | {status: 429, retryAfter, body} | {status: 503}}
392
+ */
393
+ export function pickClaudeLogin({ logins, usage, ceilingPct = 95, now }) {
394
+ const valid = logins.filter((l) => l.accessToken && l.expiresAt > now + 30 * 60000);
395
+ if (!valid.length) return { status: 503 };
396
+ const used = (pct, resetMs) => (resetMs <= now ? 0 : pct);
397
+ const byValidity = (a, b) => b.expiresAt - a.expiresAt;
398
+ const known = [], unknown = [], blocked = [];
399
+ for (const l of valid) {
400
+ const u = l.n !== null && l.n !== undefined ? usage.get(l.n) : undefined;
401
+ if (!u) { unknown.push(l); continue; }
402
+ const five = used(u.fiveHourPct, u.fiveHourResetMs), week = used(u.weeklyPct, u.weeklyResetMs);
403
+ if (five >= ceilingPct || week >= 100) {
404
+ const r = [];
405
+ if (five >= ceilingPct) r.push(u.fiveHourResetMs);
406
+ if (week >= 100) r.push(u.weeklyResetMs);
407
+ blocked.push(Math.max(...r));
408
+ } else known.push({ l, pct: five });
409
+ }
410
+ if (known.length) {
411
+ known.sort((a, b) => a.pct - b.pct || byValidity(a.l, b.l));
412
+ return { status: 200, login: known[0].l, pct: known[0].pct, fallback: false };
413
+ }
414
+ if (unknown.length) {
415
+ unknown.sort(byValidity);
416
+ return { status: 200, login: unknown[0], pct: null, fallback: !blocked.length };
417
+ }
418
+ const resetMs = Math.min(...blocked), at = new Date(resetMs).toISOString();
419
+ return { status: 429, retryAfter: Math.max(1, Math.ceil((resetMs - now) / 1000)),
420
+ body: { error: "usage limit", code: "rate_limited", reason: `usage limit hit; resets at ${at}`, reset_at: at } };
421
+ }
422
+
358
423
  export function startRouter(cfg, deps) {
359
424
  const router = createRouter(cfg, deps);
360
425
  const server = createServer(router.handle);
@@ -385,6 +385,8 @@ async function finalize(a, io) {
385
385
  // Incomplete-panel guard + 6.75 delivery
386
386
  const d = await bash(io, "review-staging-post.sh", ["delivery", ...common], env);
387
387
  relay(io, d);
388
+ // Exit 2 = the comments could not be listed on any attempt (forge#5166): rerun, never re-dispatch reviewers that may have posted.
389
+ if (d.code === 2) throw new Stop(1, ["REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery could not be verified (GitHub comment listing failed); rerun --finalize, do not re-dispatch reviewers", BLOCK_DEPLOY]);
388
390
  if (d.code !== 0) throw new Stop(1, [`REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery incomplete (exit ${d.code}); re-dispatch each missing reviewer once with the note "structured findings block required", else the panel is degraded`, BLOCK_DEPLOY]);
389
391
 
390
392
  // 6.5 verdict handling (override, posture, test-gate markers)
package/bin/runner.mjs CHANGED
@@ -1352,6 +1352,27 @@ export function detectUsageLimit(stdout, output = "") {
1352
1352
  return hasUsageLimitResetClause(output || "");
1353
1353
  }
1354
1354
 
1355
+ // forge#5168: the claude CLI's own login/authentication failure texts. Literal
1356
+ // alternatives only (linear on untrusted input).
1357
+ const AUTH_FAILURE_RE = /Failed to authenticate|authentication_error|OAuth token (?:has )?expired|reached its time limit and cannot be renewed|Invalid API key|Please run \/login/i;
1358
+
1359
+ /**
1360
+ * Detects whether a failed CLI run was an authentication failure of the claude
1361
+ * login (forge#5168): an expired or invalid login is an account condition, not
1362
+ * a transient engine error. With an envelope on stdout, it must be
1363
+ * `is_error: true` with a `result` naming the failure; without one, the plain
1364
+ * combined output must. `stdout` is parsed ALONE (forge#2422); never throws.
1365
+ *
1366
+ * @param {string} stdout - raw stdout of the CLI run
1367
+ * @param {string} [output] - combined stdout+stderr, for the plain-text fallback
1368
+ * @returns {boolean}
1369
+ */
1370
+ export function detectAuthFailure(stdout, output = "") {
1371
+ const env = parseCliEnvelope(stdout);
1372
+ if (env) return env.is_error === true && typeof env.result === "string" && AUTH_FAILURE_RE.test(env.result);
1373
+ return typeof output === "string" && AUTH_FAILURE_RE.test(output);
1374
+ }
1375
+
1355
1376
  /**
1356
1377
  * Best-effort extraction of a Claude CLI session-limit reset time from
1357
1378
  * captured stdout/stderr (forge#2241). The CLI's own session-limit message
@@ -1844,6 +1865,9 @@ export function runCliBackend({
1844
1865
  // default when the reset time itself is missing or unparseable. The
1845
1866
  // engine, not the runner, owns that default — no epoch is fabricated here.
1846
1867
  if (isUsageLimit) err.usageLimit = true;
1868
+ // forge#5168: a failed login (never when the usage limit already matched,
1869
+ // which keeps precedence) lets the lane defer as `backend.auth`.
1870
+ else if (detectAuthFailure(stdout, output)) err.authFailure = true;
1847
1871
  if (resetAt) {
1848
1872
  err.resetAt = resetAt;
1849
1873
  // forge#2524: also attach a machine-usable epoch-ms timestamp so
@@ -401,14 +401,14 @@ node "$FD_ROOT/bin/forgedock.mjs" review-staging "$PR_NUMBER" --finalize --test-
401
401
  ```
402
402
 
403
403
  The call runs these steps in order:
404
- 1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying.
404
+ 1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying. The PR comments are listed once per attempt. A missing reviewer or a GitHub API error is re-listed with backoff (5 s, 15 s, 45 s) before any verdict, and each domain is reported with the attempt it was found on.
405
405
  2. The Phase 6.5 verdict handling.
406
406
  3. `scripts/review-finalize.sh`:
407
407
  - `extract`: trusted comments only, dedup, and the `commands/review-pr.md` §6B.4 provenance and concreteness gates. **CRITICAL/HIGH are never routed as pre-existing**, whatever their confidence.
408
408
  - `classify --dry-run`, then `classify` for real. This passes `--scope-files` (the 0B.1 scoped set when `SCOPE_MODE=scoped`) and `--prior-prs` (the bundle members), and posts `FORGE:NOTE_DISPOSITION`.
409
409
  - `file-prepare`.
410
410
 
411
- Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A.
411
+ Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A. `REVIEW DELIVERY UNVERIFIED` means GitHub could not list the comments on the last attempt. It stops the run, but nothing is labelled or posted. Do not re-dispatch anyone: rerun 7A once GitHub answers.
412
412
 
413
413
  Exit 0 prints the following:
414
414
  - `TEST_GATE:`, `SCOPE_MODE=`.
package/docs/CONFIG.md CHANGED
@@ -438,6 +438,7 @@ orchestration:
438
438
  | `max_concurrent` | integer | No | Max concurrent top-level `/work-on` dispatches. It does not bound cascade growth; under `policy: all` it raises drain and spawn throughput together. Default: 12. |
439
439
  | `breaker.errors` / `breaker.window_min` | integer | No | `forgedock orchestrate` circuit breaker (#4121): after `errors` engine-errors within `window_min` minutes the batch stops admitting lanes, lets running lanes finish, and keeps the rest pending for `--resume`. `errors: 0` turns it off. Default: 3 in 30 |
440
440
  | `max_deferrals` | integer | No | `forgedock orchestrate`: requeues of one issue after a non-host `deferred` (usage limit, merge/push transient, CLI idle, lease held, gh unavailable) before its row is final. `host.busy` and `sandbox.capacity` (sandbox pool full) deferrals are not capped. Default: 3 |
441
+ | `transient_retries` | integer | No | `forgedock orchestrate` (#5161): backoff retries of an issue whose lane ended `engine-error` with a transport or gateway failure in its detail (curl `returned error: 5xx`, `HTTP/1.1 5xx`, `Bad Gateway`, `Service Unavailable`, `overloaded`, connection reset/refused/timeout). Delays 30s, 2m, 8m, then 15m each; dependents stay pending meanwhile. Not counted against the single engine-error relaunch; once spent, the row ends `engine-error`. Every such error still counts toward the breaker. `0` turns transient retries off. Default: 4 |
441
442
  | `blocked_label` | string | No | `forgedock orchestrate` (#4773): an open issue carrying this label is admitted to the batch but not dispatched; removing the label releases it on the next scheduler pass. A `Blocked by #N` / `Depends on #N` line in the body of a trusted author (OWNER, MEMBER, COLLABORATOR or a bot) is a DAG edge to #N, and `--add` prints the edges it created. Empty turns the label off. Default: `blocked` |
442
443
  | `auto_add_children` | string | No | `forgedock orchestrate` (#4965): which children a running batch adds by itself. `off` (no auto-adds; a decomposed lane's sub-issues stay out too), `decomposed` (a decomposed lane's sub-issues join, as before), `epic` (also the open children of every batch issue and of each `--epic N`, found by a `**Parent**: #P` line, the `parent` field or GitHub sub-issues, including children filed after the batch started) or `all`. Only children whose author is trusted (an owner, member or collaborator, or a bot, the `Blocked by` rule) are auto-added; any other is skipped with a logged row. Children labelled `needs-human` and children another live batch of the repo still holds are not added. Each pass that adds posts one batch decision and one `added` row listing the children and edges. `forgedock orchestrate --auto-children <mode>` overrides it per batch and `--resume` keeps it. Default: `decomposed` |
443
444
  | `auto_add_children_poll_s` | integer | No | Seconds between child scans for `epic`/`all` (one `gh issue list` per scan). Default: `120` |
@@ -470,6 +471,43 @@ orchestration:
470
471
 
471
472
  ---
472
473
 
474
+ ## `orchestrate.health` and `notify` (OPTIONAL)
475
+
476
+ Early batch health (#5167). `forgedock orchestrate` checks each batch at T+15 and T+60 minutes after it was created (`plan.json` `created`), and once more when the batch ends. When the batch starts, it prints the schedule: `health checks: T+15 14:05Z, T+60 14:50Z, and at batch end; forgedock status --batch <id>`. The first unhealthy check raises one `BATCH UNHEALTHY <id>` alert, which lists the top causes and the exact recovery commands (`--requeue`, a relaunch, `--resume`, `--cancel`). `health.json` in the batch dir records the checkpoints already done and the alert, so a daemon restart arms only the remaining checks and a batch never alerts twice. Checks and alerts never fail, delay or change the batch.
477
+
478
+ Causes: `root-failed` (a root issue, meaning one with no dependency inside the batch, ended non-merged); `terminal-share` (the share of rows ended `skipped`/`held`/`engine-error`/`needs-human` or another non-merged outcome except `invalid`/`decomposed` is at or above the threshold, with nothing merged); `stalled` (0 lanes running while issues are pending, operator-held issues excepted); `refused` (the launch was refused).
479
+
480
+ ```yaml
481
+ orchestrate:
482
+ health:
483
+ root_failed: true # any root ended non-merged is unhealthy (default true)
484
+ terminal_share: 0.5 # 0..1, terminal-non-merged share with 0 merged (default 0.5)
485
+ stalled: true # 0 running while pending > 0 (default true)
486
+ issue: 123 # comment target; default: the batch's --epic issue(s); a milestone gets no comment
487
+ checks_min: [15, 60] # minutes after launch (default [15, 60])
488
+ notify:
489
+ command: "curl -fsS -H 'content-type: application/json' -d @- https://hooks.example/forge"
490
+ ```
491
+
492
+ | Key | Default | Meaning |
493
+ |---|---|---|
494
+ | `orchestrate.health.root_failed` | `true` | Raise `root-failed`. |
495
+ | `orchestrate.health.terminal_share` | `0.5` | Threshold for `terminal-share`. |
496
+ | `orchestrate.health.stalled` | `true` | Raise `stalled`. |
497
+ | `orchestrate.health.issue` | none | Issue to comment on. When unset, the alert comments on each `--epic` issue. |
498
+ | `orchestrate.health.checks_min` | `[15, 60]` | Timed checkpoints, in minutes after launch. |
499
+ | `notify.command` | none | Shell command run once per alert. It gets the alert JSON (`{alert, batch, repo, text, causes, recovery}`) on stdin and `FORGEDOCK_ALERT=BATCH_UNHEALTHY`, `FORGEDOCK_ALERT_BATCH` and `FORGEDOCK_ALERT_TEXT` in its env. It is killed after 30 s. A failure only prints a `health: notify.command failed` line. |
500
+
501
+ The keys are flattened one level by the parser, the same as `orchestrate.drift.*`. Health keys alone do not turn drift on.
502
+
503
+ Every channel is best effort: the runner line, the GitHub comment (marker `FORGE:BATCH-HEALTH`), `notify.command`, and the `forgedock status` line. `forgedock status` lists the unhealthy batches that are unresolved (no report yet, or alerted within the last day) and the launch refusals from the last day. `forgedock status --batch <id>` shows that batch's health (healthy, or unhealthy with causes and recovery) and its pending checkpoints.
504
+
505
+ A refused milestone start (#4659, `parallel milestone start refused: …`) still exits 1 with the error on stderr and creates no batch dir. It is also recorded in `<batch root>/refusals.json` (the newest 20 are kept) and sends the same alert, with the merge-the-dependency or `--force "<reason>"` relaunch.
506
+
507
+ **Commands that use this section**: `forgedock orchestrate`, `forgedock status`
508
+
509
+ ---
510
+
473
511
  ## `decisions` (OPTIONAL)
474
512
 
475
513
  Decision engine (#4766, see [DECISIONS.md](DECISIONS.md)). The issue-watch stage (#4793) calls `settle` for the open `needs-human` issues a watch selects: it runs when `mode: live` (a repo setting: every watch of the repo runs the stage), or when the watch sets `sweep_needs_human: true`; `policy` and `human_only` are also read by prodwatch (`loadPolicy`). The parser has one nesting level and no arrays, so lists are comma-separated.
@@ -1185,7 +1223,7 @@ decomposition:
1185
1223
 
1186
1224
  **With `close_parent: false`** (default): unchanged. The parent stays open and is closed by C3 when the last child closes.
1187
1225
 
1188
- The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `not planned`, keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
1226
+ The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `completed` (#5152; scope moved to the sub-issues, not dropped), keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
1189
1227
 
1190
1228
  **Limitation**: `--retry` and engine resume refuse closed issues, so a closed decomposed parent cannot be re-run. It has nothing left to run; work on its open children instead.
1191
1229
 
@@ -44,6 +44,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
44
44
  | `gh.unavailable` | `run-issue --repo` could not verify the cwd repo because `gh repo view` kept failing after 3 attempts with backoff (typically a timeout under host load); the core or full run is `deferred` (exit 75, retry in 60s) before any lease, nothing posted | engine CLI | `bin/engine-cli.mjs` |
45
45
  | `gh.auth` | A `gh` call in a core run answered HTTP 401 (expired or revoked bot token) and one run of `scripts/refresh-bot-token.sh` plus a retry did not fix it; the run is `deferred` (retry in 120s), never `engine-error`. See `docs/CONFIG.md` "Bot token refresh" | lean | `bin/lean/retry.mjs` |
46
46
  | `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
47
+ | `backend.auth` | The claude CLI login failed ("Failed to authenticate", expired or invalid login); the lane is `deferred` (retry in 900s) under a per-account hold like a usage limit, never `engine-error`, never relaunched or counted by the breaker (#5168) | lean | `bin/runner.mjs` `err.authFailure` |
47
48
  | `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
48
49
  | `engine.dco_rewrite` | Core work-on (forge#4997): the DCO sign-off rewrite (`git filter-branch`) of the lane's unpushed commits failed, or the lane's uncommitted changes could not be restored after it (kept on `refs/forgedock/dco-stash`). A mechanical engine step, not a human block: the lane ends `engine-error` (never `needs-human`), which `forgedock orchestrate` relaunches like any engine-error | lean | `bin/lean/signoff.mjs` `ensureSignedOff` |
49
50
  | `engine.validate_result_lost` | Core work-on (forge#4708): a validate result recorded before a merge-time CI fix round is missing from the fix session's result file and could not be carried forward; an engine bookkeeping error, not an unimplemented requirement | lean | `bin/lean/lean.mjs` `mergeFixRound` |
@@ -70,7 +71,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
70
71
  | `lane.branch_missing` | Core work-on: the lane branch (e.g. `milestone/<slug>`) does not exist on origin; fail closed | lean | `git ls-remote --heads origin` |
71
72
  | `lane.unresolved` | Core work-on: the lane could not be worked out (milestone or default branch unreadable, empty milestone slug, bad `feature_pattern`); fail closed | lean | Failed `gh` read or config |
72
73
  | `invalid` | Issue is invalid or in a divergent closed state | investigate, engine | `INVESTIGATION:INVALID` marker, or live snapshot |
73
- | `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `not planned` (label kept) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
74
+ | `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `completed` (label kept; #5152: its scope moved, it was not dropped) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
74
75
  | `builder.blocked` | Core work-on: the builder reported an external action agents cannot take, or a product decision too critical for agents: `block.kind: "human"` with `why` one of `credentials`, `spend`, `promotion`, `customer`, `product-decision`. An upstream issue, a needed split, or prod data that monitoring access can read never count | lean | `result.json` status `blocked` + typed `block` |
75
76
  | `builder.block_untyped` | Core work-on: the builder stayed `blocked` without a valid typed reason (missing `block`, a non-human `why` such as prod data, or a dependency that is not open) after one corrective resume | lean | `result.json` status `blocked`, twice |
76
77
  | `builder.no_evidence` | Core work-on: the builder reported invalid/already-fixed without file:line evidence, so the issue was not closed | lean | `result.json` with empty `evidence` |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "forgedock",
3
- "version": "1.50.0",
3
+ "version": "1.51.0",
4
4
  "description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
5
5
  "bin": {
6
6
  "forgedock": "bin/forgedock.mjs"
package/pool/README.md CHANGED
@@ -10,4 +10,6 @@ ForgeDock's built-in remote compute pool (imported from computeshare; AGPL-3.0-o
10
10
 
11
11
  The module boundary is the ForgeDock HTTP sandbox protocol: the router speaks it to the dispatcher and relays to agents over HTTP. Sandboxes receive only short-lived GitHub and Claude access tokens; refresh tokens stay on the router host.
12
12
 
13
+ Claude login choice (#5165): among the router host's logins with 30+ minutes left, the router hands out the one with the lowest 5-hour use in tokenbaton's `live.<n>` (account `n` = `~/.claude-<n>`), read from router.json `usage_source` (default `$TOKENBATON_HOME`, else `~/.tokenbaton`); validity only breaks ties. An account at or above `usage_ceiling_pct` (default 95) or at 100% weekly use is skipped; when every login is, `/agent/creds/claude` answers a typed 429 (`code: rate_limited`, `reset_at` = the earliest reset, `retry-after`), which the agent passes through and `cs-cred` prints as `usage limit hit; resets at <ISO>` so the lane defers. Without readable usage it falls back to the longest validity and logs a warning. The selected account index and usage are logged, never the token.
14
+
13
15
  The `forgedock pool` CLI wiring is tracked separately (#4544).
@@ -7,6 +7,13 @@ kind="${1:?kind}"; ttl=240; [ "$kind" = claude ] && ttl=600
7
7
  cache="/tmp/.cs-cred-$kind"
8
8
  if [ -s "$cache" ] && [ $(( $(date +%s) - $(stat -c %Y "$cache") )) -lt $ttl ]; then cat "$cache"; exit 0; fi
9
9
  [ -n "${CS_CRED_BASE:-}" ] || { echo "no credential relay (CS_CRED_BASE)" >&2; exit 1; }
10
- tok=$(curl -fsS --max-time 30 -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind" | jq -r .token)
10
+ body=$(mktemp); trap 'rm -f "$body"' EXIT
11
+ code=$(curl -sS --max-time 30 -o "$body" -w '%{http_code}' -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind") || code=000
12
+ if [ "$code" = 429 ]; then
13
+ # #5165: every router login is at its usage ceiling; this line is what the lane's log tail defers on.
14
+ why=$(jq -r '.reason // empty' "$body" 2>/dev/null || true)
15
+ echo "${why:-usage limit hit; resets at unknown}" >&2; exit 1
16
+ fi
17
+ tok=""; [ "$code" = 200 ] && tok=$(jq -r .token "$body" 2>/dev/null || true)
11
18
  [ -n "$tok" ] && [ "$tok" != null ] || { echo "credential fetch failed ($kind)" >&2; exit 1; }
12
19
  umask 077; printf '%s' "$tok" > "$cache"; printf '%s' "$tok"
@@ -10,12 +10,15 @@
10
10
  # delivery counts trusted, current-head FORGE:REVIEW-AGENT comments (findings block required) for every domain listed in
11
11
  # DIR/<N>_launched_review_agents. A shortfall labels the PR review-degraded + needs-human and posts
12
12
  # FORGE:GATE_FAILURE:TYPE=review-panel-integrity; an empty/unreadable list or a missing domain is a delivery failure.
13
- # Exit 0 = every launched reviewer delivered, 1 = not.
13
+ # The comments are listed once per attempt; a missing domain or an API error re-lists after each
14
+ # FORGE_REVIEW_DELIVERY_RETRY_DELAYS wait (seconds, default "5 15 45"). An API error on the last attempt is
15
+ # REVIEW DELIVERY UNVERIFIED (exit 2, nothing labelled or posted), never "missing" (forge#5166).
16
+ # Exit 0 = every launched reviewer delivered, 1 = not, 2 = could not be verified.
14
17
  # test-gate reads the saved /test-gate output FILE (the last FORGE:TEST_GATE:RESULT= marker; an absent marker stays empty and
15
18
  # fails closed as BLOCK, never SKIP), applies the trusted override, the posture and the FORGE:GATE_PASS /
16
19
  # FORGE:GATE_FAILURE test-gate comments, and persists DIR/<N>_test_gate_verdict and _reason.
17
20
  # Exit 0 = continue, 1 = RESULT: BLOCK DEPLOY.
18
- # Exit: 2 on a usage error.
21
+ # Exit: 2 on a usage error (and delivery UNVERIFIED).
19
22
  #
20
23
  # The `# BEGIN <name>` ... `# END <name>` blocks below are the spec's former fences, copied without their forge_resolve locator
21
24
  # stubs (this script resolves helpers once, below). Helpers (trusted-comments.sh) resolve through forge_resolve from trusted
@@ -59,24 +62,77 @@ type forge_resolve >/dev/null 2>&1 || { setopt nonomatch 2>/dev/null || true; _S
59
62
 
60
63
  if [ "$CMD" = delivery ]; then
61
64
  echo "=== Incomplete-panel guard and Phase 6.75: verify agent delivery ==="
65
+ # BEGIN delivery-fetch
66
+ # The PR's comments are listed ONCE per attempt into a cache file; the panel guard and the per-domain check both read it (forge#5166).
67
+ # An attempt is an API error unless gh exits 0 and every page is a JSON array: an error is never evidence that a comment is absent.
68
+ DELIVERY_CACHE="$_SD/${PR_NUMBER}_delivery_comments.json"
69
+ delivery_fetch() {
70
+ rm -f "$DELIVERY_CACHE" "$DELIVERY_CACHE.tmp"
71
+ gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" > "$DELIVERY_CACHE.tmp" 2>/dev/null || { rm -f "$DELIVERY_CACHE.tmp"; return 1; }
72
+ jq -se 'length > 0 and all(.[]; type == "array")' "$DELIVERY_CACHE.tmp" >/dev/null 2>&1 || { rm -f "$DELIVERY_CACHE.tmp"; return 1; }
73
+ mv -f "$DELIVERY_CACHE.tmp" "$DELIVERY_CACHE"
74
+ }
75
+ # Unique trusted current-head reviewer domains in the cached listing (findings block not required), one per line.
76
+ delivery_domains() {
77
+ bash "$TRUSTED_SCRIPT" bodies "^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" < "$DELIVERY_CACHE" 2>/dev/null \
78
+ | jq -r 'scan("^<!-- FORGE:REVIEW-AGENT:([a-z0-9-]+) -->") | .[0]' 2>/dev/null | sort -u
79
+ }
80
+ # Trusted current-head comments for one domain that carry a findings block, in the cached listing.
81
+ delivery_count() {
82
+ local AGENT_DOMAIN="$1" n
83
+ n=$(bash "$TRUSTED_SCRIPT" count "^(?=[\\s\\S]*<!-- REVIEW-FINDINGS-START -->[\\s\\S]*<!-- REVIEW-FINDINGS-END -->)<!-- FORGE:REVIEW-AGENT:${AGENT_DOMAIN} -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" < "$DELIVERY_CACHE" 2>/dev/null)
84
+ case "$n" in ''|*[!0-9]*) n=0 ;; esac # fail closed: counts as missing
85
+ printf '%s\n' "$n"
86
+ }
87
+ # END delivery-fetch
62
88
  # BEGIN panel-guard
63
89
  # Each Bash call is a fresh shell: re-resolve the full head SHA if it is not set, and refuse when it is not a commit id
64
90
  # (an empty SHA would make the Reviewed-SHA match accept any head).
65
91
  [ -n "${REVIEW_SHA_FULL:-}" ] || REVIEW_SHA_FULL=$(gh pr view "$PR_NUMBER" -R "$GH_REPO" --json headRefOid --jq '.headRefOid' 2>/dev/null || echo "")
66
92
  case "$REVIEW_SHA_FULL" in *[!0-9a-f]*|"") REVIEW_SHA_FULL="" ;; *) [ "${#REVIEW_SHA_FULL}" -eq 40 ] || [ "${#REVIEW_SHA_FULL}" -eq 64 ] || REVIEW_SHA_FULL="" ;; esac
67
93
  TRUSTED_SCRIPT=$(forge_resolve trusted-comments.sh)
68
- ACTUAL_AGENT_COUNT=0
69
- if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ]; then
70
- # Only TRUSTED, current-head reviewer comments count, over every page of comments. Unique domains, so a duplicate does not inflate the count.
71
- ACTUAL_AGENT_COUNT=$(gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
72
- | bash "$TRUSTED_SCRIPT" bodies "^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" 2>/dev/null \
73
- | jq -r 'scan("^<!-- FORGE:REVIEW-AGENT:([a-z0-9-]+) -->") | .[0]' 2>/dev/null | sort -u | grep -c . || echo 0)
74
- fi
75
- case "$ACTUAL_AGENT_COUNT" in ''|*[!0-9]*) ACTUAL_AGENT_COUNT=0 ;; esac # fail closed: unreadable or untrusted counts as not posted
76
94
  # Fresh shell: SELECTED_AGENT_COUNT is not carried over. Reload it from the PR-keyed launched-agent file, then fail closed when it is empty, non-integer or zero (a staging review always selects at least one reviewer).
77
95
  [ -n "${SELECTED_AGENT_COUNT:-}" ] || { [ -n "${PR_NUMBER:-}" ] && [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && SELECTED_AGENT_COUNT=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | grep -c .); }
78
96
  SELECTED_COUNT_INVALID=false
79
97
  case "${SELECTED_AGENT_COUNT:-}" in ''|*[!0-9]*|0) SELECTED_COUNT_INVALID=true; SELECTED_AGENT_COUNT=0 ;; esac
98
+ # Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only).
99
+ [ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] || { [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && LAUNCHED_REVIEW_AGENTS=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | tr '\n' ' '); }
100
+ ACTUAL_AGENT_COUNT=0
101
+ DELIVERY_LAST=skipped # ok = the last attempt was a successful listing, error = it was an API error
102
+ DELIVERY_FOUND=" " # launched domains with a findings block, found on any attempt (found stays found)
103
+ DELIVERY_SEEN="" # union of trusted current-head reviewer domains over the successful listings
104
+ if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ] && [ "${DISPATCH_FAILED:-}" != "true" ]; then
105
+ if [ "$SELECTED_COUNT_INVALID" = "true" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
106
+ # Nothing to wait for: one listing for the report only, no retry.
107
+ if delivery_fetch; then DELIVERY_LAST=ok; DELIVERY_SEEN=$(delivery_domains); else DELIVERY_LAST=error; fi
108
+ else
109
+ # Retry on listing lag or an API error, after each FORGE_REVIEW_DELIVERY_RETRY_DELAYS wait (seconds; non-integers ignored).
110
+ _ATTEMPT=0
111
+ for _DELAY in 0 ${FORGE_REVIEW_DELIVERY_RETRY_DELAYS-5 15 45}; do
112
+ case "$_DELAY" in ''|*[!0-9]*) continue ;; esac
113
+ _ATTEMPT=$((_ATTEMPT + 1))
114
+ [ "$_ATTEMPT" -gt 1 ] && [ "$_DELAY" -gt 0 ] && sleep "$_DELAY"
115
+ if ! delivery_fetch; then DELIVERY_LAST=error; echo "DELIVERY: attempt ${_ATTEMPT}: API error listing PR comments"; continue; fi
116
+ DELIVERY_LAST=ok
117
+ DELIVERY_SEEN=$(printf '%s\n%s\n' "$DELIVERY_SEEN" "$(delivery_domains)" | grep -E '^[a-z0-9-]+$' | sort -u)
118
+ _PENDING=""
119
+ for _D in $LAUNCHED_REVIEW_AGENTS; do
120
+ case "$DELIVERY_FOUND" in *" $_D "*) continue ;; esac
121
+ if [ "$(delivery_count "$_D")" -ge 1 ]; then DELIVERY_FOUND="${DELIVERY_FOUND}${_D} "; echo "DELIVERY: ${_D} found on attempt ${_ATTEMPT}"; else _PENDING="${_PENDING} ${_D}"; fi
122
+ done
123
+ [ -z "$_PENDING" ] && break
124
+ echo "DELIVERY: attempt ${_ATTEMPT}: missing:${_PENDING}"
125
+ done
126
+ fi
127
+ ACTUAL_AGENT_COUNT=$(printf '%s\n' "$DELIVERY_SEEN" | grep -c . || true)
128
+ fi
129
+ case "$ACTUAL_AGENT_COUNT" in ''|*[!0-9]*) ACTUAL_AGENT_COUNT=0 ;; esac # fail closed: unreadable or untrusted counts as not posted
130
+ if [ "${DISPATCH_FAILED:-}" != "true" ] && [ "$SELECTED_COUNT_INVALID" != "true" ] && [ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] && [ "$DELIVERY_LAST" = error ]; then
131
+ # Never "missing": the comments could not be listed, so nothing is labelled or posted (forge#5166).
132
+ echo "REVIEW DELIVERY UNVERIFIED: could not list PR comments after ${_ATTEMPT:-1} attempt(s) (GitHub API error); found so far:${DELIVERY_FOUND% }"
133
+ echo "Do not re-dispatch reviewers: rerun the finalize step once GitHub answers."
134
+ exit 2
135
+ fi
80
136
  if [ "${DISPATCH_FAILED:-}" = "true" ] || [ "$SELECTED_COUNT_INVALID" = "true" ] || [ "$ACTUAL_AGENT_COUNT" -lt "$SELECTED_AGENT_COUNT" ]; then
81
137
  gh label create "review-degraded" --color "E4E669" --description "PR review panel was incomplete; re-review required before deployment. Managed by ForgeDock." --force -R "$GH_REPO" 2>/dev/null || true
82
138
  gh pr edit "$PR_NUMBER" -R "$GH_REPO" --add-label "review-degraded" --add-label "needs-human" 2>/dev/null || true # allowlist:check-command-side-effects
@@ -91,7 +147,7 @@ Re-run the full staging review in a fresh session before deployment."
91
147
  fi
92
148
  # END panel-guard
93
149
  # BEGIN delivery-check
94
- # Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only).
150
+ # Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only); a no-op when the panel guard loaded it.
95
151
  [ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] || { [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && LAUNCHED_REVIEW_AGENTS=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | tr '\n' ' '); }
96
152
  # Fail closed: a staging review always launches at least the Phase 2 material-change agent, so an empty list means lost state or skipped bookkeeping, never "nothing to verify".
97
153
  if [ -z "${PR_NUMBER:-}" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
@@ -100,21 +156,9 @@ if [ -z "${PR_NUMBER:-}" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
100
156
  exit 1
101
157
  fi
102
158
  MISSING_AGENT_COMMENTS=""
103
- # Each Bash call is a fresh shell: re-resolve the full head SHA if it is not set, and refuse when it is not a commit id
104
- # (an empty SHA would make the Reviewed-SHA match accept any head).
105
- [ -n "${REVIEW_SHA_FULL:-}" ] || REVIEW_SHA_FULL=$(gh pr view "$PR_NUMBER" -R "$GH_REPO" --json headRefOid --jq '.headRefOid' 2>/dev/null || echo "")
106
- case "$REVIEW_SHA_FULL" in *[!0-9a-f]*|"") REVIEW_SHA_FULL="" ;; *) [ "${#REVIEW_SHA_FULL}" -eq 40 ] || [ "${#REVIEW_SHA_FULL}" -eq 64 ] || REVIEW_SHA_FULL="" ;; esac
107
- TRUSTED_SCRIPT=$(forge_resolve trusted-comments.sh)
159
+ # Counted once per attempt from the cached listing above (delivery_count); a domain not found on any attempt is missing.
108
160
  for AGENT_DOMAIN in $LAUNCHED_REVIEW_AGENTS; do
109
- AGENT_COMMENT_COUNT=0
110
- if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ]; then
111
- AGENT_COMMENT_COUNT=$(gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
112
- | bash "$TRUSTED_SCRIPT" count "^(?=[\\s\\S]*<!-- REVIEW-FINDINGS-START -->[\\s\\S]*<!-- REVIEW-FINDINGS-END -->)<!-- FORGE:REVIEW-AGENT:${AGENT_DOMAIN} -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" 2>/dev/null || echo 0)
113
- fi
114
- case "$AGENT_COMMENT_COUNT" in ''|*[!0-9]*) AGENT_COMMENT_COUNT=0 ;; esac # fail closed: counts as missing
115
- if [ "${AGENT_COMMENT_COUNT:-0}" -lt 1 ]; then
116
- MISSING_AGENT_COMMENTS="${MISSING_AGENT_COMMENTS} ${AGENT_DOMAIN}"
117
- fi
161
+ case "$DELIVERY_FOUND" in *" ${AGENT_DOMAIN} "*) ;; *) MISSING_AGENT_COMMENTS="${MISSING_AGENT_COMMENTS} ${AGENT_DOMAIN}" ;; esac
118
162
  done
119
163
 
120
164
  if [ -n "$MISSING_AGENT_COMMENTS" ]; then
@@ -123,6 +167,7 @@ if [ -n "$MISSING_AGENT_COMMENTS" ]; then
123
167
  exit 1
124
168
  fi
125
169
  # END delivery-check
170
+ rm -f "$DELIVERY_CACHE"
126
171
  echo "DELIVERY: OK (${ACTUAL_AGENT_COUNT}/${SELECTED_AGENT_COUNT} launched reviewers posted a findings block for head ${REVIEW_SHA_FULL})"
127
172
  exit 0
128
173
  fi