forgedock 1.50.0 → 1.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +2 -0
- package/bin/engine-cli.mjs +90 -9
- package/bin/forgedock.mjs +30 -2
- package/bin/lean/batch-health.mjs +219 -0
- package/bin/lean/batch-health.test.mjs +156 -0
- package/bin/lean/batch.mjs +3 -3
- package/bin/lean/batch.test.mjs +236 -8
- package/bin/lean/conformance.mjs +50 -5
- package/bin/lean/conformance.test.mjs +39 -1
- package/bin/lean/context.mjs +8 -2
- package/bin/lean/context.test.mjs +11 -1
- package/bin/lean/credentials.mjs +18 -0
- package/bin/lean/execution.mjs +11 -6
- package/bin/lean/execution.test.mjs +28 -0
- package/bin/lean/fixclosereasons.mjs +52 -0
- package/bin/lean/fixclosereasons.test.mjs +64 -0
- package/bin/lean/lean.mjs +28 -17
- package/bin/lean/lean.test.mjs +74 -13
- package/bin/lean/orchestrate-blocked.test.mjs +47 -0
- package/bin/lean/orchestrate.mjs +124 -12
- package/bin/lean/prefilter.mjs +4 -3
- package/bin/lean/prefilter.test.mjs +5 -4
- package/bin/lean/prompts.mjs +4 -2
- package/bin/lean/recovery.test.mjs +5 -5
- package/bin/lean/sandbox-provider.contract.test.mjs +15 -1
- package/bin/pool/agent.mjs +3 -1
- package/bin/pool/router.mjs +72 -7
- package/bin/review-staging.mjs +2 -0
- package/bin/runner.mjs +24 -0
- package/commands/review-pr-staging.md +2 -2
- package/docs/CONFIG.md +39 -1
- package/docs/OUTCOME-CODES.md +2 -1
- package/package.json +1 -1
- package/pool/README.md +2 -0
- package/pool/image/cs-cred +8 -1
- package/scripts/review-staging-post.sh +70 -25
- package/scripts/review-staging-post.test.sh +40 -8
|
@@ -65,9 +65,9 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
65
65
|
assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
|
|
66
66
|
assert.equal(r[1].outcome, "needs-human");
|
|
67
67
|
assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
|
|
68
|
-
assert.equal(r[2].outcome, "
|
|
68
|
+
assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
|
|
69
69
|
const lines = formatOutcomeLists(rows);
|
|
70
|
-
assert.equal(lines.length,
|
|
70
|
+
assert.equal(lines.length, 2); assert.match(lines[0], /^held subtree: #1 \(needs-human\) holds #2; release: forgedock orchestrate --requeue <batch> 1$/); assert.match(lines[1], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
|
|
71
71
|
});
|
|
72
72
|
|
|
73
73
|
it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
|
|
@@ -92,7 +92,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
92
92
|
assert.equal(calls.filter((c) => c.n === 1).length, 1);
|
|
93
93
|
assert.equal(r[1].outcome, "budget-exceeded");
|
|
94
94
|
assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
|
|
95
|
-
assert.equal(r[2].outcome, "
|
|
95
|
+
assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
|
|
96
96
|
assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
|
|
97
97
|
});
|
|
98
98
|
|
|
@@ -124,7 +124,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
124
124
|
assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
|
|
125
125
|
assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
|
|
126
126
|
assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
|
|
127
|
-
assert.equal(r[2].outcome, "
|
|
127
|
+
assert.equal(r[2].outcome, "held"); assert.equal(r[5].outcome, "merged");
|
|
128
128
|
const lines = formatOutcomeLists(rows);
|
|
129
129
|
assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
|
|
130
130
|
const esc = lines.find((l) => l.startsWith("escalations"));
|
|
@@ -187,7 +187,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
187
187
|
const calls = [];
|
|
188
188
|
const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
|
|
189
189
|
runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
|
|
190
|
-
assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "
|
|
190
|
+
assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
|
|
191
191
|
});
|
|
192
192
|
|
|
193
193
|
it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
|
|
@@ -9,7 +9,7 @@ import { dirname, join } from "node:path";
|
|
|
9
9
|
import { tmpDir } from "../tests/_tmp.mjs";
|
|
10
10
|
import { containerProvider, parseExecution, providerByName, providerLanes, remoteTransportFor } from "./execution.mjs";
|
|
11
11
|
import { remoteProvider, httpTransport, isActionsEndpoint, redact } from "./sandbox-remote.mjs";
|
|
12
|
-
import { resolveCredential } from "./credentials.mjs";
|
|
12
|
+
import { memoKeychain, resolveCredential } from "./credentials.mjs";
|
|
13
13
|
import { makeLedger } from "./ledger.mjs";
|
|
14
14
|
import { openBatch } from "./batch.mjs";
|
|
15
15
|
import { orchestrate } from "./orchestrate.mjs";
|
|
@@ -286,6 +286,20 @@ describe("remote config and the Actions constraint (#4254)", () => {
|
|
|
286
286
|
assert.equal(resolveCredential("FD_T", { env: {}, keychain: () => { throw new Error("locked"); } }), null);
|
|
287
287
|
assert.equal(resolveCredential("bad name", { env: { "bad name": "x" } }), null);
|
|
288
288
|
});
|
|
289
|
+
|
|
290
|
+
it("5159 memoKeychain: one lookup per name per TTL, misses and failures cached as null", () => {
|
|
291
|
+
let t = 0;
|
|
292
|
+
const calls = [];
|
|
293
|
+
const kc = memoKeychain({ keychain: (n) => { calls.push(n); if (n === "BAD") throw new Error("locked"); return n === "HIT" ? "v" : null; }, ttlMs: 1000, now: () => t });
|
|
294
|
+
assert.equal(kc("HIT"), "v"); assert.equal(kc("HIT"), "v");
|
|
295
|
+
assert.equal(kc("MISS"), null); assert.equal(kc("MISS"), null);
|
|
296
|
+
assert.equal(kc("BAD"), null); assert.equal(kc("BAD"), null);
|
|
297
|
+
assert.deepEqual(calls, ["HIT", "MISS", "BAD"]);
|
|
298
|
+
t = 1000;
|
|
299
|
+
assert.equal(kc("MISS"), null);
|
|
300
|
+
assert.deepEqual(calls, ["HIT", "MISS", "BAD", "MISS"], "re-read after the TTL");
|
|
301
|
+
assert.equal(resolveCredential("MISS", { env: { MISS: "e" }, keychain: kc }), "e", "the env still wins");
|
|
302
|
+
});
|
|
289
303
|
});
|
|
290
304
|
|
|
291
305
|
describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
|
package/bin/pool/agent.mjs
CHANGED
|
@@ -245,6 +245,8 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
|
|
|
245
245
|
const c = credCache[kind];
|
|
246
246
|
if (!c || now() - c.at > CRED_TTL[kind]) {
|
|
247
247
|
const x = await call(cfg.router_url, "GET", `/agent/creds/${kind}`, { token: cfg.token, timeoutMs: 20000 });
|
|
248
|
+
// #5165: every router login at its usage ceiling -> pass the typed 429 (reason, reset) through, uncached, so the lane defers.
|
|
249
|
+
if (x.status === 429 && x.data && typeof x.data === "object") return { status: 429, body: x.data, retryAfter: x.retryAfter };
|
|
248
250
|
if (!x.ok || !x.data?.token) return { status: 502, body: { error: `router credentials (${kind}): HTTP ${x.status}` } };
|
|
249
251
|
credCache[kind] = { token: x.data.token, at: now() };
|
|
250
252
|
}
|
|
@@ -397,7 +399,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
|
|
|
397
399
|
const url = new URL(req.url, "http://agent");
|
|
398
400
|
try {
|
|
399
401
|
const cred = req.method === "GET" && /^\/sandboxes\/([^/]+)\/creds\/([a-z]+)$/.exec(url.pathname);
|
|
400
|
-
if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body); }
|
|
402
|
+
if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body, r.retryAfter ? { "retry-after": String(r.retryAfter) } : {}); }
|
|
401
403
|
const relay = (req.method === "POST" && /^\/sandboxes\/([^/]+)\/relay$/.exec(url.pathname)) || (req.method === "GET" && /^\/sandboxes\/([^/]+)\/relay\/([^/]+)$/.exec(url.pathname));
|
|
402
404
|
if (relay) { const r = await sandboxRelay(decodeURIComponent(relay[1]), relay[2] ? decodeURIComponent(relay[2]) : null, req); return send(res, r.status, r.body ?? {}); }
|
|
403
405
|
if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized", hop: "agent", reason: "agent auth: router token rejected by agent" });
|
package/bin/pool/router.mjs
CHANGED
|
@@ -93,19 +93,28 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
93
93
|
}
|
|
94
94
|
return gh.token;
|
|
95
95
|
}
|
|
96
|
-
// Claude access token for sandboxes: the
|
|
97
|
-
// short-lived access token is shared; refresh tokens stay here, so this host
|
|
96
|
+
// Claude access token for sandboxes (#5165): among the logins with 30+ minutes left, the one with the lowest tokenbaton
|
|
97
|
+
// usage (validity breaks ties). Only the short-lived access token is shared; refresh tokens stay here, so this host
|
|
98
|
+
// keeps refreshing its own sessions.
|
|
98
99
|
function claudeToken() {
|
|
99
100
|
const dirs = cfg.claude_config_dirs ?? readdirSync(homedir()).filter((d) => /^\.claude(-\d+)?$/.test(d)).map((d) => join(homedir(), d));
|
|
100
|
-
|
|
101
|
+
const logins = [];
|
|
101
102
|
for (const d of dirs) {
|
|
102
103
|
try {
|
|
103
104
|
const o = JSON.parse(readFileSync(join(d, ".credentials.json"), "utf8")).claudeAiOauth;
|
|
104
|
-
|
|
105
|
+
const n = /\.claude-(\d+)$/.exec(d)?.[1];
|
|
106
|
+
if (o?.accessToken) logins.push({ n: n ? Number(n) : null, accessToken: o.accessToken, expiresAt: o.expiresAt });
|
|
105
107
|
} catch { /* no login in this dir */ }
|
|
106
108
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
const src = usageSource(cfg);
|
|
110
|
+
const usage = readUsage(src, logins.map((l) => l.n).filter((n) => n !== null));
|
|
111
|
+
const pick = pickClaudeLogin({ logins, usage, ceilingPct: cfg.usage_ceiling_pct ?? 95, now: now() });
|
|
112
|
+
if (pick.status === 503) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
|
|
113
|
+
if (pick.status === 429) { log.warn?.(`claude login: every account at or above the usage ceiling; ${pick.body.reason}`); return pick; }
|
|
114
|
+
if (pick.fallback) log.warn?.(`claude login: no usage source (${src}), falling back to longest validity`);
|
|
115
|
+
const { login, pct } = pick;
|
|
116
|
+
log.info?.(`claude login selected: account=${login.n ?? "default"} usage=${pct ?? "unknown"}% expires_at=${new Date(login.expiresAt).toISOString()}`);
|
|
117
|
+
return { status: 200, body: { token: login.accessToken, expires_at: new Date(login.expiresAt).toISOString() } };
|
|
109
118
|
}
|
|
110
119
|
const agentCall = (a, method, path, body, timeoutMs) => call(a.url, method, path, { body, token: a.token, timeoutMs, fetchFn });
|
|
111
120
|
|
|
@@ -285,7 +294,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
285
294
|
[route("GET", "/agents"), fd(async () => ({ status: 200, body: { agents: await capacities(), sandboxes: sbx() } }))],
|
|
286
295
|
[route("POST", "/codes"), fd(async () => ({ status: 200, body: issueCode() }))],
|
|
287
296
|
[route("GET", "/agent/creds/github"), { fn: async () => ({ status: 200, body: { token: await githubToken() } }), auth: "agent" }],
|
|
288
|
-
[route("GET", "/agent/creds/claude"), { fn: async () =>
|
|
297
|
+
[route("GET", "/agent/creds/claude"), { fn: async () => claudeToken(), auth: "agent" }],
|
|
289
298
|
[route("POST", "/agent/relay/:id"), { fn: async (p, body, _u, req) => relayPost(req, p, body), auth: "agent" }], // #4778
|
|
290
299
|
[route("GET", "/agent/relay/:id/:rid"), { fn: async (p, _b, _u, req) => relayPoll(req, p), auth: "agent" }],
|
|
291
300
|
[route("GET", "/agent/bundle"), { fn: async () => ({ bundle: true }), auth: "agent" }],
|
|
@@ -355,6 +364,62 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
355
364
|
return { handle, launch, claim, tick, enroll, issueCode, capacities, status, store };
|
|
356
365
|
}
|
|
357
366
|
|
|
367
|
+
/** tokenbaton's usage dir (#5165): router.json `usage_source`, else `$TOKENBATON_HOME`, else `~/.tokenbaton`. */
|
|
368
|
+
export function usageSource(cfg, env = process.env) {
|
|
369
|
+
return cfg.usage_source ?? env.TOKENBATON_HOME ?? join(homedir(), ".tokenbaton");
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** One tokenbaton `live.<n>` line (`<5h %> <5h reset s> <weekly %> <weekly reset s> [updated s]`), or null; the format of bin/lean/rotator.mjs parseLive. */
|
|
373
|
+
export function parseLiveUsage(text) {
|
|
374
|
+
const f = String(text ?? "").trim().split(/\s+/).map(Number);
|
|
375
|
+
if (f.length < 4 || !f.slice(0, 4).every(Number.isFinite)) return null;
|
|
376
|
+
return { fiveHourPct: f[0], fiveHourResetMs: f[1] * 1000, weeklyPct: f[2], weeklyResetMs: f[3] * 1000 };
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Map account n -> parsed `live.<n>` for the given accounts; unreadable or malformed files are left out. */
|
|
380
|
+
export function readUsage(dir, ns, readFile = (p) => readFileSync(p, "utf8")) {
|
|
381
|
+
const out = new Map();
|
|
382
|
+
for (const n of ns) { try { const u = parseLiveUsage(readFile(join(dir, `live.${n}`))); if (u) out.set(n, u); } catch { /* no usage for n */ } }
|
|
383
|
+
return out;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Pick the Claude login to hand out (#5165). Candidates have 30+ minutes of validity. A login with known usage is
|
|
388
|
+
* excluded at or above `ceilingPct` 5-hour use or 100% weekly use (a passed reset counts as 0, like the local rotator);
|
|
389
|
+
* the rest rank by 5-hour use, then validity. Logins with unknown usage follow. No usage at all: longest validity,
|
|
390
|
+
* `fallback: true`. Every candidate excluded: a typed 429 carrying the earliest reset.
|
|
391
|
+
* @returns {{status: 200, login, pct, fallback} | {status: 429, retryAfter, body} | {status: 503}}
|
|
392
|
+
*/
|
|
393
|
+
export function pickClaudeLogin({ logins, usage, ceilingPct = 95, now }) {
|
|
394
|
+
const valid = logins.filter((l) => l.accessToken && l.expiresAt > now + 30 * 60000);
|
|
395
|
+
if (!valid.length) return { status: 503 };
|
|
396
|
+
const used = (pct, resetMs) => (resetMs <= now ? 0 : pct);
|
|
397
|
+
const byValidity = (a, b) => b.expiresAt - a.expiresAt;
|
|
398
|
+
const known = [], unknown = [], blocked = [];
|
|
399
|
+
for (const l of valid) {
|
|
400
|
+
const u = l.n !== null && l.n !== undefined ? usage.get(l.n) : undefined;
|
|
401
|
+
if (!u) { unknown.push(l); continue; }
|
|
402
|
+
const five = used(u.fiveHourPct, u.fiveHourResetMs), week = used(u.weeklyPct, u.weeklyResetMs);
|
|
403
|
+
if (five >= ceilingPct || week >= 100) {
|
|
404
|
+
const r = [];
|
|
405
|
+
if (five >= ceilingPct) r.push(u.fiveHourResetMs);
|
|
406
|
+
if (week >= 100) r.push(u.weeklyResetMs);
|
|
407
|
+
blocked.push(Math.max(...r));
|
|
408
|
+
} else known.push({ l, pct: five });
|
|
409
|
+
}
|
|
410
|
+
if (known.length) {
|
|
411
|
+
known.sort((a, b) => a.pct - b.pct || byValidity(a.l, b.l));
|
|
412
|
+
return { status: 200, login: known[0].l, pct: known[0].pct, fallback: false };
|
|
413
|
+
}
|
|
414
|
+
if (unknown.length) {
|
|
415
|
+
unknown.sort(byValidity);
|
|
416
|
+
return { status: 200, login: unknown[0], pct: null, fallback: !blocked.length };
|
|
417
|
+
}
|
|
418
|
+
const resetMs = Math.min(...blocked), at = new Date(resetMs).toISOString();
|
|
419
|
+
return { status: 429, retryAfter: Math.max(1, Math.ceil((resetMs - now) / 1000)),
|
|
420
|
+
body: { error: "usage limit", code: "rate_limited", reason: `usage limit hit; resets at ${at}`, reset_at: at } };
|
|
421
|
+
}
|
|
422
|
+
|
|
358
423
|
export function startRouter(cfg, deps) {
|
|
359
424
|
const router = createRouter(cfg, deps);
|
|
360
425
|
const server = createServer(router.handle);
|
package/bin/review-staging.mjs
CHANGED
|
@@ -385,6 +385,8 @@ async function finalize(a, io) {
|
|
|
385
385
|
// Incomplete-panel guard + 6.75 delivery
|
|
386
386
|
const d = await bash(io, "review-staging-post.sh", ["delivery", ...common], env);
|
|
387
387
|
relay(io, d);
|
|
388
|
+
// Exit 2 = the comments could not be listed on any attempt (forge#5166): rerun, never re-dispatch reviewers that may have posted.
|
|
389
|
+
if (d.code === 2) throw new Stop(1, ["REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery could not be verified (GitHub comment listing failed); rerun --finalize, do not re-dispatch reviewers", BLOCK_DEPLOY]);
|
|
388
390
|
if (d.code !== 0) throw new Stop(1, [`REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery incomplete (exit ${d.code}); re-dispatch each missing reviewer once with the note "structured findings block required", else the panel is degraded`, BLOCK_DEPLOY]);
|
|
389
391
|
|
|
390
392
|
// 6.5 verdict handling (override, posture, test-gate markers)
|
package/bin/runner.mjs
CHANGED
|
@@ -1352,6 +1352,27 @@ export function detectUsageLimit(stdout, output = "") {
|
|
|
1352
1352
|
return hasUsageLimitResetClause(output || "");
|
|
1353
1353
|
}
|
|
1354
1354
|
|
|
1355
|
+
// forge#5168: the claude CLI's own login/authentication failure texts. Literal
|
|
1356
|
+
// alternatives only (linear on untrusted input).
|
|
1357
|
+
const AUTH_FAILURE_RE = /Failed to authenticate|authentication_error|OAuth token (?:has )?expired|reached its time limit and cannot be renewed|Invalid API key|Please run \/login/i;
|
|
1358
|
+
|
|
1359
|
+
/**
|
|
1360
|
+
* Detects whether a failed CLI run was an authentication failure of the claude
|
|
1361
|
+
* login (forge#5168): an expired or invalid login is an account condition, not
|
|
1362
|
+
* a transient engine error. With an envelope on stdout, it must be
|
|
1363
|
+
* `is_error: true` with a `result` naming the failure; without one, the plain
|
|
1364
|
+
* combined output must. `stdout` is parsed ALONE (forge#2422); never throws.
|
|
1365
|
+
*
|
|
1366
|
+
* @param {string} stdout - raw stdout of the CLI run
|
|
1367
|
+
* @param {string} [output] - combined stdout+stderr, for the plain-text fallback
|
|
1368
|
+
* @returns {boolean}
|
|
1369
|
+
*/
|
|
1370
|
+
export function detectAuthFailure(stdout, output = "") {
|
|
1371
|
+
const env = parseCliEnvelope(stdout);
|
|
1372
|
+
if (env) return env.is_error === true && typeof env.result === "string" && AUTH_FAILURE_RE.test(env.result);
|
|
1373
|
+
return typeof output === "string" && AUTH_FAILURE_RE.test(output);
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1355
1376
|
/**
|
|
1356
1377
|
* Best-effort extraction of a Claude CLI session-limit reset time from
|
|
1357
1378
|
* captured stdout/stderr (forge#2241). The CLI's own session-limit message
|
|
@@ -1844,6 +1865,9 @@ export function runCliBackend({
|
|
|
1844
1865
|
// default when the reset time itself is missing or unparseable. The
|
|
1845
1866
|
// engine, not the runner, owns that default — no epoch is fabricated here.
|
|
1846
1867
|
if (isUsageLimit) err.usageLimit = true;
|
|
1868
|
+
// forge#5168: a failed login (never when the usage limit already matched,
|
|
1869
|
+
// which keeps precedence) lets the lane defer as `backend.auth`.
|
|
1870
|
+
else if (detectAuthFailure(stdout, output)) err.authFailure = true;
|
|
1847
1871
|
if (resetAt) {
|
|
1848
1872
|
err.resetAt = resetAt;
|
|
1849
1873
|
// forge#2524: also attach a machine-usable epoch-ms timestamp so
|
|
@@ -401,14 +401,14 @@ node "$FD_ROOT/bin/forgedock.mjs" review-staging "$PR_NUMBER" --finalize --test-
|
|
|
401
401
|
```
|
|
402
402
|
|
|
403
403
|
The call runs these steps in order:
|
|
404
|
-
1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying.
|
|
404
|
+
1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying. The PR comments are listed once per attempt. A missing reviewer or a GitHub API error is re-listed with backoff (5 s, 15 s, 45 s) before any verdict, and each domain is reported with the attempt it was found on.
|
|
405
405
|
2. The Phase 6.5 verdict handling.
|
|
406
406
|
3. `scripts/review-finalize.sh`:
|
|
407
407
|
- `extract`: trusted comments only, dedup, and the `commands/review-pr.md` §6B.4 provenance and concreteness gates. **CRITICAL/HIGH are never routed as pre-existing**, whatever their confidence.
|
|
408
408
|
- `classify --dry-run`, then `classify` for real. This passes `--scope-files` (the 0B.1 scoped set when `SCOPE_MODE=scoped`) and `--prior-prs` (the bundle members), and posts `FORGE:NOTE_DISPOSITION`.
|
|
409
409
|
- `file-prepare`.
|
|
410
410
|
|
|
411
|
-
Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A.
|
|
411
|
+
Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A. `REVIEW DELIVERY UNVERIFIED` means GitHub could not list the comments on the last attempt. It stops the run, but nothing is labelled or posted. Do not re-dispatch anyone: rerun 7A once GitHub answers.
|
|
412
412
|
|
|
413
413
|
Exit 0 prints the following:
|
|
414
414
|
- `TEST_GATE:`, `SCOPE_MODE=`.
|
package/docs/CONFIG.md
CHANGED
|
@@ -438,6 +438,7 @@ orchestration:
|
|
|
438
438
|
| `max_concurrent` | integer | No | Max concurrent top-level `/work-on` dispatches. It does not bound cascade growth; under `policy: all` it raises drain and spawn throughput together. Default: 12. |
|
|
439
439
|
| `breaker.errors` / `breaker.window_min` | integer | No | `forgedock orchestrate` circuit breaker (#4121): after `errors` engine-errors within `window_min` minutes the batch stops admitting lanes, lets running lanes finish, and keeps the rest pending for `--resume`. `errors: 0` turns it off. Default: 3 in 30 |
|
|
440
440
|
| `max_deferrals` | integer | No | `forgedock orchestrate`: requeues of one issue after a non-host `deferred` (usage limit, merge/push transient, CLI idle, lease held, gh unavailable) before its row is final. `host.busy` and `sandbox.capacity` (sandbox pool full) deferrals are not capped. Default: 3 |
|
|
441
|
+
| `transient_retries` | integer | No | `forgedock orchestrate` (#5161): backoff retries of an issue whose lane ended `engine-error` with a transport or gateway failure in its detail (curl `returned error: 5xx`, `HTTP/1.1 5xx`, `Bad Gateway`, `Service Unavailable`, `overloaded`, connection reset/refused/timeout). Delays 30s, 2m, 8m, then 15m each; dependents stay pending meanwhile. Not counted against the single engine-error relaunch; once spent, the row ends `engine-error`. Every such error still counts toward the breaker. `0` turns transient retries off. Default: 4 |
|
|
441
442
|
| `blocked_label` | string | No | `forgedock orchestrate` (#4773): an open issue carrying this label is admitted to the batch but not dispatched; removing the label releases it on the next scheduler pass. A `Blocked by #N` / `Depends on #N` line in the body of a trusted author (OWNER, MEMBER, COLLABORATOR or a bot) is a DAG edge to #N, and `--add` prints the edges it created. Empty turns the label off. Default: `blocked` |
|
|
442
443
|
| `auto_add_children` | string | No | `forgedock orchestrate` (#4965): which children a running batch adds by itself. `off` (no auto-adds; a decomposed lane's sub-issues stay out too), `decomposed` (a decomposed lane's sub-issues join, as before), `epic` (also the open children of every batch issue and of each `--epic N`, found by a `**Parent**: #P` line, the `parent` field or GitHub sub-issues, including children filed after the batch started) or `all`. Only children whose author is trusted (an owner, member or collaborator, or a bot, the `Blocked by` rule) are auto-added; any other is skipped with a logged row. Children labelled `needs-human` and children another live batch of the repo still holds are not added. Each pass that adds posts one batch decision and one `added` row listing the children and edges. `forgedock orchestrate --auto-children <mode>` overrides it per batch and `--resume` keeps it. Default: `decomposed` |
|
|
443
444
|
| `auto_add_children_poll_s` | integer | No | Seconds between child scans for `epic`/`all` (one `gh issue list` per scan). Default: `120` |
|
|
@@ -470,6 +471,43 @@ orchestration:
|
|
|
470
471
|
|
|
471
472
|
---
|
|
472
473
|
|
|
474
|
+
## `orchestrate.health` and `notify` (OPTIONAL)
|
|
475
|
+
|
|
476
|
+
Early batch health (#5167). `forgedock orchestrate` checks each batch at T+15 and T+60 minutes after it was created (`plan.json` `created`), and once more when the batch ends. When the batch starts, it prints the schedule: `health checks: T+15 14:05Z, T+60 14:50Z, and at batch end; forgedock status --batch <id>`. The first unhealthy check raises one `BATCH UNHEALTHY <id>` alert, which lists the top causes and the exact recovery commands (`--requeue`, a relaunch, `--resume`, `--cancel`). `health.json` in the batch dir records the checkpoints already done and the alert, so a daemon restart arms only the remaining checks and a batch never alerts twice. Checks and alerts never fail, delay or change the batch.
|
|
477
|
+
|
|
478
|
+
Causes: `root-failed` (a root issue, meaning one with no dependency inside the batch, ended non-merged); `terminal-share` (the share of rows ended `skipped`/`held`/`engine-error`/`needs-human` or another non-merged outcome except `invalid`/`decomposed` is at or above the threshold, with nothing merged); `stalled` (0 lanes running while issues are pending, operator-held issues excepted); `refused` (the launch was refused).
|
|
479
|
+
|
|
480
|
+
```yaml
|
|
481
|
+
orchestrate:
|
|
482
|
+
health:
|
|
483
|
+
root_failed: true # any root ended non-merged is unhealthy (default true)
|
|
484
|
+
terminal_share: 0.5 # 0..1, terminal-non-merged share with 0 merged (default 0.5)
|
|
485
|
+
stalled: true # 0 running while pending > 0 (default true)
|
|
486
|
+
issue: 123 # comment target; default: the batch's --epic issue(s); a milestone gets no comment
|
|
487
|
+
checks_min: [15, 60] # minutes after launch (default [15, 60])
|
|
488
|
+
notify:
|
|
489
|
+
command: "curl -fsS -H 'content-type: application/json' -d @- https://hooks.example/forge"
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
| Key | Default | Meaning |
|
|
493
|
+
|---|---|---|
|
|
494
|
+
| `orchestrate.health.root_failed` | `true` | Raise `root-failed`. |
|
|
495
|
+
| `orchestrate.health.terminal_share` | `0.5` | Threshold for `terminal-share`. |
|
|
496
|
+
| `orchestrate.health.stalled` | `true` | Raise `stalled`. |
|
|
497
|
+
| `orchestrate.health.issue` | none | Issue to comment on. When unset, the alert comments on each `--epic` issue. |
|
|
498
|
+
| `orchestrate.health.checks_min` | `[15, 60]` | Timed checkpoints, in minutes after launch. |
|
|
499
|
+
| `notify.command` | none | Shell command run once per alert. It gets the alert JSON (`{alert, batch, repo, text, causes, recovery}`) on stdin and `FORGEDOCK_ALERT=BATCH_UNHEALTHY`, `FORGEDOCK_ALERT_BATCH` and `FORGEDOCK_ALERT_TEXT` in its env. It is killed after 30 s. A failure only prints a `health: notify.command failed` line. |
|
|
500
|
+
|
|
501
|
+
The keys are flattened one level by the parser, the same as `orchestrate.drift.*`. Health keys alone do not turn drift on.
|
|
502
|
+
|
|
503
|
+
Every channel is best effort: the runner line, the GitHub comment (marker `FORGE:BATCH-HEALTH`), `notify.command`, and the `forgedock status` line. `forgedock status` lists the unhealthy batches that are unresolved (no report yet, or alerted within the last day) and the launch refusals from the last day. `forgedock status --batch <id>` shows that batch's health (healthy, or unhealthy with causes and recovery) and its pending checkpoints.
|
|
504
|
+
|
|
505
|
+
A refused milestone start (#4659, `parallel milestone start refused: …`) still exits 1 with the error on stderr and creates no batch dir. It is also recorded in `<batch root>/refusals.json` (the newest 20 are kept) and sends the same alert, with the merge-the-dependency or `--force "<reason>"` relaunch.
|
|
506
|
+
|
|
507
|
+
**Commands that use this section**: `forgedock orchestrate`, `forgedock status`
|
|
508
|
+
|
|
509
|
+
---
|
|
510
|
+
|
|
473
511
|
## `decisions` (OPTIONAL)
|
|
474
512
|
|
|
475
513
|
Decision engine (#4766, see [DECISIONS.md](DECISIONS.md)). The issue-watch stage (#4793) calls `settle` for the open `needs-human` issues a watch selects: it runs when `mode: live` (a repo setting: every watch of the repo runs the stage), or when the watch sets `sweep_needs_human: true`; `policy` and `human_only` are also read by prodwatch (`loadPolicy`). The parser has one nesting level and no arrays, so lists are comma-separated.
|
|
@@ -1185,7 +1223,7 @@ decomposition:
|
|
|
1185
1223
|
|
|
1186
1224
|
**With `close_parent: false`** (default): unchanged. The parent stays open and is closed by C3 when the last child closes.
|
|
1187
1225
|
|
|
1188
|
-
The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `not
|
|
1226
|
+
The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `completed` (#5152; scope moved to the sub-issues, not dropped), keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
|
|
1189
1227
|
|
|
1190
1228
|
**Limitation**: `--retry` and engine resume refuse closed issues, so a closed decomposed parent cannot be re-run. It has nothing left to run; work on its open children instead.
|
|
1191
1229
|
|
package/docs/OUTCOME-CODES.md
CHANGED
|
@@ -44,6 +44,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
|
|
|
44
44
|
| `gh.unavailable` | `run-issue --repo` could not verify the cwd repo because `gh repo view` kept failing after 3 attempts with backoff (typically a timeout under host load); the core or full run is `deferred` (exit 75, retry in 60s) before any lease, nothing posted | engine CLI | `bin/engine-cli.mjs` |
|
|
45
45
|
| `gh.auth` | A `gh` call in a core run answered HTTP 401 (expired or revoked bot token) and one run of `scripts/refresh-bot-token.sh` plus a retry did not fix it; the run is `deferred` (retry in 120s), never `engine-error`. See `docs/CONFIG.md` "Bot token refresh" | lean | `bin/lean/retry.mjs` |
|
|
46
46
|
| `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
|
|
47
|
+
| `backend.auth` | The claude CLI login failed ("Failed to authenticate", expired or invalid login); the lane is `deferred` (retry in 900s) under a per-account hold like a usage limit, never `engine-error`, never relaunched or counted by the breaker (#5168) | lean | `bin/runner.mjs` `err.authFailure` |
|
|
47
48
|
| `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
|
|
48
49
|
| `engine.dco_rewrite` | Core work-on (forge#4997): the DCO sign-off rewrite (`git filter-branch`) of the lane's unpushed commits failed, or the lane's uncommitted changes could not be restored after it (kept on `refs/forgedock/dco-stash`). A mechanical engine step, not a human block: the lane ends `engine-error` (never `needs-human`), which `forgedock orchestrate` relaunches like any engine-error | lean | `bin/lean/signoff.mjs` `ensureSignedOff` |
|
|
49
50
|
| `engine.validate_result_lost` | Core work-on (forge#4708): a validate result recorded before a merge-time CI fix round is missing from the fix session's result file and could not be carried forward; an engine bookkeeping error, not an unimplemented requirement | lean | `bin/lean/lean.mjs` `mergeFixRound` |
|
|
@@ -70,7 +71,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
|
|
|
70
71
|
| `lane.branch_missing` | Core work-on: the lane branch (e.g. `milestone/<slug>`) does not exist on origin; fail closed | lean | `git ls-remote --heads origin` |
|
|
71
72
|
| `lane.unresolved` | Core work-on: the lane could not be worked out (milestone or default branch unreadable, empty milestone slug, bad `feature_pattern`); fail closed | lean | Failed `gh` read or config |
|
|
72
73
|
| `invalid` | Issue is invalid or in a divergent closed state | investigate, engine | `INVESTIGATION:INVALID` marker, or live snapshot |
|
|
73
|
-
| `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `
|
|
74
|
+
| `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `completed` (label kept; #5152: its scope moved, it was not dropped) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
|
|
74
75
|
| `builder.blocked` | Core work-on: the builder reported an external action agents cannot take, or a product decision too critical for agents: `block.kind: "human"` with `why` one of `credentials`, `spend`, `promotion`, `customer`, `product-decision`. An upstream issue, a needed split, or prod data that monitoring access can read never count | lean | `result.json` status `blocked` + typed `block` |
|
|
75
76
|
| `builder.block_untyped` | Core work-on: the builder stayed `blocked` without a valid typed reason (missing `block`, a non-human `why` such as prod data, or a dependency that is not open) after one corrective resume | lean | `result.json` status `blocked`, twice |
|
|
76
77
|
| `builder.no_evidence` | Core work-on: the builder reported invalid/already-fixed without file:line evidence, so the issue was not closed | lean | `result.json` with empty `evidence` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "forgedock",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.51.0",
|
|
4
4
|
"description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"forgedock": "bin/forgedock.mjs"
|
package/pool/README.md
CHANGED
|
@@ -10,4 +10,6 @@ ForgeDock's built-in remote compute pool (imported from computeshare; AGPL-3.0-o
|
|
|
10
10
|
|
|
11
11
|
The module boundary is the ForgeDock HTTP sandbox protocol: the router speaks it to the dispatcher and relays to agents over HTTP. Sandboxes receive only short-lived GitHub and Claude access tokens; refresh tokens stay on the router host.
|
|
12
12
|
|
|
13
|
+
Claude login choice (#5165): among the router host's logins with 30+ minutes left, the router hands out the one with the lowest 5-hour use in tokenbaton's `live.<n>` (account `n` = `~/.claude-<n>`), read from router.json `usage_source` (default `$TOKENBATON_HOME`, else `~/.tokenbaton`); validity only breaks ties. An account at or above `usage_ceiling_pct` (default 95) or at 100% weekly use is skipped; when every login is, `/agent/creds/claude` answers a typed 429 (`code: rate_limited`, `reset_at` = the earliest reset, `retry-after`), which the agent passes through and `cs-cred` prints as `usage limit hit; resets at <ISO>` so the lane defers. Without readable usage it falls back to the longest validity and logs a warning. The selected account index and usage are logged, never the token.
|
|
14
|
+
|
|
13
15
|
The `forgedock pool` CLI wiring is tracked separately (#4544).
|
package/pool/image/cs-cred
CHANGED
|
@@ -7,6 +7,13 @@ kind="${1:?kind}"; ttl=240; [ "$kind" = claude ] && ttl=600
|
|
|
7
7
|
cache="/tmp/.cs-cred-$kind"
|
|
8
8
|
if [ -s "$cache" ] && [ $(( $(date +%s) - $(stat -c %Y "$cache") )) -lt $ttl ]; then cat "$cache"; exit 0; fi
|
|
9
9
|
[ -n "${CS_CRED_BASE:-}" ] || { echo "no credential relay (CS_CRED_BASE)" >&2; exit 1; }
|
|
10
|
-
|
|
10
|
+
body=$(mktemp); trap 'rm -f "$body"' EXIT
|
|
11
|
+
code=$(curl -sS --max-time 30 -o "$body" -w '%{http_code}' -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind") || code=000
|
|
12
|
+
if [ "$code" = 429 ]; then
|
|
13
|
+
# #5165: every router login is at its usage ceiling; this line is what the lane's log tail defers on.
|
|
14
|
+
why=$(jq -r '.reason // empty' "$body" 2>/dev/null || true)
|
|
15
|
+
echo "${why:-usage limit hit; resets at unknown}" >&2; exit 1
|
|
16
|
+
fi
|
|
17
|
+
tok=""; [ "$code" = 200 ] && tok=$(jq -r .token "$body" 2>/dev/null || true)
|
|
11
18
|
[ -n "$tok" ] && [ "$tok" != null ] || { echo "credential fetch failed ($kind)" >&2; exit 1; }
|
|
12
19
|
umask 077; printf '%s' "$tok" > "$cache"; printf '%s' "$tok"
|
|
@@ -10,12 +10,15 @@
|
|
|
10
10
|
# delivery counts trusted, current-head FORGE:REVIEW-AGENT comments (findings block required) for every domain listed in
|
|
11
11
|
# DIR/<N>_launched_review_agents. A shortfall labels the PR review-degraded + needs-human and posts
|
|
12
12
|
# FORGE:GATE_FAILURE:TYPE=review-panel-integrity; an empty/unreadable list or a missing domain is a delivery failure.
|
|
13
|
-
#
|
|
13
|
+
# The comments are listed once per attempt; a missing domain or an API error re-lists after each
|
|
14
|
+
# FORGE_REVIEW_DELIVERY_RETRY_DELAYS wait (seconds, default "5 15 45"). An API error on the last attempt is
|
|
15
|
+
# REVIEW DELIVERY UNVERIFIED (exit 2, nothing labelled or posted), never "missing" (forge#5166).
|
|
16
|
+
# Exit 0 = every launched reviewer delivered, 1 = not, 2 = could not be verified.
|
|
14
17
|
# test-gate reads the saved /test-gate output FILE (the last FORGE:TEST_GATE:RESULT= marker; an absent marker stays empty and
|
|
15
18
|
# fails closed as BLOCK, never SKIP), applies the trusted override, the posture and the FORGE:GATE_PASS /
|
|
16
19
|
# FORGE:GATE_FAILURE test-gate comments, and persists DIR/<N>_test_gate_verdict and _reason.
|
|
17
20
|
# Exit 0 = continue, 1 = RESULT: BLOCK DEPLOY.
|
|
18
|
-
# Exit: 2 on a usage error.
|
|
21
|
+
# Exit: 2 on a usage error (and delivery UNVERIFIED).
|
|
19
22
|
#
|
|
20
23
|
# The `# BEGIN <name>` ... `# END <name>` blocks below are the spec's former fences, copied without their forge_resolve locator
|
|
21
24
|
# stubs (this script resolves helpers once, below). Helpers (trusted-comments.sh) resolve through forge_resolve from trusted
|
|
@@ -59,24 +62,77 @@ type forge_resolve >/dev/null 2>&1 || { setopt nonomatch 2>/dev/null || true; _S
|
|
|
59
62
|
|
|
60
63
|
if [ "$CMD" = delivery ]; then
|
|
61
64
|
echo "=== Incomplete-panel guard and Phase 6.75: verify agent delivery ==="
|
|
65
|
+
# BEGIN delivery-fetch
|
|
66
|
+
# The PR's comments are listed ONCE per attempt into a cache file; the panel guard and the per-domain check both read it (forge#5166).
|
|
67
|
+
# An attempt is an API error unless gh exits 0 and every page is a JSON array: an error is never evidence that a comment is absent.
|
|
68
|
+
DELIVERY_CACHE="$_SD/${PR_NUMBER}_delivery_comments.json"
|
|
69
|
+
delivery_fetch() {
|
|
70
|
+
rm -f "$DELIVERY_CACHE" "$DELIVERY_CACHE.tmp"
|
|
71
|
+
gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" > "$DELIVERY_CACHE.tmp" 2>/dev/null || { rm -f "$DELIVERY_CACHE.tmp"; return 1; }
|
|
72
|
+
jq -se 'length > 0 and all(.[]; type == "array")' "$DELIVERY_CACHE.tmp" >/dev/null 2>&1 || { rm -f "$DELIVERY_CACHE.tmp"; return 1; }
|
|
73
|
+
mv -f "$DELIVERY_CACHE.tmp" "$DELIVERY_CACHE"
|
|
74
|
+
}
|
|
75
|
+
# Unique trusted current-head reviewer domains in the cached listing (findings block not required), one per line.
|
|
76
|
+
delivery_domains() {
|
|
77
|
+
bash "$TRUSTED_SCRIPT" bodies "^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" < "$DELIVERY_CACHE" 2>/dev/null \
|
|
78
|
+
| jq -r 'scan("^<!-- FORGE:REVIEW-AGENT:([a-z0-9-]+) -->") | .[0]' 2>/dev/null | sort -u
|
|
79
|
+
}
|
|
80
|
+
# Trusted current-head comments for one domain that carry a findings block, in the cached listing.
|
|
81
|
+
delivery_count() {
|
|
82
|
+
local AGENT_DOMAIN="$1" n
|
|
83
|
+
n=$(bash "$TRUSTED_SCRIPT" count "^(?=[\\s\\S]*<!-- REVIEW-FINDINGS-START -->[\\s\\S]*<!-- REVIEW-FINDINGS-END -->)<!-- FORGE:REVIEW-AGENT:${AGENT_DOMAIN} -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" < "$DELIVERY_CACHE" 2>/dev/null)
|
|
84
|
+
case "$n" in ''|*[!0-9]*) n=0 ;; esac # fail closed: counts as missing
|
|
85
|
+
printf '%s\n' "$n"
|
|
86
|
+
}
|
|
87
|
+
# END delivery-fetch
|
|
62
88
|
# BEGIN panel-guard
|
|
63
89
|
# Each Bash call is a fresh shell: re-resolve the full head SHA if it is not set, and refuse when it is not a commit id
|
|
64
90
|
# (an empty SHA would make the Reviewed-SHA match accept any head).
|
|
65
91
|
[ -n "${REVIEW_SHA_FULL:-}" ] || REVIEW_SHA_FULL=$(gh pr view "$PR_NUMBER" -R "$GH_REPO" --json headRefOid --jq '.headRefOid' 2>/dev/null || echo "")
|
|
66
92
|
case "$REVIEW_SHA_FULL" in *[!0-9a-f]*|"") REVIEW_SHA_FULL="" ;; *) [ "${#REVIEW_SHA_FULL}" -eq 40 ] || [ "${#REVIEW_SHA_FULL}" -eq 64 ] || REVIEW_SHA_FULL="" ;; esac
|
|
67
93
|
TRUSTED_SCRIPT=$(forge_resolve trusted-comments.sh)
|
|
68
|
-
ACTUAL_AGENT_COUNT=0
|
|
69
|
-
if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ]; then
|
|
70
|
-
# Only TRUSTED, current-head reviewer comments count, over every page of comments. Unique domains, so a duplicate does not inflate the count.
|
|
71
|
-
ACTUAL_AGENT_COUNT=$(gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
|
|
72
|
-
| bash "$TRUSTED_SCRIPT" bodies "^<!-- FORGE:REVIEW-AGENT:[a-z0-9-]+ -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" 2>/dev/null \
|
|
73
|
-
| jq -r 'scan("^<!-- FORGE:REVIEW-AGENT:([a-z0-9-]+) -->") | .[0]' 2>/dev/null | sort -u | grep -c . || echo 0)
|
|
74
|
-
fi
|
|
75
|
-
case "$ACTUAL_AGENT_COUNT" in ''|*[!0-9]*) ACTUAL_AGENT_COUNT=0 ;; esac # fail closed: unreadable or untrusted counts as not posted
|
|
76
94
|
# Fresh shell: SELECTED_AGENT_COUNT is not carried over. Reload it from the PR-keyed launched-agent file, then fail closed when it is empty, non-integer or zero (a staging review always selects at least one reviewer).
|
|
77
95
|
[ -n "${SELECTED_AGENT_COUNT:-}" ] || { [ -n "${PR_NUMBER:-}" ] && [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && SELECTED_AGENT_COUNT=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | grep -c .); }
|
|
78
96
|
SELECTED_COUNT_INVALID=false
|
|
79
97
|
case "${SELECTED_AGENT_COUNT:-}" in ''|*[!0-9]*|0) SELECTED_COUNT_INVALID=true; SELECTED_AGENT_COUNT=0 ;; esac
|
|
98
|
+
# Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only).
|
|
99
|
+
[ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] || { [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && LAUNCHED_REVIEW_AGENTS=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | tr '\n' ' '); }
|
|
100
|
+
ACTUAL_AGENT_COUNT=0
|
|
101
|
+
DELIVERY_LAST=skipped # ok = the last attempt was a successful listing, error = it was an API error
|
|
102
|
+
DELIVERY_FOUND=" " # launched domains with a findings block, found on any attempt (found stays found)
|
|
103
|
+
DELIVERY_SEEN="" # union of trusted current-head reviewer domains over the successful listings
|
|
104
|
+
if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ] && [ "${DISPATCH_FAILED:-}" != "true" ]; then
|
|
105
|
+
if [ "$SELECTED_COUNT_INVALID" = "true" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
|
|
106
|
+
# Nothing to wait for: one listing for the report only, no retry.
|
|
107
|
+
if delivery_fetch; then DELIVERY_LAST=ok; DELIVERY_SEEN=$(delivery_domains); else DELIVERY_LAST=error; fi
|
|
108
|
+
else
|
|
109
|
+
# Retry on listing lag or an API error, after each FORGE_REVIEW_DELIVERY_RETRY_DELAYS wait (seconds; non-integers ignored).
|
|
110
|
+
_ATTEMPT=0
|
|
111
|
+
for _DELAY in 0 ${FORGE_REVIEW_DELIVERY_RETRY_DELAYS-5 15 45}; do
|
|
112
|
+
case "$_DELAY" in ''|*[!0-9]*) continue ;; esac
|
|
113
|
+
_ATTEMPT=$((_ATTEMPT + 1))
|
|
114
|
+
[ "$_ATTEMPT" -gt 1 ] && [ "$_DELAY" -gt 0 ] && sleep "$_DELAY"
|
|
115
|
+
if ! delivery_fetch; then DELIVERY_LAST=error; echo "DELIVERY: attempt ${_ATTEMPT}: API error listing PR comments"; continue; fi
|
|
116
|
+
DELIVERY_LAST=ok
|
|
117
|
+
DELIVERY_SEEN=$(printf '%s\n%s\n' "$DELIVERY_SEEN" "$(delivery_domains)" | grep -E '^[a-z0-9-]+$' | sort -u)
|
|
118
|
+
_PENDING=""
|
|
119
|
+
for _D in $LAUNCHED_REVIEW_AGENTS; do
|
|
120
|
+
case "$DELIVERY_FOUND" in *" $_D "*) continue ;; esac
|
|
121
|
+
if [ "$(delivery_count "$_D")" -ge 1 ]; then DELIVERY_FOUND="${DELIVERY_FOUND}${_D} "; echo "DELIVERY: ${_D} found on attempt ${_ATTEMPT}"; else _PENDING="${_PENDING} ${_D}"; fi
|
|
122
|
+
done
|
|
123
|
+
[ -z "$_PENDING" ] && break
|
|
124
|
+
echo "DELIVERY: attempt ${_ATTEMPT}: missing:${_PENDING}"
|
|
125
|
+
done
|
|
126
|
+
fi
|
|
127
|
+
ACTUAL_AGENT_COUNT=$(printf '%s\n' "$DELIVERY_SEEN" | grep -c . || true)
|
|
128
|
+
fi
|
|
129
|
+
case "$ACTUAL_AGENT_COUNT" in ''|*[!0-9]*) ACTUAL_AGENT_COUNT=0 ;; esac # fail closed: unreadable or untrusted counts as not posted
|
|
130
|
+
if [ "${DISPATCH_FAILED:-}" != "true" ] && [ "$SELECTED_COUNT_INVALID" != "true" ] && [ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] && [ "$DELIVERY_LAST" = error ]; then
|
|
131
|
+
# Never "missing": the comments could not be listed, so nothing is labelled or posted (forge#5166).
|
|
132
|
+
echo "REVIEW DELIVERY UNVERIFIED: could not list PR comments after ${_ATTEMPT:-1} attempt(s) (GitHub API error); found so far:${DELIVERY_FOUND% }"
|
|
133
|
+
echo "Do not re-dispatch reviewers: rerun the finalize step once GitHub answers."
|
|
134
|
+
exit 2
|
|
135
|
+
fi
|
|
80
136
|
if [ "${DISPATCH_FAILED:-}" = "true" ] || [ "$SELECTED_COUNT_INVALID" = "true" ] || [ "$ACTUAL_AGENT_COUNT" -lt "$SELECTED_AGENT_COUNT" ]; then
|
|
81
137
|
gh label create "review-degraded" --color "E4E669" --description "PR review panel was incomplete; re-review required before deployment. Managed by ForgeDock." --force -R "$GH_REPO" 2>/dev/null || true
|
|
82
138
|
gh pr edit "$PR_NUMBER" -R "$GH_REPO" --add-label "review-degraded" --add-label "needs-human" 2>/dev/null || true # allowlist:check-command-side-effects
|
|
@@ -91,7 +147,7 @@ Re-run the full staging review in a fresh session before deployment."
|
|
|
91
147
|
fi
|
|
92
148
|
# END panel-guard
|
|
93
149
|
# BEGIN delivery-check
|
|
94
|
-
# Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only).
|
|
150
|
+
# Reload the launched-agent list persisted at dispatch (validated: lowercase marker domains only); a no-op when the panel guard loaded it.
|
|
95
151
|
[ -n "${LAUNCHED_REVIEW_AGENTS:-}" ] || { [ -r "$_SD/${PR_NUMBER}_launched_review_agents" ] && LAUNCHED_REVIEW_AGENTS=$(grep -E '^[a-z0-9-]+$' "$_SD/${PR_NUMBER}_launched_review_agents" | sort -u | tr '\n' ' '); }
|
|
96
152
|
# Fail closed: a staging review always launches at least the Phase 2 material-change agent, so an empty list means lost state or skipped bookkeeping, never "nothing to verify".
|
|
97
153
|
if [ -z "${PR_NUMBER:-}" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
|
|
@@ -100,21 +156,9 @@ if [ -z "${PR_NUMBER:-}" ] || [ -z "${LAUNCHED_REVIEW_AGENTS:-}" ]; then
|
|
|
100
156
|
exit 1
|
|
101
157
|
fi
|
|
102
158
|
MISSING_AGENT_COMMENTS=""
|
|
103
|
-
#
|
|
104
|
-
# (an empty SHA would make the Reviewed-SHA match accept any head).
|
|
105
|
-
[ -n "${REVIEW_SHA_FULL:-}" ] || REVIEW_SHA_FULL=$(gh pr view "$PR_NUMBER" -R "$GH_REPO" --json headRefOid --jq '.headRefOid' 2>/dev/null || echo "")
|
|
106
|
-
case "$REVIEW_SHA_FULL" in *[!0-9a-f]*|"") REVIEW_SHA_FULL="" ;; *) [ "${#REVIEW_SHA_FULL}" -eq 40 ] || [ "${#REVIEW_SHA_FULL}" -eq 64 ] || REVIEW_SHA_FULL="" ;; esac
|
|
107
|
-
TRUSTED_SCRIPT=$(forge_resolve trusted-comments.sh)
|
|
159
|
+
# Counted once per attempt from the cached listing above (delivery_count); a domain not found on any attempt is missing.
|
|
108
160
|
for AGENT_DOMAIN in $LAUNCHED_REVIEW_AGENTS; do
|
|
109
|
-
|
|
110
|
-
if [ -n "$REVIEW_SHA_FULL" ] && [ -n "$TRUSTED_SCRIPT" ]; then
|
|
111
|
-
AGENT_COMMENT_COUNT=$(gh api --paginate "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" 2>/dev/null \
|
|
112
|
-
| bash "$TRUSTED_SCRIPT" count "^(?=[\\s\\S]*<!-- REVIEW-FINDINGS-START -->[\\s\\S]*<!-- REVIEW-FINDINGS-END -->)<!-- FORGE:REVIEW-AGENT:${AGENT_DOMAIN} -->[\\s\\S]*(^|\\n)Reviewed-SHA:[ ]${REVIEW_SHA_FULL}(\\r?\\n|$)" 2>/dev/null || echo 0)
|
|
113
|
-
fi
|
|
114
|
-
case "$AGENT_COMMENT_COUNT" in ''|*[!0-9]*) AGENT_COMMENT_COUNT=0 ;; esac # fail closed: counts as missing
|
|
115
|
-
if [ "${AGENT_COMMENT_COUNT:-0}" -lt 1 ]; then
|
|
116
|
-
MISSING_AGENT_COMMENTS="${MISSING_AGENT_COMMENTS} ${AGENT_DOMAIN}"
|
|
117
|
-
fi
|
|
161
|
+
case "$DELIVERY_FOUND" in *" ${AGENT_DOMAIN} "*) ;; *) MISSING_AGENT_COMMENTS="${MISSING_AGENT_COMMENTS} ${AGENT_DOMAIN}" ;; esac
|
|
118
162
|
done
|
|
119
163
|
|
|
120
164
|
if [ -n "$MISSING_AGENT_COMMENTS" ]; then
|
|
@@ -123,6 +167,7 @@ if [ -n "$MISSING_AGENT_COMMENTS" ]; then
|
|
|
123
167
|
exit 1
|
|
124
168
|
fi
|
|
125
169
|
# END delivery-check
|
|
170
|
+
rm -f "$DELIVERY_CACHE"
|
|
126
171
|
echo "DELIVERY: OK (${ACTUAL_AGENT_COUNT}/${SELECTED_AGENT_COUNT} launched reviewers posted a findings block for head ${REVIEW_SHA_FULL})"
|
|
127
172
|
exit 0
|
|
128
173
|
fi
|