forgedock 1.49.0 → 1.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/engine/outcome-codes.mjs +2 -0
- package/bin/engine-cli.mjs +123 -20
- package/bin/forgedock.mjs +31 -3
- package/bin/lean/batch-health.mjs +219 -0
- package/bin/lean/batch-health.test.mjs +156 -0
- package/bin/lean/batch.mjs +12 -3
- package/bin/lean/batch.test.mjs +304 -8
- package/bin/lean/conformance.mjs +79 -7
- package/bin/lean/conformance.test.mjs +63 -1
- package/bin/lean/context.mjs +8 -2
- package/bin/lean/context.test.mjs +11 -1
- package/bin/lean/credentials.mjs +18 -0
- package/bin/lean/execution.mjs +23 -9
- package/bin/lean/execution.test.mjs +93 -0
- package/bin/lean/fixclosereasons.mjs +52 -0
- package/bin/lean/fixclosereasons.test.mjs +64 -0
- package/bin/lean/heavy.mjs +14 -4
- package/bin/lean/heavy.test.mjs +55 -1
- package/bin/lean/lean.mjs +93 -36
- package/bin/lean/lean.test.mjs +143 -19
- package/bin/lean/live-execution.mjs +136 -0
- package/bin/lean/live-execution.test.mjs +83 -0
- package/bin/lean/orchestrate-blocked.test.mjs +47 -0
- package/bin/lean/orchestrate.mjs +135 -17
- package/bin/lean/prefilter.mjs +4 -3
- package/bin/lean/prefilter.test.mjs +5 -4
- package/bin/lean/prompts.mjs +4 -2
- package/bin/lean/recovery.test.mjs +5 -5
- package/bin/lean/sandbox-provider.contract.test.mjs +36 -1
- package/bin/lean/sandbox-remote.mjs +6 -2
- package/bin/lean/schedd.mjs +9 -0
- package/bin/pool/agent.mjs +3 -1
- package/bin/pool/router.mjs +72 -7
- package/bin/review-staging.mjs +2 -0
- package/bin/runner.mjs +24 -0
- package/commands/orchestrate.md +2 -1
- package/commands/review-pr-staging.md +2 -2
- package/docs/CONFIG.md +39 -1
- package/docs/OUTCOME-CODES.md +2 -1
- package/package.json +1 -1
- package/pool/README.md +2 -0
- package/pool/image/cs-cred +8 -1
- package/scripts/review-staging-post.sh +70 -25
- package/scripts/review-staging-post.test.sh +40 -8
|
@@ -73,7 +73,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
73
73
|
const call = w.runner[0];
|
|
74
74
|
assert.equal(call.model, DEFAULT_PREFILTER_MODEL); assert.equal(call.readOnly, true); assert.equal(call.resume, null); assert.equal(call.cwd, "/repo");
|
|
75
75
|
assert.match(call.prompt, /merged PR #90/); assert.doesNotMatch(call.prompt, /PR #91/);
|
|
76
|
-
assert.deepEqual(w.closed, [[12, "
|
|
76
|
+
assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
|
|
77
77
|
assert.deepEqual(w.labels, [[12, "invalid"]]);
|
|
78
78
|
assert.match(w.comments[0][1], /FORGE:PREFILTER/); assert.match(w.comments[0][1], /src\/foo\.js:3 null guard/); assert.match(w.comments[0][1], /PR #90/);
|
|
79
79
|
});
|
|
@@ -120,7 +120,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
120
120
|
assert.equal(r.closed, true); assert.equal(r.kind, "ci-green");
|
|
121
121
|
assert.equal(w.runner.length, 0);
|
|
122
122
|
assert.match(w.comments[0][1], /actions\/runs\/600/); assert.match(w.comments[0][1], /actions\/runs\/500/);
|
|
123
|
-
assert.deepEqual(w.closed, [[34, "
|
|
123
|
+
assert.deepEqual(w.closed, [[34, "completed"]]); // #5152: CI green again is delivered
|
|
124
124
|
});
|
|
125
125
|
|
|
126
126
|
it("#4305 the CI-green shortcut is skipped for intermittent, reopened and same-SHA-flaky CI failures", async () => {
|
|
@@ -204,7 +204,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
204
204
|
assert.equal(r2.closed, true); assert.equal(tiny.comments.length, 1); assert.match(tiny.comments[0][1], /^<!-- FORGE:PREFILTER -->/);
|
|
205
205
|
assert.match(flakyReason({ comments: [{ body: tiny.comments[0][1], author: { login: "forgedock-app[bot]" }, authorAssociation: "NONE" }] }, ["forgedock-app"])?.reason ?? "", /reopened/);
|
|
206
206
|
const dead = flaky(99); const r3 = await pf(dead, 34);
|
|
207
|
-
assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "
|
|
207
|
+
assert.equal(r3.closed, true); assert.equal(dead.runner.length, 0); assert.deepEqual(dead.closed, [[34, "completed"]]); assert.match(r3.detail, /no marker comment/);
|
|
208
208
|
});
|
|
209
209
|
|
|
210
210
|
it("a CI-failure issue whose workflow is still red (or whose latest run is the failing one) goes to a lane", async () => {
|
|
@@ -224,6 +224,7 @@ describe("pre-filter (#4123)", () => {
|
|
|
224
224
|
assert.deepEqual(namedPaths("fix lib/a.js <!-- x/hidden.js --!> visible/e.js and <!-- unclosed c/d.js"), ["lib/a.js", "visible/e.js"]);
|
|
225
225
|
const gone = world({ issues: { 5: { body: "crash in src/old/foo.js:12" } }, verdict: { verdict: "INVALID", evidence: ["src/old/foo.js removed in abc123"] } });
|
|
226
226
|
assert.equal((await pf(gone, 5)).closed, true);
|
|
227
|
+
assert.deepEqual(gone.closed, [[5, "not planned"]], "#5152: only an INVALID verdict closes not planned");
|
|
227
228
|
assert.match(gone.runner[0].prompt, /do not exist on origin\/staging: src\/old\/foo\.js/);
|
|
228
229
|
const kept = world({ issues: { 5: { body: "crash in src/old/foo.js:12 and src/new.js/x.ts" } }, paths: ["src/old/foo.js"] });
|
|
229
230
|
const r = await pf(kept, 5);
|
|
@@ -300,7 +301,7 @@ describe("orchestrate with the pre-filter (#4123)", () => {
|
|
|
300
301
|
assert.deepEqual(res.lanes, [13]);
|
|
301
302
|
assert.equal(logs.at(-1), "lanes: #13");
|
|
302
303
|
assert.ok(logs.some((s) => /^pre-filter: 2 checked, 1 candidate\(s\), 1 closed/.test(s)));
|
|
303
|
-
assert.deepEqual(w.closed, [[12, "
|
|
304
|
+
assert.deepEqual(w.closed, [[12, "completed"]]); // #5152: ALREADY-FIXED is delivered
|
|
304
305
|
await assert.rejects(triageFromCli([], {}), /usage/);
|
|
305
306
|
});
|
|
306
307
|
|
package/bin/lean/prompts.mjs
CHANGED
|
@@ -488,8 +488,10 @@ export function reviewerPrompt({ issue, comments = [], contract, result, files =
|
|
|
488
488
|
* #4160: the targeted re-ask after an incomplete or inconsistent reply. Only the missing or invalid ids are asked
|
|
489
489
|
* for, with the exact problems and the reviewer's previous reply; no diff is re-sent (the worktree still holds the change).
|
|
490
490
|
*/
|
|
491
|
-
export function reviewerRepairPrompt({ missing = [], problems = [], rows = null }) {
|
|
492
|
-
|
|
491
|
+
export function reviewerRepairPrompt({ missing = [], problems = [], rows = null, criteria = [] }) {
|
|
492
|
+
// #5164: a coverage misquote is re-asked with the issue's criterion lines verbatim, so the quote is copied, not retyped
|
|
493
|
+
const crit = (criteria ?? []).length ? `\n\n<issue_criteria>\nCopy every issue_coverage quote exactly (character for character, including #numbers and plurals) from one of these issue lines:\n${criteria.slice(0, 80).map((c) => `- ${String(c).slice(0, 400)}`).join("\n")}\n</issue_criteria>` : "";
|
|
494
|
+
return `Your conformance review is incomplete. The engine keeps your other rows; answer ONLY for these ids or sections: ${missing.join(", ") || "(see the problems)"}.\n\n<previous_reply_problems>\nYour previous reply was rejected as incomplete. Fix exactly these:\n${problems.slice(0, 60).map((p) => `- ${String(p).slice(0, 300)}`).join("\n")}\n</previous_reply_problems>${crit}\n\n<previous_reply>\n${cap(JSON.stringify(rows ?? {}), 20000)}\n</previous_reply>\n\nRead the changed files in your worktree as needed. Output ONLY a JSON object in a json code fence with just the rows for those ids (requirements rows by id, preserve rows by id, scope rows by file, issue_coverage in full if it is listed) plus "verdict":"CONFORMS|NONCONFORMING" consistent with all rows. Evidence is a backtick-quoted repo-relative path:line copied verbatim (e.g. \`apps/web/app/(console)/layout.tsx:12\`); for an evidence path reported as not existing, give the full repo-relative path of that file from the changed files or the repository.`;
|
|
493
495
|
}
|
|
494
496
|
|
|
495
497
|
/** Pre-filter (#4123): one read-only, single-turn check of an issue a cheap signal flagged as possibly already fixed. */
|
|
@@ -65,9 +65,9 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
65
65
|
assert.equal(calls.filter((c) => c.n === 1).length, 2, "the first run plus exactly one recovery");
|
|
66
66
|
assert.equal(r[1].outcome, "needs-human");
|
|
67
67
|
assert.deepEqual(r[1].recovery, { code: "ci.red", attempts: 1, outcome: "needs-human", result_code: "ci.red", detail: "CI verification failed again" });
|
|
68
|
-
assert.equal(r[2].outcome, "
|
|
68
|
+
assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
|
|
69
69
|
const lines = formatOutcomeLists(rows);
|
|
70
|
-
assert.equal(lines.length,
|
|
70
|
+
assert.equal(lines.length, 2); assert.match(lines[0], /^held subtree: #1 \(needs-human\) holds #2; release: forgedock orchestrate --requeue <batch> 1$/); assert.match(lines[1], /^unrecovered: #1 \(ci\.red: recovery ci\.red ended needs-human: CI verification failed again\)$/);
|
|
71
71
|
});
|
|
72
72
|
|
|
73
73
|
it("#4505 budget-exceeded: one recovery lane from the checkpoint at the next size; dependents stay queued; no relabel while it recovers", async () => {
|
|
@@ -92,7 +92,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
92
92
|
assert.equal(calls.filter((c) => c.n === 1).length, 1);
|
|
93
93
|
assert.equal(r[1].outcome, "budget-exceeded");
|
|
94
94
|
assert.equal(relabels.length, 1); assert.equal(relabels[0].issue, 1); assert.match(relabels[0].evidence, /budget\.exceeded[\s\S]*size S/);
|
|
95
|
-
assert.equal(r[2].outcome, "
|
|
95
|
+
assert.equal(r[2].outcome, "held"); assert.ok(!calls.some((c) => c.n === 2));
|
|
96
96
|
assert.ok(formatOutcomeLists(rows).some((l) => /^unrecovered: #1 \(budget\.exceeded: Implement turn budget spent/.test(l)));
|
|
97
97
|
});
|
|
98
98
|
|
|
@@ -124,7 +124,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
124
124
|
assert.equal(calls.filter((c) => c.n === 1).length, 1); assert.equal(calls.filter((c) => c.n === 3).length, 1);
|
|
125
125
|
assert.equal(calls.filter((c) => c.n === 4).length, 1, "terminal mechanical codes are not recovered either");
|
|
126
126
|
assert.ok(!decisions.some((d) => [1, 3].includes(d.issue)));
|
|
127
|
-
assert.equal(r[2].outcome, "
|
|
127
|
+
assert.equal(r[2].outcome, "held"); assert.equal(r[5].outcome, "merged");
|
|
128
128
|
const lines = formatOutcomeLists(rows);
|
|
129
129
|
assert.equal(lines.filter((l) => l.startsWith("escalations")).length, 1, "one escalation list");
|
|
130
130
|
const esc = lines.find((l) => l.startsWith("escalations"));
|
|
@@ -187,7 +187,7 @@ describe("in-batch recovery of mechanical needs-human (#4267)", () => {
|
|
|
187
187
|
const calls = [];
|
|
188
188
|
const rows = await orchestrate({ issues: [1, 2], readBody: async (n) => (n === 2 ? "**Depends on**: #1" : ""),
|
|
189
189
|
runOne: async (n) => { calls.push(n); return NH("ci.red"); } });
|
|
190
|
-
assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "
|
|
190
|
+
assert.deepEqual(calls, [1]); assert.deepEqual(rows.map((r) => r.outcome), ["needs-human", "held"]);
|
|
191
191
|
});
|
|
192
192
|
|
|
193
193
|
it("the attempt count survives a resume: a persisted recovery lane is launched with ctx.recover and is not retried again", async () => {
|
|
@@ -9,7 +9,7 @@ import { dirname, join } from "node:path";
|
|
|
9
9
|
import { tmpDir } from "../tests/_tmp.mjs";
|
|
10
10
|
import { containerProvider, parseExecution, providerByName, providerLanes, remoteTransportFor } from "./execution.mjs";
|
|
11
11
|
import { remoteProvider, httpTransport, isActionsEndpoint, redact } from "./sandbox-remote.mjs";
|
|
12
|
-
import { resolveCredential } from "./credentials.mjs";
|
|
12
|
+
import { memoKeychain, resolveCredential } from "./credentials.mjs";
|
|
13
13
|
import { makeLedger } from "./ledger.mjs";
|
|
14
14
|
import { openBatch } from "./batch.mjs";
|
|
15
15
|
import { orchestrate } from "./orchestrate.mjs";
|
|
@@ -208,6 +208,27 @@ describe("remote adapter (#4254)", () => {
|
|
|
208
208
|
await assert.rejects(r.status("x"), /router auth/);
|
|
209
209
|
});
|
|
210
210
|
|
|
211
|
+
it("5144 httpTransport: a function token is resolved per call", async () => {
|
|
212
|
+
let cur = null;
|
|
213
|
+
const seen = [];
|
|
214
|
+
const t = httpTransport({ endpoint: "https://pool.example", token: () => cur, fetchFn: async (url, init) => { seen.push(init.headers.authorization ?? null); return { status: 200, ok: true, text: async () => JSON.stringify({ state: "running" }), headers: { get: () => null } }; } });
|
|
215
|
+
await t.status("x"); cur = "tok-new"; await t.status("x"); cur = "tok-rotated"; await t.status("x"); cur = null; await t.status("x");
|
|
216
|
+
assert.deepEqual(seen, [null, "Bearer tok-new", "Bearer tok-rotated", null]);
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it("5144 remoteProvider: a token rotated after construction is redacted from pulled logs", async () => {
|
|
220
|
+
const batch = openBatch(join(tmpDir("remote-5144-"), "batches"), "bt");
|
|
221
|
+
const transport = fakeTransport();
|
|
222
|
+
let tok = null;
|
|
223
|
+
const p = remoteProvider({ batch, repo: "o/r", settings: { tokenEnv: "FD_TOKEN" }, transport, env: {}, credentials: (n) => (n === "FD_TOKEN" ? tok : null), gitIdentity: () => ({}) });
|
|
224
|
+
tok = "rotated-secret-789";
|
|
225
|
+
const h = await p.start({ issue: 3, args: [], env: {}, secrets: [] });
|
|
226
|
+
transport.boxes.get(h.id).log = `leak ${tok}\n`;
|
|
227
|
+
await p.status(h);
|
|
228
|
+
const log = readFileSync(batch.logFile(3), "utf8");
|
|
229
|
+
assert.equal(log.includes(tok), false);
|
|
230
|
+
});
|
|
231
|
+
|
|
211
232
|
it("httpTransport: bearer token, JSON protocol, 404 = gone/destroyed, 429 retryable; the token never in errors", async () => {
|
|
212
233
|
const calls = [];
|
|
213
234
|
const reply = (status, body, headers = {}) => ({ status, ok: status >= 200 && status < 300, text: async () => (body === undefined ? "" : JSON.stringify(body)), headers: { get: (k) => headers[k] ?? null } });
|
|
@@ -265,6 +286,20 @@ describe("remote config and the Actions constraint (#4254)", () => {
|
|
|
265
286
|
assert.equal(resolveCredential("FD_T", { env: {}, keychain: () => { throw new Error("locked"); } }), null);
|
|
266
287
|
assert.equal(resolveCredential("bad name", { env: { "bad name": "x" } }), null);
|
|
267
288
|
});
|
|
289
|
+
|
|
290
|
+
it("5159 memoKeychain: one lookup per name per TTL, misses and failures cached as null", () => {
|
|
291
|
+
let t = 0;
|
|
292
|
+
const calls = [];
|
|
293
|
+
const kc = memoKeychain({ keychain: (n) => { calls.push(n); if (n === "BAD") throw new Error("locked"); return n === "HIT" ? "v" : null; }, ttlMs: 1000, now: () => t });
|
|
294
|
+
assert.equal(kc("HIT"), "v"); assert.equal(kc("HIT"), "v");
|
|
295
|
+
assert.equal(kc("MISS"), null); assert.equal(kc("MISS"), null);
|
|
296
|
+
assert.equal(kc("BAD"), null); assert.equal(kc("BAD"), null);
|
|
297
|
+
assert.deepEqual(calls, ["HIT", "MISS", "BAD"]);
|
|
298
|
+
t = 1000;
|
|
299
|
+
assert.equal(kc("MISS"), null);
|
|
300
|
+
assert.deepEqual(calls, ["HIT", "MISS", "BAD", "MISS"], "re-read after the TTL");
|
|
301
|
+
assert.equal(resolveCredential("MISS", { env: { MISS: "e" }, keychain: kc }), "e", "the env still wins");
|
|
302
|
+
});
|
|
268
303
|
});
|
|
269
304
|
|
|
270
305
|
describe("gc reaps a remote sandbox left by a dead dispatcher (#4254)", () => {
|
|
@@ -43,7 +43,8 @@ export function isActionsEndpoint(endpoint) {
|
|
|
43
43
|
/**
|
|
44
44
|
* Generic HTTP transport: POST {endpoint}/sandboxes (body = launch spec), GET /sandboxes/:id -> {state},
|
|
45
45
|
* GET /sandboxes/:id/logs?cursor=N -> {text, cursor}, GET /sandboxes/:id/result -> lane result (404 = none yet),
|
|
46
|
-
* GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token
|
|
46
|
+
* GET /sandboxes/:id/usage -> {minutes, agent?} (agent: the pool agent that ran it, #4549), DELETE /sandboxes/:id (404 = gone). `Authorization: Bearer <token>`
|
|
47
|
+
* (`token` is a string, or a function called per request, #5144).
|
|
47
48
|
* 429/503 on launch is a retryable capacity error (`Retry-After` seconds). Error text never carries the token.
|
|
48
49
|
* Optional `POST /sandboxes/:id/stop` (#4612) asks the pool to SIGTERM the lane so its teardown checkpoints and it reports `deferred run.interrupted`; 2xx = sent, 404/405 = unsupported or gone (never a DELETE fallback).
|
|
49
50
|
* Optional relay (#4778): `GET /sandboxes/:id/relay -> {requests: [{rid, argv}]}` lists a sandbox's pending read-only capability
|
|
@@ -53,7 +54,9 @@ export function isActionsEndpoint(endpoint) {
|
|
|
53
54
|
export function httpTransport({ endpoint, token = null, fetchFn = globalThis.fetch }) {
|
|
54
55
|
const root = String(endpoint ?? "").replace(/\/+$/, "");
|
|
55
56
|
const call = async (method, path, body) => {
|
|
56
|
-
|
|
57
|
+
// #5144: `token` may be a function, resolved on every request so a token fixed or rotated mid-batch is the next bearer.
|
|
58
|
+
const bearer = typeof token === "function" ? token() : token;
|
|
59
|
+
const res = await fetchFn(`${root}${path}`, { method, headers: { accept: "application/json", ...(body ? { "content-type": "application/json" } : {}), ...(bearer ? { authorization: `Bearer ${bearer}` } : {}) },
|
|
57
60
|
...(body ? { body: JSON.stringify(body) } : {}) });
|
|
58
61
|
let data = null;
|
|
59
62
|
try { const t = await res.text(); data = t ? JSON.parse(t) : null; } catch { data = null; }
|
|
@@ -195,6 +198,7 @@ export function remoteProvider({ batch, repo, settings = {}, name = "remote", tr
|
|
|
195
198
|
const id = idFor(issue);
|
|
196
199
|
// Each allowlisted secret from the env, else the keychain (#4254); nothing else from the host.
|
|
197
200
|
const resolved = {};
|
|
201
|
+
if (settings.tokenEnv) remember(credentials(settings.tokenEnv)); // #5144: a token rotated since construction is redacted too
|
|
198
202
|
for (const k of secrets) { const v = laneEnv?.[k] !== undefined && laneEnv[k] !== "" ? String(laneEnv[k]) : credentials(k); if (v) { resolved[k] = v; remember(v); } }
|
|
199
203
|
const li = args.indexOf("--lane");
|
|
200
204
|
const resultFile = `/result/${issue}.json`;
|
package/bin/lean/schedd.mjs
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* {op:"attach", id, resume?} → (`resume` also clears a pending cancel, #4592) log.jsonl rows replayed, then live rows/lines and done; a finished batch replays report.json
|
|
12
12
|
* ({type:"done", replay:true}) unless it has unrun added issues; a batch nobody hosts is resumed here
|
|
13
13
|
* {op:"set-parallel", id, k} → {type:"ok", parallel}
|
|
14
|
+
* {op:"set-execution", id, maxRemote?, mode?} → {type:"ok", execution} (#5138): live max_remote / placement (remote|hybrid|local); remote/hybrid refused for a batch with no provider
|
|
14
15
|
* {op:"requeue", id, issues, reason, by} → {type:"ok", requeued, refused, hosted, reopened} (#4550): see batch.requeue; a finished batch is hosted again
|
|
15
16
|
* {op:"add", id, issues} → {type:"ok", added, skipped, hosted} (#4408): merged into the batch's `add` file, which a
|
|
16
17
|
* hosted runner reads on its next pass and an unhosted batch on its next resume
|
|
@@ -42,6 +43,7 @@ import { fileURLToPath } from "node:url";
|
|
|
42
43
|
import { schedDir, pidAlive } from "./sched.mjs";
|
|
43
44
|
import { HEARTBEAT_MS, writeHeartbeat, clearHeartbeat } from "./schedd-health.mjs";
|
|
44
45
|
import { batchRoot, listBatches, openBatch, SECRET_ENV } from "./batch.mjs";
|
|
46
|
+
import { applySetExecution } from "./live-execution.mjs";
|
|
45
47
|
|
|
46
48
|
export const IDLE_MS = 10 * 60 * 1000;
|
|
47
49
|
export const STOP_MS = 10 * 1000;
|
|
@@ -250,6 +252,13 @@ export async function startDaemon({ dir = schedDir(), runBatch, idleMs = Number(
|
|
|
250
252
|
b.setParallel(Number(req.k));
|
|
251
253
|
return send({ type: "ok", parallel: Number(req.k) }, true);
|
|
252
254
|
}
|
|
255
|
+
if (req.op === "set-execution") {
|
|
256
|
+
// #5138: live max_remote / placement mode; the hosted runner re-reads the batch's `execution` file on every admission.
|
|
257
|
+
const b = openBatch(root, req.id);
|
|
258
|
+
if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
|
|
259
|
+
const execution = applySetExecution(b, { ...(req.maxRemote !== undefined ? { maxRemote: req.maxRemote } : {}), ...(req.mode !== undefined ? { mode: req.mode } : {}) });
|
|
260
|
+
return send({ type: "ok", execution }, true);
|
|
261
|
+
}
|
|
253
262
|
if (req.op === "add") {
|
|
254
263
|
const b = openBatch(root, req.id);
|
|
255
264
|
if (!b.exists()) throw new Error(`no batch ${req.id} in ${root}`);
|
package/bin/pool/agent.mjs
CHANGED
|
@@ -245,6 +245,8 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
|
|
|
245
245
|
const c = credCache[kind];
|
|
246
246
|
if (!c || now() - c.at > CRED_TTL[kind]) {
|
|
247
247
|
const x = await call(cfg.router_url, "GET", `/agent/creds/${kind}`, { token: cfg.token, timeoutMs: 20000 });
|
|
248
|
+
// #5165: every router login at its usage ceiling -> pass the typed 429 (reason, reset) through, uncached, so the lane defers.
|
|
249
|
+
if (x.status === 429 && x.data && typeof x.data === "object") return { status: 429, body: x.data, retryAfter: x.retryAfter };
|
|
248
250
|
if (!x.ok || !x.data?.token) return { status: 502, body: { error: `router credentials (${kind}): HTTP ${x.status}` } };
|
|
249
251
|
credCache[kind] = { token: x.data.token, at: now() };
|
|
250
252
|
}
|
|
@@ -397,7 +399,7 @@ export function createAgent(cfg, { docker, now = () => Date.now(), log = console
|
|
|
397
399
|
const url = new URL(req.url, "http://agent");
|
|
398
400
|
try {
|
|
399
401
|
const cred = req.method === "GET" && /^\/sandboxes\/([^/]+)\/creds\/([a-z]+)$/.exec(url.pathname);
|
|
400
|
-
if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body); }
|
|
402
|
+
if (cred) { const r = await sandboxCred(decodeURIComponent(cred[1]), cred[2], req); return send(res, r.status, r.body, r.retryAfter ? { "retry-after": String(r.retryAfter) } : {}); }
|
|
401
403
|
const relay = (req.method === "POST" && /^\/sandboxes\/([^/]+)\/relay$/.exec(url.pathname)) || (req.method === "GET" && /^\/sandboxes\/([^/]+)\/relay\/([^/]+)$/.exec(url.pathname));
|
|
402
404
|
if (relay) { const r = await sandboxRelay(decodeURIComponent(relay[1]), relay[2] ? decodeURIComponent(relay[2]) : null, req); return send(res, r.status, r.body ?? {}); }
|
|
403
405
|
if (url.pathname !== "/health" && !bearerOk(req, cfg.token)) return send(res, 401, { error: "unauthorized", hop: "agent", reason: "agent auth: router token rejected by agent" });
|
package/bin/pool/router.mjs
CHANGED
|
@@ -93,19 +93,28 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
93
93
|
}
|
|
94
94
|
return gh.token;
|
|
95
95
|
}
|
|
96
|
-
// Claude access token for sandboxes: the
|
|
97
|
-
// short-lived access token is shared; refresh tokens stay here, so this host
|
|
96
|
+
// Claude access token for sandboxes (#5165): among the logins with 30+ minutes left, the one with the lowest tokenbaton
|
|
97
|
+
// usage (validity breaks ties). Only the short-lived access token is shared; refresh tokens stay here, so this host
|
|
98
|
+
// keeps refreshing its own sessions.
|
|
98
99
|
function claudeToken() {
|
|
99
100
|
const dirs = cfg.claude_config_dirs ?? readdirSync(homedir()).filter((d) => /^\.claude(-\d+)?$/.test(d)).map((d) => join(homedir(), d));
|
|
100
|
-
|
|
101
|
+
const logins = [];
|
|
101
102
|
for (const d of dirs) {
|
|
102
103
|
try {
|
|
103
104
|
const o = JSON.parse(readFileSync(join(d, ".credentials.json"), "utf8")).claudeAiOauth;
|
|
104
|
-
|
|
105
|
+
const n = /\.claude-(\d+)$/.exec(d)?.[1];
|
|
106
|
+
if (o?.accessToken) logins.push({ n: n ? Number(n) : null, accessToken: o.accessToken, expiresAt: o.expiresAt });
|
|
105
107
|
} catch { /* no login in this dir */ }
|
|
106
108
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
const src = usageSource(cfg);
|
|
110
|
+
const usage = readUsage(src, logins.map((l) => l.n).filter((n) => n !== null));
|
|
111
|
+
const pick = pickClaudeLogin({ logins, usage, ceilingPct: cfg.usage_ceiling_pct ?? 95, now: now() });
|
|
112
|
+
if (pick.status === 503) throw Object.assign(new Error("no Claude login on the router host with 30+ minutes left"), { status: 503 });
|
|
113
|
+
if (pick.status === 429) { log.warn?.(`claude login: every account at or above the usage ceiling; ${pick.body.reason}`); return pick; }
|
|
114
|
+
if (pick.fallback) log.warn?.(`claude login: no usage source (${src}), falling back to longest validity`);
|
|
115
|
+
const { login, pct } = pick;
|
|
116
|
+
log.info?.(`claude login selected: account=${login.n ?? "default"} usage=${pct ?? "unknown"}% expires_at=${new Date(login.expiresAt).toISOString()}`);
|
|
117
|
+
return { status: 200, body: { token: login.accessToken, expires_at: new Date(login.expiresAt).toISOString() } };
|
|
109
118
|
}
|
|
110
119
|
const agentCall = (a, method, path, body, timeoutMs) => call(a.url, method, path, { body, token: a.token, timeoutMs, fetchFn });
|
|
111
120
|
|
|
@@ -285,7 +294,7 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
285
294
|
[route("GET", "/agents"), fd(async () => ({ status: 200, body: { agents: await capacities(), sandboxes: sbx() } }))],
|
|
286
295
|
[route("POST", "/codes"), fd(async () => ({ status: 200, body: issueCode() }))],
|
|
287
296
|
[route("GET", "/agent/creds/github"), { fn: async () => ({ status: 200, body: { token: await githubToken() } }), auth: "agent" }],
|
|
288
|
-
[route("GET", "/agent/creds/claude"), { fn: async () =>
|
|
297
|
+
[route("GET", "/agent/creds/claude"), { fn: async () => claudeToken(), auth: "agent" }],
|
|
289
298
|
[route("POST", "/agent/relay/:id"), { fn: async (p, body, _u, req) => relayPost(req, p, body), auth: "agent" }], // #4778
|
|
290
299
|
[route("GET", "/agent/relay/:id/:rid"), { fn: async (p, _b, _u, req) => relayPoll(req, p), auth: "agent" }],
|
|
291
300
|
[route("GET", "/agent/bundle"), { fn: async () => ({ bundle: true }), auth: "agent" }],
|
|
@@ -355,6 +364,62 @@ export function createRouter(cfg, { fetchFn = globalThis.fetch, now = () => Date
|
|
|
355
364
|
return { handle, launch, claim, tick, enroll, issueCode, capacities, status, store };
|
|
356
365
|
}
|
|
357
366
|
|
|
367
|
+
/** tokenbaton's usage dir (#5165): router.json `usage_source`, else `$TOKENBATON_HOME`, else `~/.tokenbaton`. */
|
|
368
|
+
export function usageSource(cfg, env = process.env) {
|
|
369
|
+
return cfg.usage_source ?? env.TOKENBATON_HOME ?? join(homedir(), ".tokenbaton");
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** One tokenbaton `live.<n>` line (`<5h %> <5h reset s> <weekly %> <weekly reset s> [updated s]`), or null; the format of bin/lean/rotator.mjs parseLive. */
|
|
373
|
+
export function parseLiveUsage(text) {
|
|
374
|
+
const f = String(text ?? "").trim().split(/\s+/).map(Number);
|
|
375
|
+
if (f.length < 4 || !f.slice(0, 4).every(Number.isFinite)) return null;
|
|
376
|
+
return { fiveHourPct: f[0], fiveHourResetMs: f[1] * 1000, weeklyPct: f[2], weeklyResetMs: f[3] * 1000 };
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/** Map account n -> parsed `live.<n>` for the given accounts; unreadable or malformed files are left out. */
|
|
380
|
+
export function readUsage(dir, ns, readFile = (p) => readFileSync(p, "utf8")) {
|
|
381
|
+
const out = new Map();
|
|
382
|
+
for (const n of ns) { try { const u = parseLiveUsage(readFile(join(dir, `live.${n}`))); if (u) out.set(n, u); } catch { /* no usage for n */ } }
|
|
383
|
+
return out;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Pick the Claude login to hand out (#5165). Candidates have 30+ minutes of validity. A login with known usage is
|
|
388
|
+
* excluded at or above `ceilingPct` 5-hour use or 100% weekly use (a passed reset counts as 0, like the local rotator);
|
|
389
|
+
* the rest rank by 5-hour use, then validity. Logins with unknown usage follow. No usage at all: longest validity,
|
|
390
|
+
* `fallback: true`. Every candidate excluded: a typed 429 carrying the earliest reset.
|
|
391
|
+
* @returns {{status: 200, login, pct, fallback} | {status: 429, retryAfter, body} | {status: 503}}
|
|
392
|
+
*/
|
|
393
|
+
export function pickClaudeLogin({ logins, usage, ceilingPct = 95, now }) {
|
|
394
|
+
const valid = logins.filter((l) => l.accessToken && l.expiresAt > now + 30 * 60000);
|
|
395
|
+
if (!valid.length) return { status: 503 };
|
|
396
|
+
const used = (pct, resetMs) => (resetMs <= now ? 0 : pct);
|
|
397
|
+
const byValidity = (a, b) => b.expiresAt - a.expiresAt;
|
|
398
|
+
const known = [], unknown = [], blocked = [];
|
|
399
|
+
for (const l of valid) {
|
|
400
|
+
const u = l.n !== null && l.n !== undefined ? usage.get(l.n) : undefined;
|
|
401
|
+
if (!u) { unknown.push(l); continue; }
|
|
402
|
+
const five = used(u.fiveHourPct, u.fiveHourResetMs), week = used(u.weeklyPct, u.weeklyResetMs);
|
|
403
|
+
if (five >= ceilingPct || week >= 100) {
|
|
404
|
+
const r = [];
|
|
405
|
+
if (five >= ceilingPct) r.push(u.fiveHourResetMs);
|
|
406
|
+
if (week >= 100) r.push(u.weeklyResetMs);
|
|
407
|
+
blocked.push(Math.max(...r));
|
|
408
|
+
} else known.push({ l, pct: five });
|
|
409
|
+
}
|
|
410
|
+
if (known.length) {
|
|
411
|
+
known.sort((a, b) => a.pct - b.pct || byValidity(a.l, b.l));
|
|
412
|
+
return { status: 200, login: known[0].l, pct: known[0].pct, fallback: false };
|
|
413
|
+
}
|
|
414
|
+
if (unknown.length) {
|
|
415
|
+
unknown.sort(byValidity);
|
|
416
|
+
return { status: 200, login: unknown[0], pct: null, fallback: !blocked.length };
|
|
417
|
+
}
|
|
418
|
+
const resetMs = Math.min(...blocked), at = new Date(resetMs).toISOString();
|
|
419
|
+
return { status: 429, retryAfter: Math.max(1, Math.ceil((resetMs - now) / 1000)),
|
|
420
|
+
body: { error: "usage limit", code: "rate_limited", reason: `usage limit hit; resets at ${at}`, reset_at: at } };
|
|
421
|
+
}
|
|
422
|
+
|
|
358
423
|
export function startRouter(cfg, deps) {
|
|
359
424
|
const router = createRouter(cfg, deps);
|
|
360
425
|
const server = createServer(router.handle);
|
package/bin/review-staging.mjs
CHANGED
|
@@ -385,6 +385,8 @@ async function finalize(a, io) {
|
|
|
385
385
|
// Incomplete-panel guard + 6.75 delivery
|
|
386
386
|
const d = await bash(io, "review-staging-post.sh", ["delivery", ...common], env);
|
|
387
387
|
relay(io, d);
|
|
388
|
+
// Exit 2 = the comments could not be listed on any attempt (forge#5166): rerun, never re-dispatch reviewers that may have posted.
|
|
389
|
+
if (d.code === 2) throw new Stop(1, ["REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery could not be verified (GitHub comment listing failed); rerun --finalize, do not re-dispatch reviewers", BLOCK_DEPLOY]);
|
|
388
390
|
if (d.code !== 0) throw new Stop(1, [`REVIEW_RESULT: status: BLOCKED, blocker: reviewer delivery incomplete (exit ${d.code}); re-dispatch each missing reviewer once with the note "structured findings block required", else the panel is degraded`, BLOCK_DEPLOY]);
|
|
389
391
|
|
|
390
392
|
// 6.5 verdict handling (override, posture, test-gate markers)
|
package/bin/runner.mjs
CHANGED
|
@@ -1352,6 +1352,27 @@ export function detectUsageLimit(stdout, output = "") {
|
|
|
1352
1352
|
return hasUsageLimitResetClause(output || "");
|
|
1353
1353
|
}
|
|
1354
1354
|
|
|
1355
|
+
// forge#5168: the claude CLI's own login/authentication failure texts. Literal
|
|
1356
|
+
// alternatives only (linear on untrusted input).
|
|
1357
|
+
const AUTH_FAILURE_RE = /Failed to authenticate|authentication_error|OAuth token (?:has )?expired|reached its time limit and cannot be renewed|Invalid API key|Please run \/login/i;
|
|
1358
|
+
|
|
1359
|
+
/**
|
|
1360
|
+
* Detects whether a failed CLI run was an authentication failure of the claude
|
|
1361
|
+
* login (forge#5168): an expired or invalid login is an account condition, not
|
|
1362
|
+
* a transient engine error. With an envelope on stdout, it must be
|
|
1363
|
+
* `is_error: true` with a `result` naming the failure; without one, the plain
|
|
1364
|
+
* combined output must. `stdout` is parsed ALONE (forge#2422); never throws.
|
|
1365
|
+
*
|
|
1366
|
+
* @param {string} stdout - raw stdout of the CLI run
|
|
1367
|
+
* @param {string} [output] - combined stdout+stderr, for the plain-text fallback
|
|
1368
|
+
* @returns {boolean}
|
|
1369
|
+
*/
|
|
1370
|
+
export function detectAuthFailure(stdout, output = "") {
|
|
1371
|
+
const env = parseCliEnvelope(stdout);
|
|
1372
|
+
if (env) return env.is_error === true && typeof env.result === "string" && AUTH_FAILURE_RE.test(env.result);
|
|
1373
|
+
return typeof output === "string" && AUTH_FAILURE_RE.test(output);
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1355
1376
|
/**
|
|
1356
1377
|
* Best-effort extraction of a Claude CLI session-limit reset time from
|
|
1357
1378
|
* captured stdout/stderr (forge#2241). The CLI's own session-limit message
|
|
@@ -1844,6 +1865,9 @@ export function runCliBackend({
|
|
|
1844
1865
|
// default when the reset time itself is missing or unparseable. The
|
|
1845
1866
|
// engine, not the runner, owns that default — no epoch is fabricated here.
|
|
1846
1867
|
if (isUsageLimit) err.usageLimit = true;
|
|
1868
|
+
// forge#5168: a failed login (never when the usage limit already matched,
|
|
1869
|
+
// which keeps precedence) lets the lane defer as `backend.auth`.
|
|
1870
|
+
else if (detectAuthFailure(stdout, output)) err.authFailure = true;
|
|
1847
1871
|
if (resetAt) {
|
|
1848
1872
|
err.resetAt = resetAt;
|
|
1849
1873
|
// forge#2524: also attach a machine-usable epoch-ms timestamp so
|
package/commands/orchestrate.md
CHANGED
|
@@ -100,7 +100,8 @@ remaining maintenance once no feature is pending; otherwise the rest ends `held
|
|
|
100
100
|
(live lanes are re-adopted, nothing is launched twice). The runner is hosted by the detached `forgedock schedd` daemon,
|
|
101
101
|
which resumes every daemon-hosted batch that has no report yet when it starts, so a closed session needs no manual
|
|
102
102
|
`--resume`; a batch started without the daemon still needs `forgedock orchestrate --resume <id>`. Change K of a running batch with
|
|
103
|
-
`forgedock orchestrate --set-parallel <id> K`.
|
|
103
|
+
`forgedock orchestrate --set-parallel <id> K`. Change its remote lane cap or placement with `forgedock orchestrate --set-max-remote <id> N` and
|
|
104
|
+
`--set-placement <id> remote|hybrid|local` (remote/hybrid only for a batch started with a remote provider; running lanes keep running). Issues filed while a batch runs join it with
|
|
104
105
|
`forgedock orchestrate --add <id> N...` (or `--resume <id> N...`), under its K and DAG; never start a second batch
|
|
105
106
|
for them. After a decision (a `FORGE:DECISION` posted and `needs-human` removed), a blocked issue that already ended in the batch
|
|
106
107
|
(`needs-human`, `invalid`, `skipped`, `dependency-*`, `ci-pending`, `deferred`) returns to it with
|
|
@@ -401,14 +401,14 @@ node "$FD_ROOT/bin/forgedock.mjs" review-staging "$PR_NUMBER" --finalize --test-
|
|
|
401
401
|
```
|
|
402
402
|
|
|
403
403
|
The call runs these steps in order:
|
|
404
|
-
1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying.
|
|
404
|
+
1. The incomplete-panel guard and Phase 6.75 delivery verification. A launched reviewer without a trusted current-head comment that carries a findings block is a delivery failure. An empty or unreadable launched-agent list also fails, and is never evidence that nothing needed verifying. The PR comments are listed once per attempt. A missing reviewer or a GitHub API error is re-listed with backoff (5 s, 15 s, 45 s) before any verdict, and each domain is reported with the attempt it was found on.
|
|
405
405
|
2. The Phase 6.5 verdict handling.
|
|
406
406
|
3. `scripts/review-finalize.sh`:
|
|
407
407
|
- `extract`: trusted comments only, dedup, and the `commands/review-pr.md` §6B.4 provenance and concreteness gates. **CRITICAL/HIGH are never routed as pre-existing**, whatever their confidence.
|
|
408
408
|
- `classify --dry-run`, then `classify` for real. This passes `--scope-files` (the 0B.1 scoped set when `SCOPE_MODE=scoped`) and `--prior-prs` (the bundle members), and posts `FORGE:NOTE_DISPOSITION`.
|
|
409
409
|
- `file-prepare`.
|
|
410
410
|
|
|
411
|
-
Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A.
|
|
411
|
+
Findings come only from `FINDING:` markers, never from free text. **Exit 1 is a stop**: a `REVIEW DELIVERY FAILURE`, `RESULT: BLOCK DEPLOY` or `REVIEW_RESULT: status: BLOCKED` line names the cause. File nothing and emit no deploy approval. On a delivery failure, re-dispatch each missing reviewer once with the note "structured findings block required". A second miss is a degraded panel. Then rerun 7A. `REVIEW DELIVERY UNVERIFIED` means GitHub could not list the comments on the last attempt. It stops the run, but nothing is labelled or posted. Do not re-dispatch anyone: rerun 7A once GitHub answers.
|
|
412
412
|
|
|
413
413
|
Exit 0 prints the following:
|
|
414
414
|
- `TEST_GATE:`, `SCOPE_MODE=`.
|
package/docs/CONFIG.md
CHANGED
|
@@ -438,6 +438,7 @@ orchestration:
|
|
|
438
438
|
| `max_concurrent` | integer | No | Max concurrent top-level `/work-on` dispatches. It does not bound cascade growth; under `policy: all` it raises drain and spawn throughput together. Default: 12. |
|
|
439
439
|
| `breaker.errors` / `breaker.window_min` | integer | No | `forgedock orchestrate` circuit breaker (#4121): after `errors` engine-errors within `window_min` minutes the batch stops admitting lanes, lets running lanes finish, and keeps the rest pending for `--resume`. `errors: 0` turns it off. Default: 3 in 30 |
|
|
440
440
|
| `max_deferrals` | integer | No | `forgedock orchestrate`: requeues of one issue after a non-host `deferred` (usage limit, merge/push transient, CLI idle, lease held, gh unavailable) before its row is final. `host.busy` and `sandbox.capacity` (sandbox pool full) deferrals are not capped. Default: 3 |
|
|
441
|
+
| `transient_retries` | integer | No | `forgedock orchestrate` (#5161): backoff retries of an issue whose lane ended `engine-error` with a transport or gateway failure in its detail (curl `returned error: 5xx`, `HTTP/1.1 5xx`, `Bad Gateway`, `Service Unavailable`, `overloaded`, connection reset/refused/timeout). Delays 30s, 2m, 8m, then 15m each; dependents stay pending meanwhile. Not counted against the single engine-error relaunch; once spent, the row ends `engine-error`. Every such error still counts toward the breaker. `0` turns transient retries off. Default: 4 |
|
|
441
442
|
| `blocked_label` | string | No | `forgedock orchestrate` (#4773): an open issue carrying this label is admitted to the batch but not dispatched; removing the label releases it on the next scheduler pass. A `Blocked by #N` / `Depends on #N` line in the body of a trusted author (OWNER, MEMBER, COLLABORATOR or a bot) is a DAG edge to #N, and `--add` prints the edges it created. Empty turns the label off. Default: `blocked` |
|
|
442
443
|
| `auto_add_children` | string | No | `forgedock orchestrate` (#4965): which children a running batch adds by itself. `off` (no auto-adds; a decomposed lane's sub-issues stay out too), `decomposed` (a decomposed lane's sub-issues join, as before), `epic` (also the open children of every batch issue and of each `--epic N`, found by a `**Parent**: #P` line, the `parent` field or GitHub sub-issues, including children filed after the batch started) or `all`. Only children whose author is trusted (an owner, member or collaborator, or a bot, the `Blocked by` rule) are auto-added; any other is skipped with a logged row. Children labelled `needs-human` and children another live batch of the repo still holds are not added. Each pass that adds posts one batch decision and one `added` row listing the children and edges. `forgedock orchestrate --auto-children <mode>` overrides it per batch and `--resume` keeps it. Default: `decomposed` |
|
|
443
444
|
| `auto_add_children_poll_s` | integer | No | Seconds between child scans for `epic`/`all` (one `gh issue list` per scan). Default: `120` |
|
|
@@ -470,6 +471,43 @@ orchestration:
|
|
|
470
471
|
|
|
471
472
|
---
|
|
472
473
|
|
|
474
|
+
## `orchestrate.health` and `notify` (OPTIONAL)
|
|
475
|
+
|
|
476
|
+
Early batch health (#5167). `forgedock orchestrate` checks each batch at T+15 and T+60 minutes after it was created (`plan.json` `created`), and once more when the batch ends. When the batch starts, it prints the schedule: `health checks: T+15 14:05Z, T+60 14:50Z, and at batch end; forgedock status --batch <id>`. The first unhealthy check raises one `BATCH UNHEALTHY <id>` alert, which lists the top causes and the exact recovery commands (`--requeue`, a relaunch, `--resume`, `--cancel`). `health.json` in the batch dir records the checkpoints already done and the alert, so a daemon restart arms only the remaining checks and a batch never alerts twice. Checks and alerts never fail, delay or change the batch.
|
|
477
|
+
|
|
478
|
+
Causes: `root-failed` (a root issue, meaning one with no dependency inside the batch, ended non-merged); `terminal-share` (the share of rows ended `skipped`/`held`/`engine-error`/`needs-human` or another non-merged outcome except `invalid`/`decomposed` is at or above the threshold, with nothing merged); `stalled` (0 lanes running while issues are pending, operator-held issues excepted); `refused` (the launch was refused).
|
|
479
|
+
|
|
480
|
+
```yaml
|
|
481
|
+
orchestrate:
|
|
482
|
+
health:
|
|
483
|
+
root_failed: true # any root ended non-merged is unhealthy (default true)
|
|
484
|
+
terminal_share: 0.5 # 0..1, terminal-non-merged share with 0 merged (default 0.5)
|
|
485
|
+
stalled: true # 0 running while pending > 0 (default true)
|
|
486
|
+
issue: 123 # comment target; default: the batch's --epic issue(s); a milestone gets no comment
|
|
487
|
+
checks_min: [15, 60] # minutes after launch (default [15, 60])
|
|
488
|
+
notify:
|
|
489
|
+
command: "curl -fsS -H 'content-type: application/json' -d @- https://hooks.example/forge"
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
| Key | Default | Meaning |
|
|
493
|
+
|---|---|---|
|
|
494
|
+
| `orchestrate.health.root_failed` | `true` | Raise `root-failed`. |
|
|
495
|
+
| `orchestrate.health.terminal_share` | `0.5` | Threshold for `terminal-share`. |
|
|
496
|
+
| `orchestrate.health.stalled` | `true` | Raise `stalled`. |
|
|
497
|
+
| `orchestrate.health.issue` | none | Issue to comment on. When unset, the alert comments on each `--epic` issue. |
|
|
498
|
+
| `orchestrate.health.checks_min` | `[15, 60]` | Timed checkpoints, in minutes after launch. |
|
|
499
|
+
| `notify.command` | none | Shell command run once per alert. It gets the alert JSON (`{alert, batch, repo, text, causes, recovery}`) on stdin and `FORGEDOCK_ALERT=BATCH_UNHEALTHY`, `FORGEDOCK_ALERT_BATCH` and `FORGEDOCK_ALERT_TEXT` in its env. It is killed after 30 s. A failure only prints a `health: notify.command failed` line. |
|
|
500
|
+
|
|
501
|
+
The keys are flattened one level by the parser, the same as `orchestrate.drift.*`. Health keys alone do not turn drift on.
|
|
502
|
+
|
|
503
|
+
Every channel is best effort: the runner line, the GitHub comment (marker `FORGE:BATCH-HEALTH`), `notify.command`, and the `forgedock status` line. `forgedock status` lists the unhealthy batches that are unresolved (no report yet, or alerted within the last day) and the launch refusals from the last day. `forgedock status --batch <id>` shows that batch's health (healthy, or unhealthy with causes and recovery) and its pending checkpoints.
|
|
504
|
+
|
|
505
|
+
A refused milestone start (#4659, `parallel milestone start refused: …`) still exits 1 with the error on stderr and creates no batch dir. It is also recorded in `<batch root>/refusals.json` (the newest 20 are kept) and sends the same alert, with the merge-the-dependency or `--force "<reason>"` relaunch.
|
|
506
|
+
|
|
507
|
+
**Commands that use this section**: `forgedock orchestrate`, `forgedock status`
|
|
508
|
+
|
|
509
|
+
---
|
|
510
|
+
|
|
473
511
|
## `decisions` (OPTIONAL)
|
|
474
512
|
|
|
475
513
|
Decision engine (#4766, see [DECISIONS.md](DECISIONS.md)). The issue-watch stage (#4793) calls `settle` for the open `needs-human` issues a watch selects: it runs when `mode: live` (a repo setting: every watch of the repo runs the stage), or when the watch sets `sweep_needs_human: true`; `policy` and `human_only` are also read by prodwatch (`loadPolicy`). The parser has one nesting level and no arrays, so lists are comma-separated.
|
|
@@ -1185,7 +1223,7 @@ decomposition:
|
|
|
1185
1223
|
|
|
1186
1224
|
**With `close_parent: false`** (default): unchanged. The parent stays open and is closed by C3 when the last child closes.
|
|
1187
1225
|
|
|
1188
|
-
The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `not
|
|
1226
|
+
The lean (core) path reads the same key (#4337): with `close_parent: true` it closes the parent at decomposition as `completed` (#5152; scope moved to the sub-issues, not dropped), keeps `workflow:decomposed` and lists the sub-issues in the closing comment; dependents still wait on the sub-issues, and a trusted `FORGE:DECOMPOSED:REVOKED` reopens it. With the default `false` the lean parent stays open.
|
|
1189
1227
|
|
|
1190
1228
|
**Limitation**: `--retry` and engine resume refuse closed issues, so a closed decomposed parent cannot be re-run. It has nothing left to run; work on its open children instead.
|
|
1191
1229
|
|
package/docs/OUTCOME-CODES.md
CHANGED
|
@@ -44,6 +44,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
|
|
|
44
44
|
| `gh.unavailable` | `run-issue --repo` could not verify the cwd repo because `gh repo view` kept failing after 3 attempts with backoff (typically a timeout under host load); the core or full run is `deferred` (exit 75, retry in 60s) before any lease, nothing posted | engine CLI | `bin/engine-cli.mjs` |
|
|
45
45
|
| `gh.auth` | A `gh` call in a core run answered HTTP 401 (expired or revoked bot token) and one run of `scripts/refresh-bot-token.sh` plus a retry did not fix it; the run is `deferred` (retry in 120s), never `engine-error`. See `docs/CONFIG.md` "Bot token refresh" | lean | `bin/lean/retry.mjs` |
|
|
46
46
|
| `rate_limited` | Session or usage limit outlasted the pause budget | engine | `err.usageLimit` / `err.resetAtEpochMs` |
|
|
47
|
+
| `backend.auth` | The claude CLI login failed ("Failed to authenticate", expired or invalid login); the lane is `deferred` (retry in 900s) under a per-account hold like a usage limit, never `engine-error`, never relaunched or counted by the breaker (#5168) | lean | `bin/runner.mjs` `err.authFailure` |
|
|
47
48
|
| `engine.error` | Engine or tool failure (missing key/SDK, CLI crash, max turns, unresolved args, runner threw every attempt) | engine | `err.code` |
|
|
48
49
|
| `engine.dco_rewrite` | Core work-on (forge#4997): the DCO sign-off rewrite (`git filter-branch`) of the lane's unpushed commits failed, or the lane's uncommitted changes could not be restored after it (kept on `refs/forgedock/dco-stash`). A mechanical engine step, not a human block: the lane ends `engine-error` (never `needs-human`), which `forgedock orchestrate` relaunches like any engine-error | lean | `bin/lean/signoff.mjs` `ensureSignedOff` |
|
|
49
50
|
| `engine.validate_result_lost` | Core work-on (forge#4708): a validate result recorded before a merge-time CI fix round is missing from the fix session's result file and could not be carried forward; an engine bookkeeping error, not an unimplemented requirement | lean | `bin/lean/lean.mjs` `mergeFixRound` |
|
|
@@ -70,7 +71,7 @@ An absent or unknown code is recorded as `unclassified`, with the raw reason kep
|
|
|
70
71
|
| `lane.branch_missing` | Core work-on: the lane branch (e.g. `milestone/<slug>`) does not exist on origin; fail closed | lean | `git ls-remote --heads origin` |
|
|
71
72
|
| `lane.unresolved` | Core work-on: the lane could not be worked out (milestone or default branch unreadable, empty milestone slug, bad `feature_pattern`); fail closed | lean | Failed `gh` read or config |
|
|
72
73
|
| `invalid` | Issue is invalid or in a divergent closed state | investigate, engine | `INVESTIGATION:INVALID` marker, or live snapshot |
|
|
73
|
-
| `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `
|
|
74
|
+
| `decomposed` | Issue was decomposed into sub-issues. In core work-on, the builder reported `block.kind: "split-required"`; the engine files the sub-issues (`**Parent**` and `**Depends on**` lines) and returns them as `subissues` for `forgedock orchestrate` to run. With `decomposition.close_parent: true` the parent is then closed `completed` (label kept; #5152: its scope moved, it was not dropped) and its dependents wait on the sub-issues | investigate, decompose, lean | `FORGE:DECOMPOSED` or the decomposition assessment; `result.json` `block.subissues` |
|
|
74
75
|
| `builder.blocked` | Core work-on: the builder reported an external action agents cannot take, or a product decision too critical for agents: `block.kind: "human"` with `why` one of `credentials`, `spend`, `promotion`, `customer`, `product-decision`. An upstream issue, a needed split, or prod data that monitoring access can read never count | lean | `result.json` status `blocked` + typed `block` |
|
|
75
76
|
| `builder.block_untyped` | Core work-on: the builder stayed `blocked` without a valid typed reason (missing `block`, a non-human `why` such as prod data, or a dependency that is not open) after one corrective resume | lean | `result.json` status `blocked`, twice |
|
|
76
77
|
| `builder.no_evidence` | Core work-on: the builder reported invalid/already-fixed without file:line evidence, so the issue was not closed | lean | `result.json` with empty `evidence` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "forgedock",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.51.0",
|
|
4
4
|
"description": "Autonomous development pipeline for coding agents. GitHub as institutional memory, with native Claude Code and OpenCode entrypoints.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"forgedock": "bin/forgedock.mjs"
|
package/pool/README.md
CHANGED
|
@@ -10,4 +10,6 @@ ForgeDock's built-in remote compute pool (imported from computeshare; AGPL-3.0-o
|
|
|
10
10
|
|
|
11
11
|
The module boundary is the ForgeDock HTTP sandbox protocol: the router speaks it to the dispatcher and relays to agents over HTTP. Sandboxes receive only short-lived GitHub and Claude access tokens; refresh tokens stay on the router host.
|
|
12
12
|
|
|
13
|
+
Claude login choice (#5165): among the router host's logins with 30+ minutes left, the router hands out the one with the lowest 5-hour use in tokenbaton's `live.<n>` (account `n` = `~/.claude-<n>`), read from router.json `usage_source` (default `$TOKENBATON_HOME`, else `~/.tokenbaton`); validity only breaks ties. An account at or above `usage_ceiling_pct` (default 95) or at 100% weekly use is skipped; when every login is, `/agent/creds/claude` answers a typed 429 (`code: rate_limited`, `reset_at` = the earliest reset, `retry-after`), which the agent passes through and `cs-cred` prints as `usage limit hit; resets at <ISO>` so the lane defers. Without readable usage it falls back to the longest validity and logs a warning. The selected account index and usage are logged, never the token.
|
|
14
|
+
|
|
13
15
|
The `forgedock pool` CLI wiring is tracked separately (#4544).
|
package/pool/image/cs-cred
CHANGED
|
@@ -7,6 +7,13 @@ kind="${1:?kind}"; ttl=240; [ "$kind" = claude ] && ttl=600
|
|
|
7
7
|
cache="/tmp/.cs-cred-$kind"
|
|
8
8
|
if [ -s "$cache" ] && [ $(( $(date +%s) - $(stat -c %Y "$cache") )) -lt $ttl ]; then cat "$cache"; exit 0; fi
|
|
9
9
|
[ -n "${CS_CRED_BASE:-}" ] || { echo "no credential relay (CS_CRED_BASE)" >&2; exit 1; }
|
|
10
|
-
|
|
10
|
+
body=$(mktemp); trap 'rm -f "$body"' EXIT
|
|
11
|
+
code=$(curl -sS --max-time 30 -o "$body" -w '%{http_code}' -H "authorization: Bearer ${CS_CRED_TOKEN}" "$CS_CRED_BASE/$kind") || code=000
|
|
12
|
+
if [ "$code" = 429 ]; then
|
|
13
|
+
# #5165: every router login is at its usage ceiling; this line is what the lane's log tail defers on.
|
|
14
|
+
why=$(jq -r '.reason // empty' "$body" 2>/dev/null || true)
|
|
15
|
+
echo "${why:-usage limit hit; resets at unknown}" >&2; exit 1
|
|
16
|
+
fi
|
|
17
|
+
tok=""; [ "$code" = 200 ] && tok=$(jq -r .token "$body" 2>/dev/null || true)
|
|
11
18
|
[ -n "$tok" ] && [ "$tok" != null ] || { echo "credential fetch failed ($kind)" >&2; exit 1; }
|
|
12
19
|
umask 077; printf '%s' "$tok" > "$cache"; printf '%s' "$tok"
|