@codedology/code-forge 0.15.3 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +1 -1
- package/docs/reference/blocks.json +9 -1
- package/docs/reference/config.md +11 -1
- package/native/README.md +10 -0
- package/native/bin/darwin-arm64/code-forge-native +0 -0
- package/native/bin/darwin-x64/code-forge-native +0 -0
- package/native/bin/linux-arm64/code-forge-native +0 -0
- package/native/bin/linux-x64/code-forge-native +0 -0
- package/package.json +4 -1
- package/schema/code-forge.schema.json +26 -1
- package/skill/references/autopilot.md +6 -1
- package/skill/references/code.md +37 -0
- package/skill/references/plan.md +5 -1
- package/skill/references/review.md +46 -8
- package/skill/scripts/forge +1 -1
- package/src/cli/block.mjs +284 -11
- package/src/cli/doctor.mjs +32 -1
- package/src/cli/help.mjs +1 -0
- package/src/cli/native-status.mjs +13 -0
- package/src/cli/review-file.mjs +121 -14
- package/src/cli/status.mjs +43 -9
- package/src/doctor/index.mjs +11 -0
- package/src/doctor/worker-probe.mjs +1 -1
- package/src/review/agent-waiver.mjs +95 -8
- package/src/review/gate-check.mjs +13 -9
- package/src/review/model-stats.mjs +125 -0
- package/src/review/packet.mjs +54 -3
- package/src/review/reopen.mjs +38 -13
- package/src/review/route.mjs +58 -4
- package/src/session/facts.mjs +19 -7
- package/src/session/plan-check.mjs +4 -1
- package/src/state/block.mjs +39 -16
- package/src/util/native.mjs +131 -0
- package/src/worker/loop.mjs +144 -16
- package/src/worker/queue.mjs +1 -1
- package/src/worker/watch.mjs +16 -5
package/src/state/block.mjs
CHANGED
|
@@ -27,7 +27,7 @@ import { checkWorkerPin, readRun, saveRun, withRunLock, writeSigned } from './ru
|
|
|
27
27
|
import { canonicalJSON, verifyRow, loadKey, signRow } from './signer.mjs';
|
|
28
28
|
import { readAllRows } from '../ledger/write.mjs';
|
|
29
29
|
import * as reviewQueue from '../worker/queue.mjs';
|
|
30
|
-
import { contentHash, repoRootOf } from '../worker/ticket.mjs';
|
|
30
|
+
import { assertInsideRoot, contentHash, repoRootOf } from '../worker/ticket.mjs';
|
|
31
31
|
|
|
32
32
|
/** @typedef {import('./run.mjs').WriteRow} WriteRow */
|
|
33
33
|
|
|
@@ -293,6 +293,16 @@ function activeBlocks(record) {
|
|
|
293
293
|
return Object.fromEntries(Object.entries(record.blocks).filter(([, b]) => b.status === 'open'));
|
|
294
294
|
}
|
|
295
295
|
|
|
296
|
+
/** Earlier claims are authoritative only when their run, block and MAC match.
|
|
297
|
+
* @param {Record<string, any>} record @param {string} id @returns {Promise<string[]>}
|
|
298
|
+
*/
|
|
299
|
+
async function signedClaims(record, id) {
|
|
300
|
+
const key = await loadKey(record.run_id);
|
|
301
|
+
const rows = await readAllRows(record.project);
|
|
302
|
+
return [...new Set(rows.filter((row) => row.event === 'block.claim' && row.run === record.run_id && row.block === id && verifyRow(row, key).ok)
|
|
303
|
+
.map((row) => assertOwned([row.path])[0]))];
|
|
304
|
+
}
|
|
305
|
+
|
|
296
306
|
/**
|
|
297
307
|
* B58: with `review.external.required` set in the run's snapshot, the project's `.code-forge.yml`
|
|
298
308
|
* is the owner's switch past the local per-file review, so no block may own it. These are the
|
|
@@ -387,7 +397,7 @@ function refuseConfigOwner(owned, kinds, guarded) {
|
|
|
387
397
|
* decision?: {question: 'lane', answer: string, source: 'rules', facts: Record<string, unknown>}, decisionId?: string,
|
|
388
398
|
* worktree?: (ctx: {baseSha: string, baseExplicit: boolean, previous: TreeRecord | null}) => Promise<{path: string, make: () => Promise<OpenTree>}>,
|
|
389
399
|
* }} opts
|
|
390
|
-
* @returns {Promise<{block: Record<string, any>, pointer: string | null, worktreeReused: boolean}>} `worktreeReused`:
|
|
400
|
+
* @returns {Promise<{block: Record<string, any>, pointer: string | null, worktreeReused: boolean, keptClaims: number}>} `worktreeReused`:
|
|
391
401
|
* the block took an earlier attempt's worktree as it was.
|
|
392
402
|
*/
|
|
393
403
|
export async function openBlock(opts) {
|
|
@@ -398,12 +408,12 @@ export async function openBlock(opts) {
|
|
|
398
408
|
if (lines !== null && !(Number.isInteger(lines) && lines > 0)) throw new StateError('bad-lines', 'lines forecast must be a positive integer');
|
|
399
409
|
if (opts.kind !== undefined && opts.kind !== 'light' && !BLOCK_KINDS.includes(opts.kind)) throw new StateError('bad-kind', `kind must be one of ${[...BLOCK_KINDS, 'light'].join(', ')}`);
|
|
400
410
|
if (opts.decision && opts.decision.answer !== level) throw new StateError('bad-decision', `the decision answer ${opts.decision.answer} is not the requested level ${level}`);
|
|
401
|
-
const
|
|
411
|
+
const declaredOwned = assertOwned(opts.owned);
|
|
402
412
|
const acceptance = assertAcceptance(opts.acceptance);
|
|
403
413
|
const humanChecks = acceptance.filter((c) => c.kind === 'human_check').map((c) => c.clause);
|
|
404
414
|
const tests = opts.tests === undefined ? null : [...assertOwned(opts.tests, { label: 'test files', code: 'bad-tests' })];
|
|
405
415
|
const after = opts.after === undefined ? null : [...new Set(opts.after)];
|
|
406
|
-
const kind = opts.kind === 'light' ? 'light' : blockKind({ owned, declared: opts.kind });
|
|
416
|
+
const kind = opts.kind === 'light' ? 'light' : blockKind({ owned: declaredOwned, declared: opts.kind });
|
|
407
417
|
const dispatched = dispatchLevel({ lane: level, kind, cfg: opts.cfg });
|
|
408
418
|
const pointer = opts.brief ? briefPointer(opts.brief.path, opts.brief.content) : null;
|
|
409
419
|
|
|
@@ -428,7 +438,7 @@ export async function openBlock(opts) {
|
|
|
428
438
|
/** @type {OpenTree | null} */
|
|
429
439
|
let made = null;
|
|
430
440
|
try {
|
|
431
|
-
const { block, reused } = await mutateRun(runId, null, writeRow, async (record) => {
|
|
441
|
+
const { block, reused, keptClaims } = await mutateRun(runId, null, writeRow, async (record) => {
|
|
432
442
|
if (record.blocks[id]?.status === 'open') throw new StateError('already-open', `block ${id} is already open`);
|
|
433
443
|
if (record.blocks[id]?.status === 'awaiting_human_check') throw new StateError('already-open', `block ${id} is awaiting a human check`);
|
|
434
444
|
if (after !== null) {
|
|
@@ -441,6 +451,8 @@ export async function openBlock(opts) {
|
|
|
441
451
|
}
|
|
442
452
|
// B57: what each exact owned entry is on disk, under the lock (as `block claim`) — a regular
|
|
443
453
|
// file owns only itself, here and in every later check
|
|
454
|
+
const claims = attempt > 1 ? await signedClaims(record, id) : [];
|
|
455
|
+
const owned = [...new Set([...declaredOwned, ...claims])];
|
|
444
456
|
const kinds = ownedKinds(record.workspace, owned);
|
|
445
457
|
refuseConfigOwner(owned, kinds, guarded);
|
|
446
458
|
const prior = previousTree(record);
|
|
@@ -467,7 +479,7 @@ export async function openBlock(opts) {
|
|
|
467
479
|
const floor = dispatched.trigger === 'docs_floor' ? { requested_level: level, trigger: 'docs_floor' } : {};
|
|
468
480
|
const created = tree?.created ? [{ event: 'worktree.created', block: id, path: tree.worktree.path, branch: tree.worktree.branch, base: tree.worktree.base_sha }] : [];
|
|
469
481
|
return {
|
|
470
|
-
result: { block: entry, reused: tree !== null && !tree.created },
|
|
482
|
+
result: { block: entry, reused: tree !== null && !tree.created, keptClaims: claims.length },
|
|
471
483
|
rows: [
|
|
472
484
|
...(opts.decision ? [{ ...opts.decision, event: 'decision', decision_id: opts.decisionId ?? randomUUID(), block: id }] : []),
|
|
473
485
|
...created,
|
|
@@ -475,7 +487,7 @@ export async function openBlock(opts) {
|
|
|
475
487
|
],
|
|
476
488
|
};
|
|
477
489
|
});
|
|
478
|
-
return { block, pointer, worktreeReused: reused };
|
|
490
|
+
return { block, pointer, worktreeReused: reused, keptClaims };
|
|
479
491
|
} catch (err) {
|
|
480
492
|
// B62a: nothing was recorded, so a tree this open made must not outlive it
|
|
481
493
|
if (made?.created && made.rollback) {
|
|
@@ -493,9 +505,12 @@ export async function openBlock(opts) {
|
|
|
493
505
|
|
|
494
506
|
/** @param {{runId: string, id: string, writeRow: WriteRow}} opts */
|
|
495
507
|
export async function attemptBlock({ runId, id, writeRow }) {
|
|
496
|
-
return mutateRun(runId, id, writeRow, (
|
|
508
|
+
return mutateRun(runId, id, writeRow, async (record, block) => {
|
|
509
|
+
const claims = await signedClaims(record, id);
|
|
510
|
+
block.owned_files = [...new Set([...block.owned_files, ...claims])];
|
|
511
|
+
block.owned_kinds = { ...ownedKinds(record.workspace, claims), ...block.owned_kinds };
|
|
497
512
|
block.attempt += 1;
|
|
498
|
-
return { result: block.attempt, rows: [{ event: 'block.attempt', block: id, attempt: block.attempt }] };
|
|
513
|
+
return { result: { attempt: block.attempt, keptClaims: claims.length }, rows: [{ event: 'block.attempt', block: id, attempt: block.attempt }] };
|
|
499
514
|
});
|
|
500
515
|
}
|
|
501
516
|
|
|
@@ -654,25 +669,33 @@ async function rebaseWorktreeBlock({ runId, id, writeRow, acceptChanged, reason,
|
|
|
654
669
|
* B58: with `review.external.required` set, a path that owns the project's config is refused
|
|
655
670
|
* (`config-owned`), as at `block open`. B62a: two blocks that each have their own worktree may
|
|
656
671
|
* own the same path (`blocksOverlap`).
|
|
657
|
-
* @param {{runId: string, id: string, file: string, writeRow: WriteRow}} opts
|
|
672
|
+
* @param {{runId: string, id: string, file: string, writeRow: WriteRow, auto?: boolean}} opts
|
|
658
673
|
*/
|
|
659
|
-
export async function claimPath({ runId, id, file, writeRow }) {
|
|
674
|
+
export async function claimPath({ runId, id, file, writeRow, auto = false }) {
|
|
660
675
|
assertOwned([file]);
|
|
661
676
|
if (!isExactEntry(file)) throw new StateError('bad-owned', `block claim takes one exact path, not a glob (${JSON.stringify(file)})`);
|
|
662
677
|
const guarded = await guardedConfigPaths(await readRun(runId));
|
|
663
|
-
await mutateRun(runId, id, writeRow, (record, block) => {
|
|
664
|
-
const
|
|
678
|
+
await mutateRun(runId, id, writeRow, async (record, block) => {
|
|
679
|
+
const root = auto ? await repoTop(block.worktree?.path ?? record.workspace) : record.workspace;
|
|
680
|
+
if (auto) {
|
|
681
|
+
if (file.split('/').some((part) => ['.git', '.code-forge'].includes(part.toLowerCase()))) throw new StateError('unowned_change', `unowned_change ${file}: cannot auto-claim repository state`);
|
|
682
|
+
try { assertInsideRoot(root, file); }
|
|
683
|
+
catch { throw new StateError('unowned_change', `unowned_change ${file}: cannot auto-claim a path outside the repository or a symlink`); }
|
|
684
|
+
}
|
|
685
|
+
const kinds = { [file]: pathKind(root, file) };
|
|
665
686
|
refuseConfigOwner([file], kinds, guarded);
|
|
687
|
+
const prefix = auto ? path.relative(await repoTop(record.workspace), realpathSync(record.workspace)).split(path.sep).join('/') : '';
|
|
666
688
|
for (const [otherId, other] of Object.entries(activeBlocks(record))) {
|
|
667
|
-
const pair = otherId === id ? null : blocksOverlap({ owned_files: [file], owned_kinds: kinds, worktree: block.worktree ?? null }, other);
|
|
668
|
-
|
|
689
|
+
const pair = otherId === id ? null : blocksOverlap({ owned_files: [file], owned_kinds: kinds, worktree: auto ? null : block.worktree ?? null }, other);
|
|
690
|
+
const relativeOwner = auto && otherId !== id && prefix !== '' && file.startsWith(`${prefix}/`) && ownsFile(other.owned_files, file.slice(prefix.length + 1), other.owned_kinds);
|
|
691
|
+
if (pair || relativeOwner) throw new StateError(auto ? 'unowned_change' : 'overlap', `${auto ? 'unowned_change ' : ''}${file} overlaps ${pair?.[1] ?? file} owned by open block ${otherId}`);
|
|
669
692
|
}
|
|
670
693
|
if (!block.owned_files.includes(file)) {
|
|
671
694
|
block.owned_files.push(file);
|
|
672
695
|
block.owned_kinds = { ...block.owned_kinds, ...kinds };
|
|
673
696
|
}
|
|
674
697
|
record.orphans = record.orphans.filter((/** @type {string} */ o) => !ownsFile([file], o, kinds));
|
|
675
|
-
return { result: undefined, rows: [{ event: 'block.claim', block: id, path: file }] };
|
|
698
|
+
return { result: undefined, rows: [{ event: 'block.claim', block: id, path: file, ...(auto ? { auto: true, reason: 'auto-claim: changed outside owned files' } : {}) }] };
|
|
676
699
|
});
|
|
677
700
|
}
|
|
678
701
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants, statSync } from 'node:fs';
|
|
3
|
+
import net from 'node:net';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import pkg from '../../package.json' with { type: 'json' };
|
|
8
|
+
import { exec } from './exec.mjs';
|
|
9
|
+
|
|
10
|
+
/** @returns {string | null} the optional executable shipped for this host. */
|
|
11
|
+
export function nativePath() {
|
|
12
|
+
if (process.env.CODE_FORGE_NATIVE === '0') return null;
|
|
13
|
+
const binary = fileURLToPath(new URL(`../../native/bin/${process.platform}-${process.arch}/code-forge-native`, import.meta.url));
|
|
14
|
+
try {
|
|
15
|
+
if (!statSync(binary).isFile()) return null;
|
|
16
|
+
accessSync(binary, constants.X_OK);
|
|
17
|
+
return binary;
|
|
18
|
+
} catch { return null; }
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {string[]} args
|
|
23
|
+
* @returns {Promise<import('./exec.mjs').ExecResult | null>} null requests the Node fallback.
|
|
24
|
+
*/
|
|
25
|
+
export async function runNative(args) {
|
|
26
|
+
const binary = nativePath();
|
|
27
|
+
return binary ? exec([binary, ...args], { timeoutMs: 30_000 }) : null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** @param {Record<string, any> | null} [config] @param {NodeJS.ProcessEnv} [env] */
|
|
31
|
+
export function daemonEnabled(config, env = process.env) {
|
|
32
|
+
return env.CODE_FORGE_NATIVE !== '0' && config?.native?.daemon !== false;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* One connection per call. Ping negotiates the package version before sending the method;
|
|
37
|
+
* no socket is opened when disabled. Protocol and transport failures request Node fallback.
|
|
38
|
+
* @param {string} run
|
|
39
|
+
* @param {{method: string, block?: string, max_ms?: number}} request
|
|
40
|
+
* @param {{config?: Record<string, any> | null, timeoutMs?: number}} [opts]
|
|
41
|
+
* @returns {Promise<Record<string, any> | null>}
|
|
42
|
+
*/
|
|
43
|
+
export async function daemonRequest(run, request, { config, timeoutMs = request.method === 'wait-block' ? (request.max_ms ?? 0) + 2500 : 6500 } = {}) {
|
|
44
|
+
if (!daemonEnabled(config) || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(run)) return null;
|
|
45
|
+
return new Promise((resolve) => {
|
|
46
|
+
let settled = false;
|
|
47
|
+
let negotiated = false;
|
|
48
|
+
let buffer = '';
|
|
49
|
+
const socket = net.createConnection(path.join(os.homedir(), '.code-forge', 'runs', run, 'native.sock'));
|
|
50
|
+
socket.setEncoding('utf8');
|
|
51
|
+
const finish = (/** @type {Record<string, any> | null} */ value) => {
|
|
52
|
+
if (settled) return;
|
|
53
|
+
settled = true;
|
|
54
|
+
clearTimeout(timer);
|
|
55
|
+
socket.destroy();
|
|
56
|
+
resolve(value);
|
|
57
|
+
};
|
|
58
|
+
let timer = setTimeout(() => finish(null), Math.min(2000, timeoutMs));
|
|
59
|
+
const deadline = (/** @type {number} */ ms) => {
|
|
60
|
+
clearTimeout(timer);
|
|
61
|
+
timer = setTimeout(() => finish(null), ms);
|
|
62
|
+
};
|
|
63
|
+
socket.on('error', () => finish(null));
|
|
64
|
+
socket.on('close', () => finish(null));
|
|
65
|
+
socket.on('connect', () => {
|
|
66
|
+
deadline(Math.min(2000, timeoutMs));
|
|
67
|
+
socket.write('{"method":"ping"}\n');
|
|
68
|
+
});
|
|
69
|
+
socket.on('data', (chunk) => {
|
|
70
|
+
buffer += chunk;
|
|
71
|
+
if (buffer.length > 8 * 1024 * 1024) return finish(null);
|
|
72
|
+
const end = buffer.indexOf('\n');
|
|
73
|
+
if (end < 0) return;
|
|
74
|
+
let answer;
|
|
75
|
+
try { answer = JSON.parse(buffer.slice(0, end)); } catch { return finish(null); }
|
|
76
|
+
buffer = buffer.slice(end + 1);
|
|
77
|
+
if (!answer || typeof answer !== 'object' || Array.isArray(answer) || answer.ok === false) return finish(null);
|
|
78
|
+
if (!negotiated) {
|
|
79
|
+
if (answer.ok !== true || answer.version !== pkg.version || answer.run !== run) return finish(null);
|
|
80
|
+
if (request.method === 'ping') return finish(answer);
|
|
81
|
+
negotiated = true;
|
|
82
|
+
deadline(timeoutMs);
|
|
83
|
+
socket.write(`${JSON.stringify(request)}\n`);
|
|
84
|
+
} else {
|
|
85
|
+
const { stats, ...result } = answer;
|
|
86
|
+
if (request.method === 'wait-block' && (result.block !== request.block || !Array.isArray(result.files) || typeof result.done !== 'boolean'
|
|
87
|
+
|| !result.files.every((/** @type {any} */ file) => file && typeof file.file === 'string' && typeof file.status === 'string'))) return finish(null);
|
|
88
|
+
if (request.method === 'status' && (!Array.isArray(result.runs) || result.runs.length !== 1 || result.runs[0]?.run_id !== run || !Array.isArray(result.runs[0]?.blocks))) return finish(null);
|
|
89
|
+
finish(result);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}).catch(() => null);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The daemon serializes startup with its own OS lock. A loser exits harmlessly;
|
|
97
|
+
* only this child is stopped, never a daemon another worker already owns.
|
|
98
|
+
* @param {string} run @param {Record<string, any> | null} config
|
|
99
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
100
|
+
* @returns {Promise<() => Promise<void>>}
|
|
101
|
+
*/
|
|
102
|
+
export async function startNativeDaemon(run, config, env = process.env) {
|
|
103
|
+
const noop = async () => {};
|
|
104
|
+
if (!daemonEnabled(config, env)) return noop;
|
|
105
|
+
const binary = nativePath();
|
|
106
|
+
if (!binary || await daemonRequest(run, { method: 'ping' }, { config })) return noop;
|
|
107
|
+
let child;
|
|
108
|
+
try { child = spawn(binary, ['serve', '--home', os.homedir(), '--run', run], { stdio: 'ignore', env }); }
|
|
109
|
+
catch { return noop; }
|
|
110
|
+
let ended = false;
|
|
111
|
+
const exited = new Promise((resolve) => {
|
|
112
|
+
child.once('error', () => { ended = true; resolve(undefined); });
|
|
113
|
+
child.once('exit', () => { ended = true; resolve(undefined); });
|
|
114
|
+
});
|
|
115
|
+
for (let i = 0; i < 100 && !ended; i++) {
|
|
116
|
+
if (await daemonRequest(run, { method: 'ping' }, { config, timeoutMs: 100 })) break;
|
|
117
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
118
|
+
}
|
|
119
|
+
/** @type {Promise<void> | undefined} */
|
|
120
|
+
let stopping;
|
|
121
|
+
return () => {
|
|
122
|
+
stopping ??= (async () => {
|
|
123
|
+
if (ended) return;
|
|
124
|
+
child.kill('SIGTERM');
|
|
125
|
+
const timer = setTimeout(() => child.kill('SIGKILL'), 2000);
|
|
126
|
+
timer.unref();
|
|
127
|
+
try { await exited; } finally { clearTimeout(timer); }
|
|
128
|
+
})();
|
|
129
|
+
return stopping;
|
|
130
|
+
};
|
|
131
|
+
}
|
package/src/worker/loop.mjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* restarted worker boots on the snapshot, not on whatever the file says now. A record from before
|
|
21
21
|
* snapshots (or none) keeps the config the worker booted with.
|
|
22
22
|
*
|
|
23
|
-
* Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default
|
|
23
|
+
* Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default 6, 1–16) tickets at
|
|
24
24
|
* once, oldest first (or fewest running per block with `review.fair_share`); when one ends the
|
|
25
25
|
* next pending one starts. Two tickets for the same (block, file) never overlap (a file lock;
|
|
26
26
|
* the later one stays pending without taking a slot),
|
|
@@ -45,13 +45,14 @@ import { loadProjectConfig } from '../config/load.mjs';
|
|
|
45
45
|
import { reviewWorkers } from '../config/validate.mjs';
|
|
46
46
|
import { cacheHoursFrom, createDefaultKeyStore, resolveKey } from '../keys/store.mjs';
|
|
47
47
|
import { appendRow, readAllRows } from '../ledger/write.mjs';
|
|
48
|
-
import { spawnSession } from '../session/spawn.mjs';
|
|
48
|
+
import { providerLimit, spawnSession } from '../session/spawn.mjs';
|
|
49
49
|
import { snapshotFor } from '../state/config-snapshot.mjs';
|
|
50
50
|
import { runRecordPath } from '../state/paths.mjs';
|
|
51
51
|
import { processStartTime, readRun, reattachWorker } from '../state/run.mjs';
|
|
52
52
|
import { loadKey, signRow } from '../state/signer.mjs';
|
|
53
53
|
import { logWarning } from '../util/error-log.mjs';
|
|
54
54
|
import { withLock, tryLock } from '../util/file-locks.mjs';
|
|
55
|
+
import { startNativeDaemon } from '../util/native.mjs';
|
|
55
56
|
import { redact, writeSafe } from '../util/redact.mjs';
|
|
56
57
|
import { sweepProcesses } from '../util/reaper.mjs';
|
|
57
58
|
import { runRoot, setRunRoot } from '../util/tmp.mjs';
|
|
@@ -69,7 +70,8 @@ export const DEFAULT_POLL_MS = 250;
|
|
|
69
70
|
export const DEFAULT_SESSION_TIMEOUT_S = 300;
|
|
70
71
|
|
|
71
72
|
/** B42: `review.parallel_tickets` when the config does not set a valid one. */
|
|
72
|
-
export const DEFAULT_PARALLEL_TICKETS =
|
|
73
|
+
export const DEFAULT_PARALLEL_TICKETS = 6;
|
|
74
|
+
export const DEFAULT_PROVIDER_PARALLEL = 4;
|
|
73
75
|
|
|
74
76
|
/** B48: the fixed text of a failed autopilot expiry check in a drain (stderr and the warning log). */
|
|
75
77
|
export const EXPIRY_CHECK_FAILED = 'the autopilot expiry check failed in a worker drain; it runs again at the next drain';
|
|
@@ -281,6 +283,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
281
283
|
const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
|
|
282
284
|
const heartbeatMs = Number.isInteger(opts.heartbeatMs) && /** @type {number} */ (opts.heartbeatMs) >= MIN_HEARTBEAT_MS ? /** @type {number} */ (opts.heartbeatMs) : HEARTBEAT_MS;
|
|
283
285
|
let stopping = false;
|
|
286
|
+
let stopDaemon = async () => {};
|
|
287
|
+
let daemonStarted = false;
|
|
284
288
|
/** @type {ReturnType<typeof setInterval> | null} */
|
|
285
289
|
let heartbeat = null;
|
|
286
290
|
/** @type {ReturnType<typeof announceWorker> | null} */
|
|
@@ -291,6 +295,63 @@ export async function createWorker(opts, deps = {}) {
|
|
|
291
295
|
let poolWake = null;
|
|
292
296
|
let activity = 0;
|
|
293
297
|
|
|
298
|
+
// Sessions waiting for a provider retain their ticket's pool slot. Resolve the actual provider
|
|
299
|
+
// at spawn time, after routing and second-level overrides; admission hints never bypass caps.
|
|
300
|
+
let ticketLimit = parallelTickets(cfg);
|
|
301
|
+
let providerParallel = DEFAULT_PROVIDER_PARALLEL;
|
|
302
|
+
/** @type {Map<string, {active: number, pending: number}>} */
|
|
303
|
+
const ticketSessions = new Map();
|
|
304
|
+
/** @type {Map<string, number>} */
|
|
305
|
+
const providerActive = new Map();
|
|
306
|
+
/** @type {Array<{id: string, provider: string, start: () => void}>} */
|
|
307
|
+
const sessionQueue = [];
|
|
308
|
+
|
|
309
|
+
function dispatchSessions() {
|
|
310
|
+
for (let i = 0; i < sessionQueue.length;) {
|
|
311
|
+
const next = sessionQueue[i];
|
|
312
|
+
const state = ticketSessions.get(next.id);
|
|
313
|
+
const liveTickets = [...ticketSessions.values()].filter((s) => s.active > 0).length;
|
|
314
|
+
if ((providerActive.get(next.provider) ?? 0) >= providerParallel || (state.active === 0 && liveTickets >= ticketLimit)) {
|
|
315
|
+
i += 1;
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
sessionQueue.splice(i, 1);
|
|
319
|
+
state.pending -= 1;
|
|
320
|
+
state.active += 1;
|
|
321
|
+
providerActive.set(next.provider, (providerActive.get(next.provider) ?? 0) + 1);
|
|
322
|
+
next.start();
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** @param {string} id @param {Parameters<typeof spawnSession>[0]} opts @returns {ReturnType<typeof spawnSession>} */
|
|
327
|
+
function spawnPooled(id, opts) {
|
|
328
|
+
// Match the spawner's per-level override and top-level fallback; validation stays there.
|
|
329
|
+
const provider = opts.cfg?.levels?.[opts.level]?.provider ?? opts.cfg?.provider ?? 'unresolved';
|
|
330
|
+
const state = ticketSessions.get(id) ?? { active: 0, pending: 0 };
|
|
331
|
+
ticketSessions.set(id, state);
|
|
332
|
+
state.pending += 1;
|
|
333
|
+
// The spawner's existing per-attempt semaphores also cover fallback providers. Retain
|
|
334
|
+
// stricter provider_concurrency settings while applying this cap to fallback attempts.
|
|
335
|
+
const providerConcurrency = Object.fromEntries(['anthropic', 'openai', 'xai'].map((p) => [p, Math.min(providerParallel, providerLimit(opts.cfg, p))]));
|
|
336
|
+
const needsCap = Object.entries(providerConcurrency).some(([p, limit]) => limit < providerLimit(opts.cfg, p));
|
|
337
|
+
const sessionCfg = needsCap ? { ...opts.cfg, review: { ...opts.cfg.review, provider_concurrency: providerConcurrency } } : opts.cfg;
|
|
338
|
+
return new Promise((resolve, reject) => {
|
|
339
|
+
sessionQueue.push({ id, provider, start: () => {
|
|
340
|
+
Promise.resolve().then(() => spawn({ ...opts, cfg: sessionCfg },
|
|
341
|
+
{ env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
|
|
342
|
+
)).then(resolve, reject).finally(() => {
|
|
343
|
+
state.active -= 1;
|
|
344
|
+
providerActive.set(provider, providerActive.get(provider) - 1);
|
|
345
|
+
if (state.active === 0 && state.pending === 0) ticketSessions.delete(id);
|
|
346
|
+
dispatchSessions();
|
|
347
|
+
poolWake?.();
|
|
348
|
+
});
|
|
349
|
+
} });
|
|
350
|
+
dispatchSessions();
|
|
351
|
+
poolWake?.();
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
294
355
|
/** @param {Record<string, any>} row */
|
|
295
356
|
const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
|
|
296
357
|
// Shared across direct ticks and the scheduler, so running tickets are never superseded.
|
|
@@ -305,6 +366,11 @@ export async function createWorker(opts, deps = {}) {
|
|
|
305
366
|
* @throws {Error} `config-snapshot` — a record that cannot be read, or a missing/altered snapshot.
|
|
306
367
|
*/
|
|
307
368
|
async function configFor(id) {
|
|
369
|
+
return snapshotFor(await configRecord(), id)?.config ?? cfg;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Read once for pool admission hints; ticket processing still re-reads its own snapshot. */
|
|
373
|
+
async function configRecord() {
|
|
308
374
|
// The Jev key and the run root were settled at boot from `cfg`; a reload can never change
|
|
309
375
|
// them, because `keys`, `system1.key` and `tmp.root` are fixed keys `run reload` refuses.
|
|
310
376
|
let record;
|
|
@@ -313,13 +379,13 @@ export async function createWorker(opts, deps = {}) {
|
|
|
313
379
|
record = await loadRun(runId);
|
|
314
380
|
break;
|
|
315
381
|
} catch (err) {
|
|
316
|
-
if (/** @type {any} */ (err)?.code === 'no-run') return
|
|
382
|
+
if (/** @type {any} */ (err)?.code === 'no-run') return null;
|
|
317
383
|
// a transient read error (e.g. the record caught mid-replace) gets ONE retry
|
|
318
384
|
if (attempt >= 2) throw new Error('config-snapshot');
|
|
319
385
|
await new Promise((resolve) => setTimeout(resolve, CONFIG_RETRY_MS));
|
|
320
386
|
}
|
|
321
387
|
}
|
|
322
|
-
return
|
|
388
|
+
return record;
|
|
323
389
|
}
|
|
324
390
|
|
|
325
391
|
/**
|
|
@@ -338,7 +404,9 @@ export async function createWorker(opts, deps = {}) {
|
|
|
338
404
|
// A route list that cannot be used is `unavailable` — never a review on a model it did not name.
|
|
339
405
|
let routed;
|
|
340
406
|
try {
|
|
341
|
-
routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS }
|
|
407
|
+
routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS },
|
|
408
|
+
snapshotCfg.review?.adaptive?.enabled === false || !snapshotCfg.levels?.L2?.model
|
|
409
|
+
? undefined : { rows: await readRows(), ticketId: id, runId });
|
|
342
410
|
} catch (err) {
|
|
343
411
|
// only a route list that cannot be used is `route-invalid`; anything else is a real failure
|
|
344
412
|
// and goes to runOne (`worker-error`), named as a routing-step failure with its own text
|
|
@@ -382,9 +450,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
382
450
|
// `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
|
|
383
451
|
// over the ticket's; everything else the worker pins.
|
|
384
452
|
spawn: (sessionOpts) =>
|
|
385
|
-
|
|
453
|
+
spawnPooled(id,
|
|
386
454
|
/** @type {any} */ ({ cfg: tcfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
|
|
387
|
-
{ env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
|
|
388
455
|
),
|
|
389
456
|
writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
|
|
390
457
|
writeBlockRow: (row) => ledger({ ...row, block: ticket.block }),
|
|
@@ -559,16 +626,21 @@ export async function createWorker(opts, deps = {}) {
|
|
|
559
626
|
/**
|
|
560
627
|
* Pool settings from the snapshot in force, else the boot config. A run reload applies at the
|
|
561
628
|
* next drain; anything but an integer 1–16 uses the default size.
|
|
562
|
-
* @returns {Promise<{limit: number, fairShare: boolean}>}
|
|
629
|
+
* @returns {Promise<{limit: number, fairShare: boolean, record: Record<string, any> | null}>}
|
|
563
630
|
*/
|
|
564
631
|
async function poolSettings() {
|
|
565
632
|
let current = cfg;
|
|
633
|
+
let record = null;
|
|
566
634
|
try {
|
|
567
|
-
|
|
635
|
+
record = await configRecord();
|
|
636
|
+
current = snapshotFor(record, '')?.config ?? cfg;
|
|
568
637
|
} catch {
|
|
569
638
|
// an unreadable record: the boot settings (each ticket still fails closed on its own)
|
|
570
639
|
}
|
|
571
|
-
|
|
640
|
+
ticketLimit = parallelTickets(current);
|
|
641
|
+
const cap = current?.review?.provider_parallel;
|
|
642
|
+
providerParallel = Number.isInteger(cap) && cap >= 1 && cap <= MAX_PARALLEL_TICKETS ? cap : DEFAULT_PROVIDER_PARALLEL;
|
|
643
|
+
return { limit: parallelTickets(current), fairShare: current?.review?.fair_share === true, record };
|
|
572
644
|
}
|
|
573
645
|
|
|
574
646
|
/**
|
|
@@ -598,6 +670,27 @@ export async function createWorker(opts, deps = {}) {
|
|
|
598
670
|
}
|
|
599
671
|
}
|
|
600
672
|
|
|
673
|
+
/**
|
|
674
|
+
* The initial reviewer's provider is an admission hint, including snapshot and size routing.
|
|
675
|
+
* Unknown or invalid hints still enter the bounded pool so the normal checks can refuse them.
|
|
676
|
+
* Later sessions may use another provider; `spawnPooled` enforces their actual session caps.
|
|
677
|
+
* @param {string} id @param {Record<string, any> | null} record @returns {Promise<string | null>}
|
|
678
|
+
*/
|
|
679
|
+
async function providerHint(id, record) {
|
|
680
|
+
try {
|
|
681
|
+
let current = snapshotFor(record, id)?.config ?? cfg;
|
|
682
|
+
if (current?.review?.routes?.length) {
|
|
683
|
+
const ticket = loadTicket(repoRoot, id);
|
|
684
|
+
const root = await contentRootFor(runId, ticket.block, repoRoot);
|
|
685
|
+
const bytes = readRegularFileNoFollow(root, ticket.file);
|
|
686
|
+
current = routeTicket(current, { lines: countLines(bytes), deleted: bytes === null, timeoutS: DEFAULT_SESSION_TIMEOUT_S }).cfg;
|
|
687
|
+
}
|
|
688
|
+
return current?.levels?.L2?.provider ?? current?.provider ?? null;
|
|
689
|
+
} catch {
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
601
694
|
/**
|
|
602
695
|
* Serve every pending ticket (B42): up to `review.parallel_tickets` at once, oldest first; with
|
|
603
696
|
* fair share, each slot goes to the block with fewest running tickets, ties oldest first. When
|
|
@@ -625,6 +718,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
625
718
|
}
|
|
626
719
|
/** @type {number | null} */
|
|
627
720
|
let limit = null;
|
|
721
|
+
/** @type {Record<string, any> | null} */
|
|
722
|
+
let record = null;
|
|
628
723
|
let fairShare = false;
|
|
629
724
|
let done = 0;
|
|
630
725
|
/** @type {Map<string, Promise<void>>} */
|
|
@@ -633,6 +728,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
633
728
|
const busy = new Set();
|
|
634
729
|
/** @type {Map<string, number>} running tickets per block */
|
|
635
730
|
const runningByBlock = new Map();
|
|
731
|
+
/** @type {Map<string, number>} reserve provider capacity while admitted tickets prepare */
|
|
732
|
+
const admittedByProvider = new Map();
|
|
636
733
|
/** @type {Set<string>} */
|
|
637
734
|
const started = new Set();
|
|
638
735
|
for (;;) {
|
|
@@ -660,10 +757,12 @@ export async function createWorker(opts, deps = {}) {
|
|
|
660
757
|
}
|
|
661
758
|
const pending = pendingTickets(repoRoot).filter((id) => !poisoned.has(id) && !started.has(id));
|
|
662
759
|
if (pending.length > 0 || running.size > 0) activity += 1;
|
|
663
|
-
if (pending.length > 0 && limit === null) ({ limit, fairShare } = await poolSettings());
|
|
760
|
+
if (pending.length > 0 && limit === null) ({ limit, fairShare, record } = await poolSettings());
|
|
664
761
|
// Cache even unreadable hints for this tick: ranking never reads a ticket twice.
|
|
665
762
|
/** @type {Map<string, ReturnType<typeof fileKeyOf>>} */
|
|
666
763
|
const hints = new Map();
|
|
764
|
+
/** @type {Map<string, string | null>} */
|
|
765
|
+
const providers = new Map();
|
|
667
766
|
while (pending.length > 0 && running.size < limit && !stopping) {
|
|
668
767
|
let next = -1;
|
|
669
768
|
let fewest = Infinity;
|
|
@@ -672,6 +771,9 @@ export async function createWorker(opts, deps = {}) {
|
|
|
672
771
|
if (!hints.has(id)) hints.set(id, fileKeyOf(id));
|
|
673
772
|
const file = hints.get(id);
|
|
674
773
|
if (file && busy.has(file.key)) continue; // waits without a slot
|
|
774
|
+
if (!providers.has(id)) providers.set(id, await providerHint(id, record));
|
|
775
|
+
const provider = providers.get(id);
|
|
776
|
+
if (provider && Math.max(admittedByProvider.get(provider) ?? 0, providerActive.get(provider) ?? 0) >= providerParallel) continue;
|
|
675
777
|
const count = fairShare && file ? runningByBlock.get(file.block) ?? 0 : 0;
|
|
676
778
|
if (count < fewest) {
|
|
677
779
|
next = i;
|
|
@@ -691,10 +793,12 @@ export async function createWorker(opts, deps = {}) {
|
|
|
691
793
|
}
|
|
692
794
|
const file = hints.get(id);
|
|
693
795
|
const fileKey = file?.key ?? null;
|
|
796
|
+
const provider = providers.get(id);
|
|
694
797
|
started.add(id);
|
|
695
798
|
activeTickets.add(id);
|
|
696
799
|
if (fileKey !== null) busy.add(fileKey);
|
|
697
800
|
if (file) runningByBlock.set(file.block, (runningByBlock.get(file.block) ?? 0) + 1);
|
|
801
|
+
if (provider) admittedByProvider.set(provider, (admittedByProvider.get(provider) ?? 0) + 1);
|
|
698
802
|
const slot = runOne(id)
|
|
699
803
|
.then((n) => {
|
|
700
804
|
done += n;
|
|
@@ -707,6 +811,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
707
811
|
fileKeys.delete(id);
|
|
708
812
|
if (fileKey !== null) busy.delete(fileKey);
|
|
709
813
|
if (file) runningByBlock.set(file.block, runningByBlock.get(file.block) - 1);
|
|
814
|
+
if (provider) admittedByProvider.set(provider, admittedByProvider.get(provider) - 1);
|
|
710
815
|
});
|
|
711
816
|
running.set(id, slot);
|
|
712
817
|
}
|
|
@@ -769,14 +874,22 @@ export async function createWorker(opts, deps = {}) {
|
|
|
769
874
|
heartbeat = null;
|
|
770
875
|
}
|
|
771
876
|
|
|
877
|
+
async function startDaemon() {
|
|
878
|
+
if (daemonStarted) return;
|
|
879
|
+
daemonStarted = true;
|
|
880
|
+
stopDaemon = await startNativeDaemon(runId, cfg, env);
|
|
881
|
+
if (stopping) await stopDaemon();
|
|
882
|
+
}
|
|
883
|
+
|
|
772
884
|
/** Serve the queue until `stop()`. */
|
|
773
885
|
async function run() {
|
|
774
|
-
announce();
|
|
775
886
|
let interval = pollMs;
|
|
776
887
|
let polled = false;
|
|
777
888
|
/** @type {import('node:fs').FSWatcher | null} */
|
|
778
889
|
let queueWatch = null;
|
|
779
890
|
try {
|
|
891
|
+
await startDaemon();
|
|
892
|
+
announce();
|
|
780
893
|
// Enqueue publishes a ticket by rename. Wake even during the longest idle back-off;
|
|
781
894
|
// heartbeat and temporary files do not wake an idle worker.
|
|
782
895
|
try {
|
|
@@ -818,37 +931,42 @@ export async function createWorker(opts, deps = {}) {
|
|
|
818
931
|
} finally {
|
|
819
932
|
queueWatch?.close();
|
|
820
933
|
release();
|
|
934
|
+
await stopDaemon();
|
|
821
935
|
}
|
|
822
936
|
}
|
|
823
937
|
|
|
824
938
|
/** Drain the queue once while announced, then release (`worker --once`). */
|
|
825
939
|
async function once() {
|
|
826
|
-
announce();
|
|
827
940
|
try {
|
|
941
|
+
await startDaemon();
|
|
942
|
+
announce();
|
|
828
943
|
const done = await drain();
|
|
829
944
|
if (done > 0) await cleanOrphans();
|
|
830
945
|
return done;
|
|
831
946
|
} finally {
|
|
832
947
|
stop();
|
|
833
948
|
release();
|
|
949
|
+
await stopDaemon();
|
|
834
950
|
}
|
|
835
951
|
}
|
|
836
952
|
|
|
837
953
|
/** Stop taking tickets; the running ones are awaited (a killed one is abandoned, see `processTicket`). */
|
|
838
954
|
function stop() {
|
|
839
955
|
stopping = true;
|
|
956
|
+
void stopDaemon();
|
|
840
957
|
wake?.();
|
|
841
958
|
poolWake?.();
|
|
842
959
|
}
|
|
843
960
|
|
|
844
961
|
/** Withdraw this process's announcement (`worker.json`) if it is still ours. */
|
|
845
962
|
function release() {
|
|
963
|
+
void stopDaemon();
|
|
846
964
|
stopBeating();
|
|
847
965
|
if (self !== null) retractWorker(repoRoot, self.pid, self.start_time);
|
|
848
966
|
self = null;
|
|
849
967
|
}
|
|
850
968
|
|
|
851
|
-
return { processTicket, drain, run, once, stop, release, announce, watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
|
|
969
|
+
return { processTicket, drain, run, once, stop, release, announce, startDaemon, stopDaemon: () => stopDaemon(), watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
|
|
852
970
|
}
|
|
853
971
|
|
|
854
972
|
/**
|
|
@@ -890,7 +1008,17 @@ export async function bootWorker(opts, deps = {}) {
|
|
|
890
1008
|
if (entries.some((entry) => entry.live && !entry.worker) || (legacy && legacy.run !== opts.runId) || entries.some((entry) => entry.live && entry.worker?.run !== opts.runId) || count >= limit) {
|
|
891
1009
|
throw new WorkerError('worker-running', `review.workers limit ${limit}: a live or unreadable worker entry already serves this queue`);
|
|
892
1010
|
}
|
|
893
|
-
worker.
|
|
1011
|
+
// Startup must finish before worker.json signals readiness: an owner may
|
|
1012
|
+
// remove HOME immediately after that announcement to stop an orphan worker.
|
|
1013
|
+
try {
|
|
1014
|
+
await worker.startDaemon();
|
|
1015
|
+
worker.announce();
|
|
1016
|
+
} catch (error) {
|
|
1017
|
+
worker.stop();
|
|
1018
|
+
worker.release();
|
|
1019
|
+
await worker.stopDaemon();
|
|
1020
|
+
throw error;
|
|
1021
|
+
}
|
|
894
1022
|
});
|
|
895
1023
|
return { ...worker, repoRoot, runRootDir };
|
|
896
1024
|
}
|
package/src/worker/queue.mjs
CHANGED
|
@@ -589,7 +589,7 @@ export function readTicket(repoRoot, id) {
|
|
|
589
589
|
const t = readJSON(path.join(queueDir(repoRoot), `${assertTicketId(id)}.json`));
|
|
590
590
|
if (!t || typeof t !== 'object') throw new WorkerError('bad-ticket', `ticket ${id} is not readable JSON`);
|
|
591
591
|
assertRowPath(t.file);
|
|
592
|
-
if (t.tree
|
|
592
|
+
if (t.tree != null && (typeof t.tree !== 'string' || !path.isAbsolute(t.tree))) throw new WorkerError('bad-ticket', `ticket ${id} names a tree that is not an absolute path`);
|
|
593
593
|
if (typeof t.run !== 'string' || typeof t.block !== 'string' || typeof t.content_hash !== 'string') {
|
|
594
594
|
throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
|
|
595
595
|
}
|