@codedology/code-forge 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +5 -1
- package/docs/README.md +2 -0
- package/docs/reference/blocks.json +162 -1
- package/docs/reference/config.md +38 -15
- package/native/README.md +10 -0
- package/native/bin/darwin-arm64/code-forge-native +0 -0
- package/native/bin/darwin-x64/code-forge-native +0 -0
- package/native/bin/linux-arm64/code-forge-native +0 -0
- package/native/bin/linux-x64/code-forge-native +0 -0
- package/package.json +4 -1
- package/schema/code-forge.schema.json +83 -3
- package/schema/outcome.schema.json +45 -0
- package/schema/pipeline.schema.json +30 -0
- package/scripts/first-use.mjs +8 -0
- package/scripts/gen-config-doc.mjs +1 -1
- package/skill/references/autopilot.md +6 -1
- package/skill/references/code.md +52 -0
- package/skill/references/ledger.md +13 -1
- package/skill/references/plan.md +5 -1
- package/skill/references/review.md +46 -8
- package/skill/scripts/forge +1 -1
- package/src/cli/block.mjs +427 -38
- package/src/cli/context.mjs +17 -7
- package/src/cli/doctor.mjs +32 -1
- package/src/cli/gates.mjs +76 -10
- package/src/cli/help.mjs +5 -0
- package/src/cli/native-status.mjs +13 -0
- package/src/cli/outcome.mjs +87 -0
- package/src/cli/pipeline.mjs +61 -0
- package/src/cli/proof.mjs +10 -7
- package/src/cli/report.mjs +53 -6
- package/src/cli/review-file.mjs +278 -41
- package/src/cli/run.mjs +3 -2
- package/src/cli/status.mjs +43 -9
- package/src/cli/sweep.mjs +28 -0
- package/src/cli/validate.mjs +6 -0
- package/src/config/validate.mjs +83 -1
- package/src/context/pack.mjs +53 -3
- package/src/doctor/index.mjs +11 -0
- package/src/doctor/local.mjs +22 -4
- package/src/doctor/worker-probe.mjs +1 -1
- package/src/engines/builders/codex.mjs +27 -7
- package/src/engines/builders/grok.mjs +11 -5
- package/src/engines/codex-home.mjs +18 -15
- package/src/engines/sentinel.mjs +11 -0
- package/src/gates/detect.mjs +6 -3
- package/src/gates/run.mjs +295 -27
- package/src/ledger/export.mjs +1 -1
- package/src/ledger/report.mjs +143 -5
- package/src/outcome/outcome.mjs +108 -0
- package/src/pipeline/pipeline.mjs +112 -0
- package/src/proof/red-green.mjs +41 -13
- package/src/proof/runners.mjs +28 -4
- package/src/proof/worktree.mjs +25 -15
- package/src/review/agent-waiver.mjs +113 -12
- package/src/review/engine.mjs +25 -5
- package/src/review/gate-check.mjs +52 -10
- package/src/review/lenses/full.md +1 -0
- package/src/review/lenses/quick.md +1 -0
- package/src/review/model-stats.mjs +125 -0
- package/src/review/packet.mjs +88 -9
- package/src/review/related.mjs +112 -0
- package/src/review/reopen.mjs +42 -16
- package/src/review/route.mjs +58 -4
- package/src/review/session-retry.mjs +59 -0
- package/src/session/facts.mjs +19 -7
- package/src/session/plan-check.mjs +4 -1
- package/src/session/spawn.mjs +54 -3
- package/src/state/block.mjs +139 -49
- package/src/state/config-snapshot.mjs +13 -3
- package/src/state/run.mjs +3 -2
- package/src/util/exec.mjs +48 -31
- package/src/util/native.mjs +131 -0
- package/src/util/reaper.mjs +238 -8
- package/src/worker/engine.mjs +7 -3
- package/src/worker/loop.mjs +211 -30
- package/src/worker/queue.mjs +107 -24
- package/src/worker/watch.mjs +18 -7
package/src/worker/loop.mjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* restarted worker boots on the snapshot, not on whatever the file says now. A record from before
|
|
21
21
|
* snapshots (or none) keeps the config the worker booted with.
|
|
22
22
|
*
|
|
23
|
-
* Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default
|
|
23
|
+
* Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default 6, 1–16) tickets at
|
|
24
24
|
* once, oldest first (or fewest running per block with `review.fair_share`); when one ends the
|
|
25
25
|
* next pending one starts. Two tickets for the same (block, file) never overlap (a file lock;
|
|
26
26
|
* the later one stays pending without taking a slot),
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
*/
|
|
37
37
|
|
|
38
38
|
import { spawn as nodeSpawn } from 'node:child_process';
|
|
39
|
-
import { existsSync, statSync } from 'node:fs';
|
|
39
|
+
import { existsSync, statSync, watch } from 'node:fs';
|
|
40
40
|
import os from 'node:os';
|
|
41
41
|
import path from 'node:path';
|
|
42
42
|
import { fileURLToPath } from 'node:url';
|
|
@@ -45,19 +45,21 @@ import { loadProjectConfig } from '../config/load.mjs';
|
|
|
45
45
|
import { reviewWorkers } from '../config/validate.mjs';
|
|
46
46
|
import { cacheHoursFrom, createDefaultKeyStore, resolveKey } from '../keys/store.mjs';
|
|
47
47
|
import { appendRow, readAllRows } from '../ledger/write.mjs';
|
|
48
|
-
import { spawnSession } from '../session/spawn.mjs';
|
|
48
|
+
import { providerLimit, spawnSession } from '../session/spawn.mjs';
|
|
49
49
|
import { snapshotFor } from '../state/config-snapshot.mjs';
|
|
50
50
|
import { runRecordPath } from '../state/paths.mjs';
|
|
51
51
|
import { processStartTime, readRun, reattachWorker } from '../state/run.mjs';
|
|
52
52
|
import { loadKey, signRow } from '../state/signer.mjs';
|
|
53
53
|
import { logWarning } from '../util/error-log.mjs';
|
|
54
54
|
import { withLock, tryLock } from '../util/file-locks.mjs';
|
|
55
|
+
import { startNativeDaemon } from '../util/native.mjs';
|
|
55
56
|
import { redact, writeSafe } from '../util/redact.mjs';
|
|
57
|
+
import { sweepProcesses } from '../util/reaper.mjs';
|
|
56
58
|
import { runRoot, setRunRoot } from '../util/tmp.mjs';
|
|
57
59
|
import { openFindings } from '../review/gate-check.mjs';
|
|
58
60
|
import { countLines, RouteError, routeTicket } from '../review/route.mjs';
|
|
59
61
|
import { canonicalPath, contentRootFor, reviewTicket } from './engine.mjs';
|
|
60
|
-
import { announceWorker, beatWorker, HEARTBEAT_MS, isDone, liveWorker, pendingTickets, readTicket, retractWorker, workerRegistry, writeResult } from './queue.mjs';
|
|
62
|
+
import { announceWorker, beatWorker, HEARTBEAT_MS, isDone, liveWorker, pendingTickets, queueDir, readTicket, retractWorker, workerRegistry, writeResult, writeTicketTimings } from './queue.mjs';
|
|
61
63
|
import { DELETED_HASH, hashBytes, readRegularFileNoFollow, repoRootOf, WorkerError } from './ticket.mjs';
|
|
62
64
|
import { watchTick } from './watch.mjs';
|
|
63
65
|
|
|
@@ -68,7 +70,8 @@ export const DEFAULT_POLL_MS = 250;
|
|
|
68
70
|
export const DEFAULT_SESSION_TIMEOUT_S = 300;
|
|
69
71
|
|
|
70
72
|
/** B42: `review.parallel_tickets` when the config does not set a valid one. */
|
|
71
|
-
export const DEFAULT_PARALLEL_TICKETS =
|
|
73
|
+
export const DEFAULT_PARALLEL_TICKETS = 6;
|
|
74
|
+
export const DEFAULT_PROVIDER_PARALLEL = 4;
|
|
72
75
|
|
|
73
76
|
/** B48: the fixed text of a failed autopilot expiry check in a drain (stderr and the warning log). */
|
|
74
77
|
export const EXPIRY_CHECK_FAILED = 'the autopilot expiry check failed in a worker drain; it runs again at the next drain';
|
|
@@ -236,6 +239,7 @@ export function recordedVerdict(result) {
|
|
|
236
239
|
* @property {typeof reviewTicket} [review] - the engine hook (default `./engine.mjs`).
|
|
237
240
|
* @property {typeof spawnSession} [spawn]
|
|
238
241
|
* @property {typeof readRun} [readRun] - the run-record read behind the per-ticket config (a test seam).
|
|
242
|
+
* @property {(ms: number) => Promise<unknown>} [sleep] - idle scheduler wait (a test seam).
|
|
239
243
|
* @property {{beforeLock?: () => void, afterCount?: (count: number) => Promise<unknown>}} [hooks] - deterministic boot races.
|
|
240
244
|
* @property {typeof readTicket} [readTicket] - the ticket read (a test seam).
|
|
241
245
|
* @property {typeof import('../util/exec.mjs').exec} [watchExec] - watch git exec seam.
|
|
@@ -279,6 +283,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
279
283
|
const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
|
|
280
284
|
const heartbeatMs = Number.isInteger(opts.heartbeatMs) && /** @type {number} */ (opts.heartbeatMs) >= MIN_HEARTBEAT_MS ? /** @type {number} */ (opts.heartbeatMs) : HEARTBEAT_MS;
|
|
281
285
|
let stopping = false;
|
|
286
|
+
let stopDaemon = async () => {};
|
|
287
|
+
let daemonStarted = false;
|
|
282
288
|
/** @type {ReturnType<typeof setInterval> | null} */
|
|
283
289
|
let heartbeat = null;
|
|
284
290
|
/** @type {ReturnType<typeof announceWorker> | null} */
|
|
@@ -287,6 +293,64 @@ export async function createWorker(opts, deps = {}) {
|
|
|
287
293
|
let wake = null;
|
|
288
294
|
/** @type {(() => void) | null} wakes the pool's wait in `drain` (stop) */
|
|
289
295
|
let poolWake = null;
|
|
296
|
+
let activity = 0;
|
|
297
|
+
|
|
298
|
+
// Sessions waiting for a provider retain their ticket's pool slot. Resolve the actual provider
|
|
299
|
+
// at spawn time, after routing and second-level overrides; admission hints never bypass caps.
|
|
300
|
+
let ticketLimit = parallelTickets(cfg);
|
|
301
|
+
let providerParallel = DEFAULT_PROVIDER_PARALLEL;
|
|
302
|
+
/** @type {Map<string, {active: number, pending: number}>} */
|
|
303
|
+
const ticketSessions = new Map();
|
|
304
|
+
/** @type {Map<string, number>} */
|
|
305
|
+
const providerActive = new Map();
|
|
306
|
+
/** @type {Array<{id: string, provider: string, start: () => void}>} */
|
|
307
|
+
const sessionQueue = [];
|
|
308
|
+
|
|
309
|
+
function dispatchSessions() {
|
|
310
|
+
for (let i = 0; i < sessionQueue.length;) {
|
|
311
|
+
const next = sessionQueue[i];
|
|
312
|
+
const state = ticketSessions.get(next.id);
|
|
313
|
+
const liveTickets = [...ticketSessions.values()].filter((s) => s.active > 0).length;
|
|
314
|
+
if ((providerActive.get(next.provider) ?? 0) >= providerParallel || (state.active === 0 && liveTickets >= ticketLimit)) {
|
|
315
|
+
i += 1;
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
sessionQueue.splice(i, 1);
|
|
319
|
+
state.pending -= 1;
|
|
320
|
+
state.active += 1;
|
|
321
|
+
providerActive.set(next.provider, (providerActive.get(next.provider) ?? 0) + 1);
|
|
322
|
+
next.start();
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** @param {string} id @param {Parameters<typeof spawnSession>[0]} opts @returns {ReturnType<typeof spawnSession>} */
|
|
327
|
+
function spawnPooled(id, opts) {
|
|
328
|
+
// Match the spawner's per-level override and top-level fallback; validation stays there.
|
|
329
|
+
const provider = opts.cfg?.levels?.[opts.level]?.provider ?? opts.cfg?.provider ?? 'unresolved';
|
|
330
|
+
const state = ticketSessions.get(id) ?? { active: 0, pending: 0 };
|
|
331
|
+
ticketSessions.set(id, state);
|
|
332
|
+
state.pending += 1;
|
|
333
|
+
// The spawner's existing per-attempt semaphores also cover fallback providers. Retain
|
|
334
|
+
// stricter provider_concurrency settings while applying this cap to fallback attempts.
|
|
335
|
+
const providerConcurrency = Object.fromEntries(['anthropic', 'openai', 'xai'].map((p) => [p, Math.min(providerParallel, providerLimit(opts.cfg, p))]));
|
|
336
|
+
const needsCap = Object.entries(providerConcurrency).some(([p, limit]) => limit < providerLimit(opts.cfg, p));
|
|
337
|
+
const sessionCfg = needsCap ? { ...opts.cfg, review: { ...opts.cfg.review, provider_concurrency: providerConcurrency } } : opts.cfg;
|
|
338
|
+
return new Promise((resolve, reject) => {
|
|
339
|
+
sessionQueue.push({ id, provider, start: () => {
|
|
340
|
+
Promise.resolve().then(() => spawn({ ...opts, cfg: sessionCfg },
|
|
341
|
+
{ env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
|
|
342
|
+
)).then(resolve, reject).finally(() => {
|
|
343
|
+
state.active -= 1;
|
|
344
|
+
providerActive.set(provider, providerActive.get(provider) - 1);
|
|
345
|
+
if (state.active === 0 && state.pending === 0) ticketSessions.delete(id);
|
|
346
|
+
dispatchSessions();
|
|
347
|
+
poolWake?.();
|
|
348
|
+
});
|
|
349
|
+
} });
|
|
350
|
+
dispatchSessions();
|
|
351
|
+
poolWake?.();
|
|
352
|
+
});
|
|
353
|
+
}
|
|
290
354
|
|
|
291
355
|
/** @param {Record<string, any>} row */
|
|
292
356
|
const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
|
|
@@ -302,6 +366,11 @@ export async function createWorker(opts, deps = {}) {
|
|
|
302
366
|
* @throws {Error} `config-snapshot` — a record that cannot be read, or a missing/altered snapshot.
|
|
303
367
|
*/
|
|
304
368
|
async function configFor(id) {
|
|
369
|
+
return snapshotFor(await configRecord(), id)?.config ?? cfg;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** Read once for pool admission hints; ticket processing still re-reads its own snapshot. */
|
|
373
|
+
async function configRecord() {
|
|
305
374
|
// The Jev key and the run root were settled at boot from `cfg`; a reload can never change
|
|
306
375
|
// them, because `keys`, `system1.key` and `tmp.root` are fixed keys `run reload` refuses.
|
|
307
376
|
let record;
|
|
@@ -310,13 +379,13 @@ export async function createWorker(opts, deps = {}) {
|
|
|
310
379
|
record = await loadRun(runId);
|
|
311
380
|
break;
|
|
312
381
|
} catch (err) {
|
|
313
|
-
if (/** @type {any} */ (err)?.code === 'no-run') return
|
|
382
|
+
if (/** @type {any} */ (err)?.code === 'no-run') return null;
|
|
314
383
|
// a transient read error (e.g. the record caught mid-replace) gets ONE retry
|
|
315
384
|
if (attempt >= 2) throw new Error('config-snapshot');
|
|
316
385
|
await new Promise((resolve) => setTimeout(resolve, CONFIG_RETRY_MS));
|
|
317
386
|
}
|
|
318
387
|
}
|
|
319
|
-
return
|
|
388
|
+
return record;
|
|
320
389
|
}
|
|
321
390
|
|
|
322
391
|
/**
|
|
@@ -335,7 +404,9 @@ export async function createWorker(opts, deps = {}) {
|
|
|
335
404
|
// A route list that cannot be used is `unavailable` — never a review on a model it did not name.
|
|
336
405
|
let routed;
|
|
337
406
|
try {
|
|
338
|
-
routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS }
|
|
407
|
+
routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS },
|
|
408
|
+
snapshotCfg.review?.adaptive?.enabled === false || !snapshotCfg.levels?.L2?.model
|
|
409
|
+
? undefined : { rows: await readRows(), ticketId: id, runId });
|
|
339
410
|
} catch (err) {
|
|
340
411
|
// only a route list that cannot be used is `route-invalid`; anything else is a real failure
|
|
341
412
|
// and goes to runOne (`worker-error`), named as a routing-step failure with its own text
|
|
@@ -379,9 +450,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
379
450
|
// `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
|
|
380
451
|
// over the ticket's; everything else the worker pins.
|
|
381
452
|
spawn: (sessionOpts) =>
|
|
382
|
-
|
|
453
|
+
spawnPooled(id,
|
|
383
454
|
/** @type {any} */ ({ cfg: tcfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
|
|
384
|
-
{ env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
|
|
385
455
|
),
|
|
386
456
|
writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
|
|
387
457
|
writeBlockRow: (row) => ledger({ ...row, block: ticket.block }),
|
|
@@ -407,14 +477,17 @@ export async function createWorker(opts, deps = {}) {
|
|
|
407
477
|
|
|
408
478
|
/**
|
|
409
479
|
* @param {string} id
|
|
480
|
+
* @param {{ticket: import('./queue.mjs').Ticket | null, queued_at: string | null, started_at: string}} [attempt]
|
|
410
481
|
* @returns {Promise<Record<string, any> | null>} the signed result, or null when abandoned.
|
|
411
482
|
*/
|
|
412
|
-
async function processTicket(id) {
|
|
483
|
+
async function processTicket(id, attempt = { ticket: null, queued_at: null, started_at: new Date().toISOString() }) {
|
|
413
484
|
/** @type {Record<string, any>} */
|
|
414
485
|
let result;
|
|
415
486
|
let ticket = null;
|
|
416
487
|
try {
|
|
417
488
|
ticket = loadTicket(repoRoot, id);
|
|
489
|
+
attempt.ticket = ticket;
|
|
490
|
+
attempt.queued_at = ticket.queued_at ?? ticket.enqueued_at ?? null;
|
|
418
491
|
} catch (err) {
|
|
419
492
|
result = { event: 'review.result', status: 'refused', reason: err instanceof WorkerError ? err.code : 'bad-ticket', approved: false };
|
|
420
493
|
}
|
|
@@ -423,13 +496,17 @@ export async function createWorker(opts, deps = {}) {
|
|
|
423
496
|
// read, advanced and saved by one ticket at a time; the later one waits here. The key
|
|
424
497
|
// carries the run so two workers in one process (tests) never share a lock.
|
|
425
498
|
const t = ticket;
|
|
426
|
-
result = await withLock(`${runId}\0${fileLockKey(t.block, t.file)}`, () =>
|
|
499
|
+
result = await withLock(`${runId}\0${fileLockKey(t.block, t.file)}`, () => {
|
|
500
|
+
attempt.started_at = new Date().toISOString();
|
|
501
|
+
writeTicketTimings(repoRoot, t, { queued_at: attempt.queued_at, started_at: attempt.started_at, finished_at: null, duration_ms: null });
|
|
502
|
+
return reviewOne(id, t);
|
|
503
|
+
});
|
|
427
504
|
}
|
|
428
505
|
// Stopping: a ticket whose session our own stop killed ends `unavailable` — leave it queued
|
|
429
506
|
// (no done marker) for the next worker instead of recording the kill as a failure. A ticket
|
|
430
507
|
// that reached a real outcome while the pool drained is recorded as usual.
|
|
431
508
|
if (stopping && result.status === 'unavailable') return null;
|
|
432
|
-
return complete(id, result);
|
|
509
|
+
return complete(id, result, attempt);
|
|
433
510
|
}
|
|
434
511
|
|
|
435
512
|
/**
|
|
@@ -485,10 +562,14 @@ export async function createWorker(opts, deps = {}) {
|
|
|
485
562
|
* Sign and write the result + done marker, then the signed ledger row. A ledger failure does not
|
|
486
563
|
* undo the result: the done marker is what the queue reads.
|
|
487
564
|
* @param {string} id @param {Record<string, any>} outcome - recorded through {@link recordedVerdict}.
|
|
565
|
+
* @param {{ticket: import('./queue.mjs').Ticket | null, queued_at: string | null, started_at: string}} attempt
|
|
488
566
|
*/
|
|
489
|
-
async function complete(id, outcome) {
|
|
567
|
+
async function complete(id, outcome, attempt) {
|
|
568
|
+
const finished_at = new Date().toISOString();
|
|
569
|
+
const timings = { queued_at: attempt.queued_at, started_at: attempt.started_at, finished_at, duration_ms: Math.max(0, Date.parse(finished_at) - Date.parse(attempt.started_at)) };
|
|
570
|
+
if (attempt.ticket) writeTicketTimings(repoRoot, attempt.ticket, timings);
|
|
490
571
|
/** @type {Record<string, any>} */
|
|
491
|
-
const result = { ...recordedVerdict(outcome), worker: identity };
|
|
572
|
+
const result = { ...(attempt.ticket ? { block: attempt.ticket.block, file: attempt.ticket.file, content_hash: attempt.ticket.content_hash } : {}), ...recordedVerdict(outcome), ...timings, worker: identity };
|
|
492
573
|
const signed = writeResult({ repoRoot, runId, ticket: id, result, key });
|
|
493
574
|
try {
|
|
494
575
|
await ledger({
|
|
@@ -499,6 +580,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
499
580
|
file: result.file ?? null,
|
|
500
581
|
content_hash: result.content_hash ?? null,
|
|
501
582
|
status: result.status,
|
|
583
|
+
...timings,
|
|
502
584
|
// B52: the gate reads the verdict from the ledger, never from the result file
|
|
503
585
|
approved: result.approved,
|
|
504
586
|
open: result.open,
|
|
@@ -525,13 +607,14 @@ export async function createWorker(opts, deps = {}) {
|
|
|
525
607
|
* @param {string} id @returns {Promise<number>} 1 when completed, else 0.
|
|
526
608
|
*/
|
|
527
609
|
async function runOne(id) {
|
|
610
|
+
const attempt = { ticket: /** @type {import('./queue.mjs').Ticket | null} */ (null), queued_at: /** @type {string | null} */ (null), started_at: new Date().toISOString() };
|
|
528
611
|
try {
|
|
529
|
-
return (await processTicket(id)) !== null ? 1 : 0;
|
|
612
|
+
return (await processTicket(id, attempt)) !== null ? 1 : 0;
|
|
530
613
|
} catch (err) {
|
|
531
614
|
// B90: a failure a step names (StepError) keeps its step and original text, redacted
|
|
532
615
|
const detail = err instanceof StepError ? { detail: redact(err.message) } : {};
|
|
533
616
|
try {
|
|
534
|
-
await complete(id, { event: 'review.result', status: 'unavailable', reason: 'worker-error', ...detail, approved: false });
|
|
617
|
+
await complete(id, { event: 'review.result', status: 'unavailable', reason: 'worker-error', ...detail, approved: false }, attempt);
|
|
535
618
|
return 1;
|
|
536
619
|
} catch {
|
|
537
620
|
poisoned.add(id);
|
|
@@ -543,21 +626,26 @@ export async function createWorker(opts, deps = {}) {
|
|
|
543
626
|
/**
|
|
544
627
|
* Pool settings from the snapshot in force, else the boot config. A run reload applies at the
|
|
545
628
|
* next drain; anything but an integer 1–16 uses the default size.
|
|
546
|
-
* @returns {Promise<{limit: number, fairShare: boolean}>}
|
|
629
|
+
* @returns {Promise<{limit: number, fairShare: boolean, record: Record<string, any> | null}>}
|
|
547
630
|
*/
|
|
548
631
|
async function poolSettings() {
|
|
549
632
|
let current = cfg;
|
|
633
|
+
let record = null;
|
|
550
634
|
try {
|
|
551
|
-
|
|
635
|
+
record = await configRecord();
|
|
636
|
+
current = snapshotFor(record, '')?.config ?? cfg;
|
|
552
637
|
} catch {
|
|
553
638
|
// an unreadable record: the boot settings (each ticket still fails closed on its own)
|
|
554
639
|
}
|
|
555
|
-
|
|
640
|
+
ticketLimit = parallelTickets(current);
|
|
641
|
+
const cap = current?.review?.provider_parallel;
|
|
642
|
+
providerParallel = Number.isInteger(cap) && cap >= 1 && cap <= MAX_PARALLEL_TICKETS ? cap : DEFAULT_PROVIDER_PARALLEL;
|
|
643
|
+
return { limit: parallelTickets(current), fairShare: current?.review?.fair_share === true, record };
|
|
556
644
|
}
|
|
557
645
|
|
|
558
646
|
/**
|
|
559
647
|
* The block and (block, file) key of each ticket read so far, by id. A written ticket never
|
|
560
|
-
* changes
|
|
648
|
+
* changes its identity as lifecycle metadata advances, so scheduling reads it once across wake-ups.
|
|
561
649
|
* Dropped once the ticket is done.
|
|
562
650
|
* @type {Map<string, {key: string, block: string}>}
|
|
563
651
|
*/
|
|
@@ -582,6 +670,27 @@ export async function createWorker(opts, deps = {}) {
|
|
|
582
670
|
}
|
|
583
671
|
}
|
|
584
672
|
|
|
673
|
+
/**
|
|
674
|
+
* The initial reviewer's provider is an admission hint, including snapshot and size routing.
|
|
675
|
+
* Unknown or invalid hints still enter the bounded pool so the normal checks can refuse them.
|
|
676
|
+
* Later sessions may use another provider; `spawnPooled` enforces their actual session caps.
|
|
677
|
+
* @param {string} id @param {Record<string, any> | null} record @returns {Promise<string | null>}
|
|
678
|
+
*/
|
|
679
|
+
async function providerHint(id, record) {
|
|
680
|
+
try {
|
|
681
|
+
let current = snapshotFor(record, id)?.config ?? cfg;
|
|
682
|
+
if (current?.review?.routes?.length) {
|
|
683
|
+
const ticket = loadTicket(repoRoot, id);
|
|
684
|
+
const root = await contentRootFor(runId, ticket.block, repoRoot);
|
|
685
|
+
const bytes = readRegularFileNoFollow(root, ticket.file);
|
|
686
|
+
current = routeTicket(current, { lines: countLines(bytes), deleted: bytes === null, timeoutS: DEFAULT_SESSION_TIMEOUT_S }).cfg;
|
|
687
|
+
}
|
|
688
|
+
return current?.levels?.L2?.provider ?? current?.provider ?? null;
|
|
689
|
+
} catch {
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
585
694
|
/**
|
|
586
695
|
* Serve every pending ticket (B42): up to `review.parallel_tickets` at once, oldest first; with
|
|
587
696
|
* fair share, each slot goes to the block with fewest running tickets, ties oldest first. When
|
|
@@ -609,6 +718,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
609
718
|
}
|
|
610
719
|
/** @type {number | null} */
|
|
611
720
|
let limit = null;
|
|
721
|
+
/** @type {Record<string, any> | null} */
|
|
722
|
+
let record = null;
|
|
612
723
|
let fairShare = false;
|
|
613
724
|
let done = 0;
|
|
614
725
|
/** @type {Map<string, Promise<void>>} */
|
|
@@ -617,6 +728,8 @@ export async function createWorker(opts, deps = {}) {
|
|
|
617
728
|
const busy = new Set();
|
|
618
729
|
/** @type {Map<string, number>} running tickets per block */
|
|
619
730
|
const runningByBlock = new Map();
|
|
731
|
+
/** @type {Map<string, number>} reserve provider capacity while admitted tickets prepare */
|
|
732
|
+
const admittedByProvider = new Map();
|
|
620
733
|
/** @type {Set<string>} */
|
|
621
734
|
const started = new Set();
|
|
622
735
|
for (;;) {
|
|
@@ -643,10 +756,13 @@ export async function createWorker(opts, deps = {}) {
|
|
|
643
756
|
try { await logWarning({ warning: 'watch_tick_failed', message: text }); } catch { /* a log failure never stops dispatch */ }
|
|
644
757
|
}
|
|
645
758
|
const pending = pendingTickets(repoRoot).filter((id) => !poisoned.has(id) && !started.has(id));
|
|
646
|
-
if (pending.length > 0
|
|
759
|
+
if (pending.length > 0 || running.size > 0) activity += 1;
|
|
760
|
+
if (pending.length > 0 && limit === null) ({ limit, fairShare, record } = await poolSettings());
|
|
647
761
|
// Cache even unreadable hints for this tick: ranking never reads a ticket twice.
|
|
648
762
|
/** @type {Map<string, ReturnType<typeof fileKeyOf>>} */
|
|
649
763
|
const hints = new Map();
|
|
764
|
+
/** @type {Map<string, string | null>} */
|
|
765
|
+
const providers = new Map();
|
|
650
766
|
while (pending.length > 0 && running.size < limit && !stopping) {
|
|
651
767
|
let next = -1;
|
|
652
768
|
let fewest = Infinity;
|
|
@@ -655,6 +771,9 @@ export async function createWorker(opts, deps = {}) {
|
|
|
655
771
|
if (!hints.has(id)) hints.set(id, fileKeyOf(id));
|
|
656
772
|
const file = hints.get(id);
|
|
657
773
|
if (file && busy.has(file.key)) continue; // waits without a slot
|
|
774
|
+
if (!providers.has(id)) providers.set(id, await providerHint(id, record));
|
|
775
|
+
const provider = providers.get(id);
|
|
776
|
+
if (provider && Math.max(admittedByProvider.get(provider) ?? 0, providerActive.get(provider) ?? 0) >= providerParallel) continue;
|
|
658
777
|
const count = fairShare && file ? runningByBlock.get(file.block) ?? 0 : 0;
|
|
659
778
|
if (count < fewest) {
|
|
660
779
|
next = i;
|
|
@@ -674,10 +793,12 @@ export async function createWorker(opts, deps = {}) {
|
|
|
674
793
|
}
|
|
675
794
|
const file = hints.get(id);
|
|
676
795
|
const fileKey = file?.key ?? null;
|
|
796
|
+
const provider = providers.get(id);
|
|
677
797
|
started.add(id);
|
|
678
798
|
activeTickets.add(id);
|
|
679
799
|
if (fileKey !== null) busy.add(fileKey);
|
|
680
800
|
if (file) runningByBlock.set(file.block, (runningByBlock.get(file.block) ?? 0) + 1);
|
|
801
|
+
if (provider) admittedByProvider.set(provider, (admittedByProvider.get(provider) ?? 0) + 1);
|
|
681
802
|
const slot = runOne(id)
|
|
682
803
|
.then((n) => {
|
|
683
804
|
done += n;
|
|
@@ -690,6 +811,7 @@ export async function createWorker(opts, deps = {}) {
|
|
|
690
811
|
fileKeys.delete(id);
|
|
691
812
|
if (fileKey !== null) busy.delete(fileKey);
|
|
692
813
|
if (file) runningByBlock.set(file.block, runningByBlock.get(file.block) - 1);
|
|
814
|
+
if (provider) admittedByProvider.set(provider, admittedByProvider.get(provider) - 1);
|
|
693
815
|
});
|
|
694
816
|
running.set(id, slot);
|
|
695
817
|
}
|
|
@@ -700,6 +822,13 @@ export async function createWorker(opts, deps = {}) {
|
|
|
700
822
|
return done;
|
|
701
823
|
}
|
|
702
824
|
|
|
825
|
+
/** Process inspection belongs to the worker lifecycle, outside ticket dispatch and draining. */
|
|
826
|
+
async function cleanOrphans() {
|
|
827
|
+
try {
|
|
828
|
+
await sweepProcesses({ apply: true, orphansOnly: true, runId, zombies: false });
|
|
829
|
+
} catch { /* process hygiene must never fail the worker */ }
|
|
830
|
+
}
|
|
831
|
+
|
|
703
832
|
/**
|
|
704
833
|
* Resolve when one of `slots` settles, `pollMs` passes, or `stop()` is called.
|
|
705
834
|
* @param {Array<Promise<void>>} slots
|
|
@@ -745,57 +874,99 @@ export async function createWorker(opts, deps = {}) {
|
|
|
745
874
|
heartbeat = null;
|
|
746
875
|
}
|
|
747
876
|
|
|
877
|
+
async function startDaemon() {
|
|
878
|
+
if (daemonStarted) return;
|
|
879
|
+
daemonStarted = true;
|
|
880
|
+
stopDaemon = await startNativeDaemon(runId, cfg, env);
|
|
881
|
+
if (stopping) await stopDaemon();
|
|
882
|
+
}
|
|
883
|
+
|
|
748
884
|
/** Serve the queue until `stop()`. */
|
|
749
885
|
async function run() {
|
|
750
|
-
|
|
886
|
+
let interval = pollMs;
|
|
887
|
+
let polled = false;
|
|
888
|
+
/** @type {import('node:fs').FSWatcher | null} */
|
|
889
|
+
let queueWatch = null;
|
|
751
890
|
try {
|
|
891
|
+
await startDaemon();
|
|
892
|
+
announce();
|
|
893
|
+
// Enqueue publishes a ticket by rename. Wake even during the longest idle back-off;
|
|
894
|
+
// heartbeat and temporary files do not wake an idle worker.
|
|
895
|
+
try {
|
|
896
|
+
queueWatch = watch(queueDir(repoRoot), (_event, file) => {
|
|
897
|
+
if (!file || !/^[a-f0-9]{24}\.json$/.test(String(file))) return;
|
|
898
|
+
activity += 1;
|
|
899
|
+
wake?.();
|
|
900
|
+
poolWake?.();
|
|
901
|
+
});
|
|
902
|
+
queueWatch.on('error', () => {
|
|
903
|
+
queueWatch?.close();
|
|
904
|
+
queueWatch = null;
|
|
905
|
+
wake?.();
|
|
906
|
+
});
|
|
907
|
+
} catch { /* without filesystem notifications, keep polling at the fast interval */ }
|
|
752
908
|
while (!stopping) {
|
|
909
|
+
const before = activity;
|
|
753
910
|
try {
|
|
754
|
-
await drain();
|
|
911
|
+
if (await drain() > 0) await cleanOrphans();
|
|
755
912
|
} catch {
|
|
756
913
|
// the queue directory itself failed to read: try again on the next poll
|
|
757
914
|
}
|
|
758
915
|
if (stopping) break;
|
|
759
|
-
|
|
760
|
-
|
|
916
|
+
const unchanged = before === activity;
|
|
917
|
+
interval = polled && unchanged ? Math.min(5000, interval * 2) : pollMs;
|
|
918
|
+
if (queueWatch === null && !deps.sleep) interval = pollMs;
|
|
919
|
+
polled = true;
|
|
920
|
+
await new Promise((resolve, reject) => {
|
|
921
|
+
let timer;
|
|
761
922
|
wake = () => {
|
|
762
923
|
clearTimeout(timer);
|
|
763
924
|
resolve(undefined);
|
|
764
925
|
};
|
|
926
|
+
if (deps.sleep) deps.sleep(interval).then(resolve, reject);
|
|
927
|
+
else timer = setTimeout(resolve, interval);
|
|
765
928
|
});
|
|
766
929
|
wake = null;
|
|
767
930
|
}
|
|
768
931
|
} finally {
|
|
932
|
+
queueWatch?.close();
|
|
769
933
|
release();
|
|
934
|
+
await stopDaemon();
|
|
770
935
|
}
|
|
771
936
|
}
|
|
772
937
|
|
|
773
938
|
/** Drain the queue once while announced, then release (`worker --once`). */
|
|
774
939
|
async function once() {
|
|
775
|
-
announce();
|
|
776
940
|
try {
|
|
777
|
-
|
|
941
|
+
await startDaemon();
|
|
942
|
+
announce();
|
|
943
|
+
const done = await drain();
|
|
944
|
+
if (done > 0) await cleanOrphans();
|
|
945
|
+
return done;
|
|
778
946
|
} finally {
|
|
779
947
|
stop();
|
|
780
948
|
release();
|
|
949
|
+
await stopDaemon();
|
|
781
950
|
}
|
|
782
951
|
}
|
|
783
952
|
|
|
784
953
|
/** Stop taking tickets; the running ones are awaited (a killed one is abandoned, see `processTicket`). */
|
|
785
954
|
function stop() {
|
|
786
955
|
stopping = true;
|
|
956
|
+
void stopDaemon();
|
|
787
957
|
wake?.();
|
|
788
958
|
poolWake?.();
|
|
789
959
|
}
|
|
790
960
|
|
|
791
961
|
/** Withdraw this process's announcement (`worker.json`) if it is still ours. */
|
|
792
962
|
function release() {
|
|
963
|
+
void stopDaemon();
|
|
793
964
|
stopBeating();
|
|
794
965
|
if (self !== null) retractWorker(repoRoot, self.pid, self.start_time);
|
|
795
966
|
self = null;
|
|
796
967
|
}
|
|
797
968
|
|
|
798
|
-
return { processTicket, drain, run, once, stop, release, announce, watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
|
|
969
|
+
return { processTicket, drain, run, once, stop, release, announce, startDaemon, stopDaemon: () => stopDaemon(), watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
|
|
799
970
|
}
|
|
800
971
|
|
|
801
972
|
/**
|
|
@@ -837,7 +1008,17 @@ export async function bootWorker(opts, deps = {}) {
|
|
|
837
1008
|
if (entries.some((entry) => entry.live && !entry.worker) || (legacy && legacy.run !== opts.runId) || entries.some((entry) => entry.live && entry.worker?.run !== opts.runId) || count >= limit) {
|
|
838
1009
|
throw new WorkerError('worker-running', `review.workers limit ${limit}: a live or unreadable worker entry already serves this queue`);
|
|
839
1010
|
}
|
|
840
|
-
worker.
|
|
1011
|
+
// Startup must finish before worker.json signals readiness: an owner may
|
|
1012
|
+
// remove HOME immediately after that announcement to stop an orphan worker.
|
|
1013
|
+
try {
|
|
1014
|
+
await worker.startDaemon();
|
|
1015
|
+
worker.announce();
|
|
1016
|
+
} catch (error) {
|
|
1017
|
+
worker.stop();
|
|
1018
|
+
worker.release();
|
|
1019
|
+
await worker.stopDaemon();
|
|
1020
|
+
throw error;
|
|
1021
|
+
}
|
|
841
1022
|
});
|
|
842
1023
|
return { ...worker, repoRoot, runRootDir };
|
|
843
1024
|
}
|