@codedology/code-forge 0.15.3 → 0.16.0

package/src/worker/loop.mjs

@@ -20,7 +20,7 @@
  * restarted worker boots on the snapshot, not on whatever the file says now. A record from before
  * snapshots (or none) keeps the config the worker booted with.
  *
- * Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default 3, 1–16) tickets at
+ * Ticket pool (B42): the worker runs up to `review.parallel_tickets` (default 6, 1–16) tickets at
  * once, oldest first (or fewest running per block with `review.fair_share`); when one ends the
  * next pending one starts. Two tickets for the same (block, file) never overlap (a file lock;
  * the later one stays pending without taking a slot),
@@ -44,14 +44,16 @@ import { expiryChecks } from '../autopilot/limits.mjs';
 import { loadProjectConfig } from '../config/load.mjs';
 import { reviewWorkers } from '../config/validate.mjs';
 import { cacheHoursFrom, createDefaultKeyStore, resolveKey } from '../keys/store.mjs';
-import { appendRow, readAllRows } from '../ledger/write.mjs';
-import { spawnSession } from '../session/spawn.mjs';
+import { createTailReader } from '../ledger/tail.mjs';
+import { appendRow } from '../ledger/write.mjs';
+import { providerLimit, spawnSession } from '../session/spawn.mjs';
 import { snapshotFor } from '../state/config-snapshot.mjs';
 import { runRecordPath } from '../state/paths.mjs';
 import { processStartTime, readRun, reattachWorker } from '../state/run.mjs';
 import { loadKey, signRow } from '../state/signer.mjs';
 import { logWarning } from '../util/error-log.mjs';
 import { withLock, tryLock } from '../util/file-locks.mjs';
+import { startNativeDaemon } from '../util/native.mjs';
 import { redact, writeSafe } from '../util/redact.mjs';
 import { sweepProcesses } from '../util/reaper.mjs';
 import { runRoot, setRunRoot } from '../util/tmp.mjs';
@@ -69,7 +71,8 @@ export const DEFAULT_POLL_MS = 250;
 export const DEFAULT_SESSION_TIMEOUT_S = 300;
 
 /** B42: `review.parallel_tickets` when the config does not set a valid one. */
-export const DEFAULT_PARALLEL_TICKETS = 3;
+export const DEFAULT_PARALLEL_TICKETS = 6;
+export const DEFAULT_PROVIDER_PARALLEL = 4;
 
 /** B48: the fixed text of a failed autopilot expiry check in a drain (stderr and the warning log). */
 export const EXPIRY_CHECK_FAILED = 'the autopilot expiry check failed in a worker drain; it runs again at the next drain';
@@ -241,6 +244,7 @@ export function recordedVerdict(result) {
  * @property {{beforeLock?: () => void, afterCount?: (count: number) => Promise<unknown>}} [hooks] - deterministic boot races.
  * @property {typeof readTicket} [readTicket] - the ticket read (a test seam).
  * @property {typeof import('../util/exec.mjs').exec} [watchExec] - watch git exec seam.
+ * @property {() => number} [watchNow] - watch debounce clock (default: Date.now).
  * @property {Parameters<typeof watchTick>[0]['hooks']} [watchHooks] - deterministic watch races.
  * @property {import('../keys/store.mjs').KeyStore} [store] - default: B2's production store.
  * @property {any} [opRead]
@@ -248,7 +252,7 @@ export function recordedVerdict(result) {
  * @property {Partial<Record<'claude'|'codex'|'grok', string>>} [bins]
  * @property {(row: Record<string, any>) => Promise<unknown>} [writeRow] - default: B6 `appendRow`.
  * @property {() => Promise<Array<Record<string, any>>>} [readRows] - the ledger rows the fix-loop
- *   state is anchored in; default: B6 `readAllRows(slug)`.
+ *   state is anchored in; default: one project tail reader for the worker.
  * @property {{write: (s: string) => unknown}} [stderr]
  * @property {import('../review/triage.mjs').JevAsk} [jev] - S1 for triage and the recheck (tests
  *   inject a mock); default: `askJev` with the resolved Jev key, none without one.
@@ -270,7 +274,7 @@ export async function createWorker(opts, deps = {}) {
   const loadRun = deps.readRun ?? readRun;
   const loadTicket = deps.readTicket ?? readTicket;
   const writeRow = deps.writeRow ?? ((/** @type {Record<string, any>} */ row) => appendRow(row, { slug }));
-  const readRows = deps.readRows ?? (() => readAllRows(slug));
+  const readRows = deps.readRows ?? createTailReader(slug).read;
   // B48: the run's own writer (unsigned base; the checks sign their rows with the run key)
   const expiryCheck = deps.expiryCheck ?? (() => expiryChecks(runId, new Date(), { writeRow }));
   const warnStream = deps.stderr ?? process.stderr;
@@ -281,6 +285,8 @@ export async function createWorker(opts, deps = {}) {
   const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
   const heartbeatMs = Number.isInteger(opts.heartbeatMs) && /** @type {number} */ (opts.heartbeatMs) >= MIN_HEARTBEAT_MS ? /** @type {number} */ (opts.heartbeatMs) : HEARTBEAT_MS;
   let stopping = false;
+  let stopDaemon = async () => {};
+  let daemonStarted = false;
   /** @type {ReturnType<typeof setInterval> | null} */
   let heartbeat = null;
   /** @type {ReturnType<typeof announceWorker> | null} */
@@ -291,11 +297,68 @@ export async function createWorker(opts, deps = {}) {
   let poolWake = null;
   let activity = 0;
 
+  // Sessions waiting for a provider retain their ticket's pool slot. Resolve the actual provider
+  // at spawn time, after routing and second-level overrides; admission hints never bypass caps.
+  let ticketLimit = parallelTickets(cfg);
+  let providerParallel = DEFAULT_PROVIDER_PARALLEL;
+  /** @type {Map<string, {active: number, pending: number}>} */
+  const ticketSessions = new Map();
+  /** @type {Map<string, number>} */
+  const providerActive = new Map();
+  /** @type {Array<{id: string, provider: string, start: () => void}>} */
+  const sessionQueue = [];
+
+  function dispatchSessions() {
+    for (let i = 0; i < sessionQueue.length;) {
+      const next = sessionQueue[i];
+      const state = ticketSessions.get(next.id);
+      const liveTickets = [...ticketSessions.values()].filter((s) => s.active > 0).length;
+      if ((providerActive.get(next.provider) ?? 0) >= providerParallel || (state.active === 0 && liveTickets >= ticketLimit)) {
+        i += 1;
+        continue;
+      }
+      sessionQueue.splice(i, 1);
+      state.pending -= 1;
+      state.active += 1;
+      providerActive.set(next.provider, (providerActive.get(next.provider) ?? 0) + 1);
+      next.start();
+    }
+  }
+
+  /** @param {string} id @param {Parameters<typeof spawnSession>[0]} opts @returns {ReturnType<typeof spawnSession>} */
+  function spawnPooled(id, opts) {
+    // Match the spawner's per-level override and top-level fallback; validation stays there.
+    const provider = opts.cfg?.levels?.[opts.level]?.provider ?? opts.cfg?.provider ?? 'unresolved';
+    const state = ticketSessions.get(id) ?? { active: 0, pending: 0 };
+    ticketSessions.set(id, state);
+    state.pending += 1;
+    // The spawner's existing per-attempt semaphores also cover fallback providers. Retain
+    // stricter provider_concurrency settings while applying this cap to fallback attempts.
+    const providerConcurrency = Object.fromEntries(['anthropic', 'openai', 'xai'].map((p) => [p, Math.min(providerParallel, providerLimit(opts.cfg, p))]));
+    const needsCap = Object.entries(providerConcurrency).some(([p, limit]) => limit < providerLimit(opts.cfg, p));
+    const sessionCfg = needsCap ? { ...opts.cfg, review: { ...opts.cfg.review, provider_concurrency: providerConcurrency } } : opts.cfg;
+    return new Promise((resolve, reject) => {
+      sessionQueue.push({ id, provider, start: () => {
+        Promise.resolve().then(() => spawn({ ...opts, cfg: sessionCfg },
+          { env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
+        )).then(resolve, reject).finally(() => {
+          state.active -= 1;
+          providerActive.set(provider, providerActive.get(provider) - 1);
+          if (state.active === 0 && state.pending === 0) ticketSessions.delete(id);
+          dispatchSessions();
+          poolWake?.();
+        });
+      } });
+      dispatchSessions();
+      poolWake?.();
+    });
+  }
+
   /** @param {Record<string, any>} row */
   const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
   // Shared across direct ticks and the scheduler, so running tickets are never superseded.
   const activeTickets = new Set();
-  const tickWatch = () => watchTick({ repoRoot, runId, cfg, readRun: loadRun, writeRow: ledger, running: activeTickets, exec: deps.watchExec, hooks: deps.watchHooks });
+  const tickWatch = () => watchTick({ repoRoot, runId, cfg, readRun: loadRun, writeRow: ledger, running: activeTickets, exec: deps.watchExec, hooks: deps.watchHooks, now: deps.watchNow });
 
   /**
    * The config ticket `id` is reviewed with (B35): its snapshot from the run record, re-read per
@@ -305,6 +368,11 @@ export async function createWorker(opts, deps = {}) {
    * @throws {Error} `config-snapshot` — a record that cannot be read, or a missing/altered snapshot.
    */
   async function configFor(id) {
+    return snapshotFor(await configRecord(), id)?.config ?? cfg;
+  }
+
+  /** Read once for pool admission hints; ticket processing still re-reads its own snapshot. */
+  async function configRecord() {
     // The Jev key and the run root were settled at boot from `cfg`; a reload can never change
     // them, because `keys`, `system1.key` and `tmp.root` are fixed keys `run reload` refuses.
     let record;
@@ -313,13 +381,13 @@ export async function createWorker(opts, deps = {}) {
         record = await loadRun(runId);
         break;
       } catch (err) {
-        if (/** @type {any} */ (err)?.code === 'no-run') return cfg;
+        if (/** @type {any} */ (err)?.code === 'no-run') return null;
         // a transient read error (e.g. the record caught mid-replace) gets ONE retry
         if (attempt >= 2) throw new Error('config-snapshot');
         await new Promise((resolve) => setTimeout(resolve, CONFIG_RETRY_MS));
       }
     }
-    return snapshotFor(record, id)?.config ?? cfg;
+    return record;
   }
 
   /**
@@ -338,7 +406,9 @@ export async function createWorker(opts, deps = {}) {
     // A route list that cannot be used is `unavailable` — never a review on a model it did not name.
     let routed;
     try {
-      routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS });
+      routed = routeTicket(snapshotCfg, { lines: size.lines, deleted: size.deleted, timeoutS: baseTimeoutS },
+        snapshotCfg.review?.adaptive?.enabled === false || !snapshotCfg.levels?.L2?.model
+          ? undefined : { rows: await readRows(), ticketId: id, runId });
     } catch (err) {
       // only a route list that cannot be used is `route-invalid`; anything else is a real failure
       // and goes to runOne (`worker-error`), named as a routing-step failure with its own text
@@ -382,9 +452,8 @@ export async function createWorker(opts, deps = {}) {
       // `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
       // over the ticket's; everything else the worker pins.
       spawn: (sessionOpts) =>
-        spawn(
+        spawnPooled(id,
           /** @type {any} */ ({ cfg: tcfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
-          { env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
         ),
       writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
       writeBlockRow: (row) => ledger({ ...row, block: ticket.block }),
@@ -559,16 +628,21 @@ export async function createWorker(opts, deps = {}) {
   /**
    * Pool settings from the snapshot in force, else the boot config. A run reload applies at the
    * next drain; anything but an integer 1–16 uses the default size.
-   * @returns {Promise<{limit: number, fairShare: boolean}>}
+   * @returns {Promise<{limit: number, fairShare: boolean, record: Record<string, any> | null}>}
    */
   async function poolSettings() {
     let current = cfg;
+    let record = null;
     try {
-      current = await configFor('');
+      record = await configRecord();
+      current = snapshotFor(record, '')?.config ?? cfg;
     } catch {
       // an unreadable record: the boot settings (each ticket still fails closed on its own)
     }
-    return { limit: parallelTickets(current), fairShare: current?.review?.fair_share === true };
+    ticketLimit = parallelTickets(current);
+    const cap = current?.review?.provider_parallel;
+    providerParallel = Number.isInteger(cap) && cap >= 1 && cap <= MAX_PARALLEL_TICKETS ? cap : DEFAULT_PROVIDER_PARALLEL;
+    return { limit: parallelTickets(current), fairShare: current?.review?.fair_share === true, record };
   }
 
   /**
@@ -598,6 +672,27 @@ export async function createWorker(opts, deps = {}) {
     }
   }
 
+  /**
+   * The initial reviewer's provider is an admission hint, including snapshot and size routing.
+   * Unknown or invalid hints still enter the bounded pool so the normal checks can refuse them.
+   * Later sessions may use another provider; `spawnPooled` enforces their actual session caps.
+   * @param {string} id @param {Record<string, any> | null} record @returns {Promise<string | null>}
+   */
+  async function providerHint(id, record) {
+    try {
+      let current = snapshotFor(record, id)?.config ?? cfg;
+      if (current?.review?.routes?.length) {
+        const ticket = loadTicket(repoRoot, id);
+        const root = await contentRootFor(runId, ticket.block, repoRoot);
+        const bytes = readRegularFileNoFollow(root, ticket.file);
+        current = routeTicket(current, { lines: countLines(bytes), deleted: bytes === null, timeoutS: DEFAULT_SESSION_TIMEOUT_S }).cfg;
+      }
+      return current?.levels?.L2?.provider ?? current?.provider ?? null;
+    } catch {
+      return null;
+    }
+  }
+
   /**
    * Serve every pending ticket (B42): up to `review.parallel_tickets` at once, oldest first; with
    * fair share, each slot goes to the block with fewest running tickets, ties oldest first. When
@@ -625,6 +720,8 @@ export async function createWorker(opts, deps = {}) {
     }
     /** @type {number | null} */
     let limit = null;
+    /** @type {Record<string, any> | null} */
+    let record = null;
     let fairShare = false;
     let done = 0;
     /** @type {Map<string, Promise<void>>} */
@@ -633,6 +730,8 @@ export async function createWorker(opts, deps = {}) {
     const busy = new Set();
     /** @type {Map<string, number>} running tickets per block */
     const runningByBlock = new Map();
+    /** @type {Map<string, number>} reserve provider capacity while admitted tickets prepare */
+    const admittedByProvider = new Map();
     /** @type {Set<string>} */
     const started = new Set();
     for (;;) {
@@ -660,10 +759,12 @@ export async function createWorker(opts, deps = {}) {
         }
         const pending = pendingTickets(repoRoot).filter((id) => !poisoned.has(id) && !started.has(id));
         if (pending.length > 0 || running.size > 0) activity += 1;
-        if (pending.length > 0 && limit === null) ({ limit, fairShare } = await poolSettings());
+        if (pending.length > 0 && limit === null) ({ limit, fairShare, record } = await poolSettings());
         // Cache even unreadable hints for this tick: ranking never reads a ticket twice.
         /** @type {Map<string, ReturnType<typeof fileKeyOf>>} */
         const hints = new Map();
+        /** @type {Map<string, string | null>} */
+        const providers = new Map();
         while (pending.length > 0 && running.size < limit && !stopping) {
           let next = -1;
           let fewest = Infinity;
@@ -672,6 +773,9 @@ export async function createWorker(opts, deps = {}) {
             if (!hints.has(id)) hints.set(id, fileKeyOf(id));
             const file = hints.get(id);
             if (file && busy.has(file.key)) continue; // waits without a slot
+            if (!providers.has(id)) providers.set(id, await providerHint(id, record));
+            const provider = providers.get(id);
+            if (provider && Math.max(admittedByProvider.get(provider) ?? 0, providerActive.get(provider) ?? 0) >= providerParallel) continue;
             const count = fairShare && file ? runningByBlock.get(file.block) ?? 0 : 0;
             if (count < fewest) {
               next = i;
@@ -691,10 +795,12 @@ export async function createWorker(opts, deps = {}) {
           }
           const file = hints.get(id);
           const fileKey = file?.key ?? null;
+          const provider = providers.get(id);
           started.add(id);
           activeTickets.add(id);
           if (fileKey !== null) busy.add(fileKey);
           if (file) runningByBlock.set(file.block, (runningByBlock.get(file.block) ?? 0) + 1);
+          if (provider) admittedByProvider.set(provider, (admittedByProvider.get(provider) ?? 0) + 1);
           const slot = runOne(id)
             .then((n) => {
               done += n;
@@ -707,6 +813,7 @@ export async function createWorker(opts, deps = {}) {
               fileKeys.delete(id);
               if (fileKey !== null) busy.delete(fileKey);
               if (file) runningByBlock.set(file.block, runningByBlock.get(file.block) - 1);
+              if (provider) admittedByProvider.set(provider, admittedByProvider.get(provider) - 1);
             });
           running.set(id, slot);
         }
@@ -769,14 +876,22 @@ export async function createWorker(opts, deps = {}) {
     heartbeat = null;
   }
 
+  async function startDaemon() {
+    if (daemonStarted) return;
+    daemonStarted = true;
+    stopDaemon = await startNativeDaemon(runId, cfg, env);
+    if (stopping) await stopDaemon();
+  }
+
   /** Serve the queue until `stop()`. */
   async function run() {
-    announce();
     let interval = pollMs;
     let polled = false;
     /** @type {import('node:fs').FSWatcher | null} */
     let queueWatch = null;
     try {
+      await startDaemon();
+      announce();
       // Enqueue publishes a ticket by rename. Wake even during the longest idle back-off;
       // heartbeat and temporary files do not wake an idle worker.
       try {
@@ -818,37 +933,42 @@ export async function createWorker(opts, deps = {}) {
     } finally {
       queueWatch?.close();
       release();
+      await stopDaemon();
     }
   }
 
   /** Drain the queue once while announced, then release (`worker --once`). */
   async function once() {
-    announce();
     try {
+      await startDaemon();
+      announce();
       const done = await drain();
       if (done > 0) await cleanOrphans();
       return done;
     } finally {
       stop();
       release();
+      await stopDaemon();
     }
   }
 
   /** Stop taking tickets; the running ones are awaited (a killed one is abandoned, see `processTicket`). */
   function stop() {
     stopping = true;
+    void stopDaemon();
     wake?.();
     poolWake?.();
   }
 
   /** Withdraw this process's announcement (`worker.json`) if it is still ours. */
   function release() {
+    void stopDaemon();
     stopBeating();
     if (self !== null) retractWorker(repoRoot, self.pid, self.start_time);
     self = null;
   }
 
-  return { processTicket, drain, run, once, stop, release, announce, watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
+  return { processTicket, drain, run, once, stop, release, announce, startDaemon, stopDaemon: () => stopDaemon(), watchTick: tickWatch, jevKeyResolved: jevKey !== null, sessionEnv: childEnv };
 }
 
 /**
@@ -890,7 +1010,17 @@ export async function bootWorker(opts, deps = {}) {
     if (entries.some((entry) => entry.live && !entry.worker) || (legacy && legacy.run !== opts.runId) || entries.some((entry) => entry.live && entry.worker?.run !== opts.runId) || count >= limit) {
       throw new WorkerError('worker-running', `review.workers limit ${limit}: a live or unreadable worker entry already serves this queue`);
     }
-    worker.announce();
+    // Startup must finish before worker.json signals readiness: an owner may
+    // remove HOME immediately after that announcement to stop an orphan worker.
+    try {
+      await worker.startDaemon();
+      worker.announce();
+    } catch (error) {
+      worker.stop();
+      worker.release();
+      await worker.stopDaemon();
+      throw error;
+    }
   });
   return { ...worker, repoRoot, runRootDir };
 }

package/src/worker/queue.mjs

@@ -520,8 +520,9 @@ export function pendingTickets(repoRoot) {
     return [];
   }
   const claimed = new Set(names.map((n) => CLAIM.exec(n)?.[1]));
+  const present = new Set(names);
   const ids = names.filter((n) => /^[0-9a-f]{24}\.json$/.test(n)).map((n) => n.slice(0, 24));
-  const pending = ids.filter((id) => !names.includes(`${id}.done`) && !claimed.has(id));
+  const pending = ids.filter((id) => !present.has(`${id}.done`) && !claimed.has(id));
   const at = (/** @type {string} */ id) => {
     const t = readJSON(path.join(queueDir(repoRoot), `${id}.json`));
     return typeof t?.enqueued_at === 'string' ? t.enqueued_at : '';
@@ -589,7 +590,7 @@ export function readTicket(repoRoot, id) {
   const t = readJSON(path.join(queueDir(repoRoot), `${assertTicketId(id)}.json`));
   if (!t || typeof t !== 'object') throw new WorkerError('bad-ticket', `ticket ${id} is not readable JSON`);
   assertRowPath(t.file);
-  if (t.tree !== undefined && (typeof t.tree !== 'string' || !path.isAbsolute(t.tree))) throw new WorkerError('bad-ticket', `ticket ${id} names a tree that is not an absolute path`);
+  if (t.tree != null && (typeof t.tree !== 'string' || !path.isAbsolute(t.tree))) throw new WorkerError('bad-ticket', `ticket ${id} names a tree that is not an absolute path`);
   if (typeof t.run !== 'string' || typeof t.block !== 'string' || typeof t.content_hash !== 'string') {
     throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
   }

package/src/worker/review-state.mjs

@@ -25,7 +25,7 @@
  * (the engine saves it as a new seq and writes `review.state_restored`); an anchor with no body
  * makes a missing file `lost` (refused, never a round 1). A
  * `review.state` row of the run whose MAC does not verify is `tampered` for every block and file
- * of it (`blockAnchors`). Whether the block's one L3 rung was used comes from the anchors, never
+ * of it (`blockAnchors`). Whether a file's one L3 rung was used comes from the anchors, never
  * from the state files.
  *
  * B78: the reviewed text itself is kept as a git blob in the content root's object store
@@ -427,8 +427,8 @@ export async function saveState({ runId, block, file, key, state, seq, writeRow,
 }
 
 /**
- * Whether the block's one L3 rung (§3.6 rules 5/6) was already used on ANY of its files — the
- * rung is once per block. Derived from the ledger anchors only; fails closed: a block anchor that
+ * Whether any file of the block has used its L3 rung (§3.6 rules 5/6).
+ * Derived from the ledger anchors only; fails closed: a block anchor that
  * does not verify is `tampered`.
  * @param {{runId: string, block: string, key: Buffer, rows: Array<Record<string, any>>}} opts
  * @returns {{status: 'ok' | 'tampered', used: boolean}}
@@ -438,3 +438,15 @@ export function blockRungUsed(opts) {
   if (anchors === null) return { status: 'tampered', used: true };
   return { status: 'ok', used: anchors.some((a) => a.l3_rung_used === true) };
 }
+
+/**
+ * Whether this file has used its L3 rung, once per file across all its anchors.
+ * Verify the run's anchors before filtering by file, preserving fail-closed tamper detection.
+ * @param {{runId: string, block: string, file: string, key: Buffer, rows: Array<Record<string, any>>}} opts
+ * @returns {{status: 'ok' | 'tampered', used: boolean}}
+ */
+export function fileRungUsed(opts) {
+  const anchors = blockAnchors(opts);
+  if (anchors === null) return { status: 'tampered', used: true };
+  return { status: 'ok', used: anchors.some((a) => a.file === opts.file && a.l3_rung_used === true) };
+}