@codedology/code-forge 0.3.2 → 0.4.0

package/src/worker/loop.mjs

@@ -14,6 +14,12 @@
  * Then it polls the queue, hands each ticket to the engine hook (`./engine.mjs`, B12a), signs the
  * outcome with the run key and writes it with its done marker and a signed ledger row.
  *
+ * Config (B35): the worker reviews each ticket with the run's config snapshot, read from the run
+ * record per ticket — the ticket's pin when `run reload` pinned it (it was queued before the
+ * reload), else the snapshot in force. So `run reload` reaches the worker without a restart, and a
+ * restarted worker boots on the snapshot, not on whatever the file says now. A record from before
+ * snapshots (or none) keeps the config the worker booted with.
+ *
  * Restart: tickets live on disk and are idempotent by content hash, so a new worker simply re-reads
  * every ticket without a done marker. A worker that is stopping abandons its in-flight ticket (no
  * done marker) instead of recording the killed session as a failure. The new worker's pid is
@@ -27,18 +33,25 @@ import { loadProjectConfig } from '../config/load.mjs';
 import { createDefaultKeyStore, resolveKey } from '../keys/store.mjs';
 import { appendRow, readAllRows } from '../ledger/write.mjs';
 import { spawnSession } from '../session/spawn.mjs';
+import { snapshotFor } from '../state/config-snapshot.mjs';
 import { readRun, reattachWorker } from '../state/run.mjs';
 import { loadKey, signRow } from '../state/signer.mjs';
 import { runRoot, setRunRoot } from '../util/tmp.mjs';
 import { reviewTicket } from './engine.mjs';
-import { announceWorker, liveWorker, pendingTickets, readTicket, retractWorker, writeResult } from './queue.mjs';
+import { announceWorker, beatWorker, HEARTBEAT_MS, liveWorker, pendingTickets, readTicket, retractWorker, writeResult } from './queue.mjs';
 import { contentHash, DELETED_HASH, repoRootOf, WorkerError } from './ticket.mjs';
 
 /** Name of the key the worker resolves for System 1 (`keys.jev` / `system1.key` reference). */
 export const JEV_KEY_NAME = 'jev';
 
 export const DEFAULT_POLL_MS = 250;
-export const DEFAULT_SESSION_TIMEOUT_S = 600;
+export const DEFAULT_SESSION_TIMEOUT_S = 300;
+
+/** B35: how long `configFor` waits before its one retry of a failed run-record read. */
+export const CONFIG_RETRY_MS = 200;
+
+/** The shortest heartbeat interval a worker accepts; anything else falls back to `HEARTBEAT_MS`. */
+export const MIN_HEARTBEAT_MS = 100;
 
 /**
  * Provider-CLI auth variables a reviewer CLI may legitimately need (the reviewer runs on the
@@ -104,6 +117,8 @@ export function loopResult(outcome) {
  * @property {string} slug - ledger project slug (the run record's `project`).
  * @property {Buffer} key - the run's HMAC key.
  * @property {number} [pollMs]
+ * @property {number} [heartbeatMs] - how often `heartbeat_at` is refreshed: an integer ≥ `MIN_HEARTBEAT_MS`,
+ *   else `HEARTBEAT_MS`.
  */
 
 /**
@@ -135,15 +150,86 @@ export async function createWorker(opts, deps = {}) {
   const store = deps.store ?? (await createDefaultKeyStore(env));
   const jevKey = await resolveJevKey(cfg, { store, env, opRead: deps.opRead });
   const childEnv = sessionEnv(env, [jevKey, key.toString('hex')]);
-  const timeoutS = Number.isInteger(cfg?.review?.session_timeout_s) && cfg.review.session_timeout_s > 0 ? cfg.review.session_timeout_s : DEFAULT_SESSION_TIMEOUT_S;
   const pollMs = opts.pollMs ?? DEFAULT_POLL_MS;
+  const heartbeatMs = Number.isInteger(opts.heartbeatMs) && /** @type {number} */ (opts.heartbeatMs) >= MIN_HEARTBEAT_MS ? /** @type {number} */ (opts.heartbeatMs) : HEARTBEAT_MS;
   let stopping = false;
+  /** @type {ReturnType<typeof setInterval> | null} */
+  let heartbeat = null;
   /** @type {(() => void) | null} */
   let wake = null;
 
   /** @param {Record<string, any>} row */
   const ledger = (row) => writeRow(signRow({ run: runId, ...row }, key));
 
+  /**
+   * The config ticket `id` is reviewed with (B35): its snapshot from the run record, re-read per
+   * ticket; the boot `cfg` when the run has no record or the record has no snapshot.
+   * @param {string} id
+   * @returns {Promise<Record<string, any>>}
+   * @throws {Error} `config-snapshot` — a record that cannot be read, or a missing/altered snapshot.
+   */
+  async function configFor(id) {
+    // The Jev key and the run root were settled at boot from `cfg`; a reload can never change
+    // them, because `keys`, `system1.key` and `tmp.root` are fixed keys `run reload` refuses.
+    let record;
+    for (let attempt = 1; ; attempt += 1) {
+      try {
+        record = await readRun(runId);
+        break;
+      } catch (err) {
+        if (/** @type {any} */ (err)?.code === 'no-run') return cfg;
+        // a transient read error (e.g. the record caught mid-replace) gets ONE retry
+        if (attempt >= 2) throw new Error('config-snapshot');
+        await new Promise((resolve) => setTimeout(resolve, CONFIG_RETRY_MS));
+      }
+    }
+    return snapshotFor(record, id)?.config ?? cfg;
+  }
+
+  /**
+   * One ticket through the engine hook with `tcfg` (the ticket's config snapshot): the unsigned result.
+   * @param {import('./queue.mjs').Ticket} ticket @param {Record<string, any>} base
+   * @param {Record<string, any>} tcfg
+   * @returns {Promise<Record<string, any>>}
+   */
+  async function reviewWith(ticket, base, tcfg) {
+    const timeoutS = Number.isInteger(tcfg?.review?.session_timeout_s) && tcfg.review.session_timeout_s > 0 ? tcfg.review.session_timeout_s : DEFAULT_SESSION_TIMEOUT_S;
+    /** @type {import('./engine.mjs').ReviewContext} */
+    const ctx = {
+      repoRoot,
+      runId,
+      runRootDir,
+      cfg: tcfg,
+      jevKey,
+      key, // signs the fix-loop state (B12c); the sessions never see it (`childEnv`)
+      readRows,
+      ...(deps.jev ? { jev: deps.jev } : {}),
+      // `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
+      // over the ticket's; everything else the worker pins.
+      spawn: (sessionOpts) =>
+        spawn(
+          /** @type {any} */ ({ cfg: tcfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
+          { env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
+        ),
+      writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
+    };
+    try {
+      const outcome = await review(ticket, ctx);
+      return {
+        ...base,
+        status: outcome.status,
+        approved: outcome.approved === true,
+        engine: outcome.engine,
+        ...(typeof outcome.reason === 'string' ? { reason: outcome.reason } : {}),
+        ...(Array.isArray(outcome.findings) ? { findings: outcome.findings } : {}),
+        ...loopResult(outcome),
+        sessions: outcome.sessions,
+      };
+    } catch {
+      return { ...base, status: 'unavailable', reason: 'engine-error', approved: false };
+    }
+  }
+
   /**
    * @param {string} id
    * @returns {Promise<Record<string, any> | null>} the signed result, or null when abandoned.
@@ -175,40 +261,14 @@ export async function createWorker(opts, deps = {}) {
       } else if (current !== ticket.content_hash) {
         result = { ...base, status: 'stale', approved: false };
       } else {
-        /** @type {import('./engine.mjs').ReviewContext} */
-        const ctx = {
-          repoRoot,
-          runId,
-          runRootDir,
-          cfg,
-          jevKey,
-          key, // signs the fix-loop state (B12c); the sessions never see it (`childEnv`)
-          readRows,
-          ...(deps.jev ? { jev: deps.jev } : {}),
-          // `sessionOpts.cfg` (a per-call config, e.g. the consensus second reviewer's level) wins
-          // over the worker's; everything else the worker pins.
-          spawn: (sessionOpts) =>
-            spawn(
-              /** @type {any} */ ({ cfg, ...sessionOpts, runRoot: runRootDir, run: runId, block: ticket.block, slug, timeoutMs: timeoutS * 1000 }),
-              { env: childEnv, ...(deps.bins ? { bins: deps.bins } : {}), ...(deps.stderr ? { stderr: deps.stderr } : {}) },
-            ),
-          writeRow: (row) => ledger({ ...row, block: ticket.block, file: ticket.file, content_hash: ticket.content_hash }),
-        };
+        /** @type {Record<string, any> | null} */
+        let ticketCfg = null;
         try {
-          const outcome = await review(ticket, ctx);
-          result = {
-            ...base,
-            status: outcome.status,
-            approved: outcome.approved === true,
-            engine: outcome.engine,
-            ...(typeof outcome.reason === 'string' ? { reason: outcome.reason } : {}),
-            ...(Array.isArray(outcome.findings) ? { findings: outcome.findings } : {}),
-            ...loopResult(outcome),
-            sessions: outcome.sessions,
-          };
+          ticketCfg = await configFor(id);
         } catch {
-          result = { ...base, status: 'unavailable', reason: 'engine-error', approved: false };
+          result = { ...base, status: 'unavailable', reason: 'config-snapshot', approved: false };
         }
+        if (ticketCfg) result = await reviewWith(ticket, base, ticketCfg);
       }
     }
     if (stopping) return null; // the session was killed by our own stop: leave the ticket queued
@@ -267,9 +327,34 @@ export async function createWorker(opts, deps = {}) {
     return done;
   }
 
+  /**
+   * Announce this worker and keep its heartbeat beating on a timer of its own (B30), independent
+   * of the ticket being processed: a review session can run for minutes, and `review-file --wait`
+   * must still see the worker as live the whole time.
+   */
+  function announce() {
+    const self = announceWorker(repoRoot, { pid: process.pid, run: runId });
+    stopBeating();
+    const timer = setInterval(() => {
+      try {
+        // another worker announced itself over us: stop beating, never overwrite its file
+        if (!beatWorker(repoRoot, self)) stopBeating();
+      } catch {
+        // a missed beat (an fs error) is retried on the next tick; the poll loop goes on
+      }
+    }, heartbeatMs);
+    timer.unref();
+    heartbeat = timer;
+  }
+
+  function stopBeating() {
+    if (heartbeat !== null) clearInterval(heartbeat);
+    heartbeat = null;
+  }
+
   /** Serve the queue until `stop()`. */
   async function run() {
-    announceWorker(repoRoot, { pid: process.pid, run: runId });
+    announce();
     try {
       while (!stopping) {
         try {
@@ -294,7 +379,7 @@ export async function createWorker(opts, deps = {}) {
 
   /** Drain the queue once while announced, then release (`worker --once`). */
   async function once() {
-    announceWorker(repoRoot, { pid: process.pid, run: runId });
+    announce();
     try {
       return await drain();
     } finally {
@@ -310,6 +395,7 @@ export async function createWorker(opts, deps = {}) {
 
   /** Withdraw this process's announcement (`worker.json`) if it is still ours. */
   function release() {
+    stopBeating();
     retractWorker(repoRoot, process.pid);
   }
 
@@ -325,11 +411,21 @@ export async function bootWorker(opts, deps = {}) {
   const record = await readRun(opts.runId);
   if (record.status !== 'active') throw new WorkerError('run-ended', `run ${opts.runId} has ended`);
   const repoRoot = await repoRootOf(path.resolve(opts.cwd ?? record.workspace));
-  const loaded = await loadProjectConfig(repoRoot);
-  if (!loaded.ok) throw new WorkerError('config', loaded.message ?? 'the project config cannot be loaded');
+  // B35: a run with a config snapshot boots on it (the snapshot in force, not on whatever the file
+  // says now); a run from before snapshots boots on the file, as before.
+  let cfg;
+  try {
+    cfg = snapshotFor(record, '')?.config;
+  } catch {
+    throw new WorkerError('config', `run ${opts.runId}: the config snapshot in the run record is missing or altered`);
+  }
+  if (!cfg) {
+    const loaded = await loadProjectConfig(repoRoot);
+    if (!loaded.ok) throw new WorkerError('config', loaded.message ?? 'the project config cannot be loaded');
+    cfg = loaded.config;
+  }
   const other = liveWorker(repoRoot);
   if (other && other.pid !== process.pid) throw new WorkerError('worker-running', `worker pid ${other.pid} already serves this queue`);
-  const cfg = loaded.config;
   const tmpRoot = typeof cfg?.tmp?.root === 'string' ? cfg.tmp.root : undefined;
   const runRootDir = runRoot(opts.runId, { root: tmpRoot });
   setRunRoot(runRootDir);

package/src/worker/queue.mjs

@@ -4,7 +4,7 @@
  *
  *   <repo>/.code-forge/queue/<ticket>.json   the ticket (written by `review-file`, idempotent)
  *   <repo>/.code-forge/queue/<ticket>.done   the worker's done marker `{ticket, status, result}`
- *   <repo>/.code-forge/queue/worker.json     the worker's announcement `{pid, start_time, run, started_at}`
+ *   <repo>/.code-forge/queue/worker.json     the worker's announcement `{pid, start_time, run, started_at, heartbeat_at}`
  *   <repo>/.code-forge/reviews/<run>/<ticket>.json   the SIGNED result (run key HMAC, §8.6)
  *
  * The forbidden list (§8.4 ★) denies a coder any write under `reviews/` and to `queue/*.done`;
@@ -20,6 +20,12 @@ import { assertRowPath, assertTicketId, contentHash, ticketId, WorkerError } fro
 
 export const WORKER_FILE = 'worker.json';
 
+/** How often a serving worker refreshes `heartbeat_at` (B30). */
+export const HEARTBEAT_MS = 5000;
+
+/** A heartbeat older than this is not fresh (twelve missed beats). */
+export const HEARTBEAT_STALE_MS = 60_000;
+
 /** @param {string} repoRoot */
 export const queueDir = (repoRoot) => path.join(repoRoot, '.code-forge', 'queue');
 /** @param {string} repoRoot @param {string} runId */
@@ -156,12 +162,60 @@ export async function verifyResult(repoRoot, runId, ticket) {
  * @param {string} repoRoot @param {{pid: number, run: string}} who
  */
 export function announceWorker(repoRoot, { pid, run }) {
-  const body = { pid, start_time: readStartTime(pid) ?? UNKNOWN_START_TIME, run, started_at: new Date().toISOString() };
+  const now = new Date().toISOString();
+  const body = { pid, start_time: readStartTime(pid) ?? UNKNOWN_START_TIME, run, started_at: now, heartbeat_at: now };
   writeAtomic(path.join(queueDir(repoRoot), WORKER_FILE), `${JSON.stringify(body)}\n`);
   return body;
 }
 
-/** @param {string} repoRoot @returns {{pid: number, start_time: string, run: string} | null} */
+/**
+ * Refresh the announcement's `heartbeat_at` (B30) while it is still THIS worker's: the file is
+ * re-read right before the write and must name the same `pid` AND `start_time` the worker
+ * announced. Another worker's announcement is never overwritten (the caller stops beating).
+ * The beat is written to a temp file, the announcement re-read and compared once more, and only
+ * then renamed over it. The worker calls this from a timer of its own, so it keeps beating while
+ * a long review session is awaited.
+ * @param {string} repoRoot @param {{pid: number, start_time: string}} self - what `announceWorker` returned.
+ * @param {Date} [now]
+ * @returns {boolean} whether a beat was written.
+ */
+export function beatWorker(repoRoot, self, now = new Date()) {
+  const mine = (/** @type {Record<string, any> | null} */ w) => w?.pid === self.pid && w.start_time === self.start_time;
+  const w = readWorker(repoRoot);
+  if (!mine(w)) return false;
+  const file = path.join(queueDir(repoRoot), WORKER_FILE);
+  const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
+  try {
+    writeFileSync(tmp, `${JSON.stringify({ ...w, heartbeat_at: now.toISOString() })}\n`, { mode: 0o644 });
+    // re-read right before the rename: an announcement that changed meanwhile is never replaced
+    if (!mine(readWorker(repoRoot))) {
+      rmSync(tmp, { force: true });
+      return false;
+    }
+    renameSync(tmp, file);
+    return true;
+  } catch (err) {
+    rmSync(tmp, { force: true });
+    throw err;
+  }
+}
+
+/** A heartbeat further than this in the future is not fresh (clock skew tolerance). */
+export const HEARTBEAT_FUTURE_MS = 5000;
+
+/**
+ * Whether the announcement's heartbeat is fresh: at most `HEARTBEAT_STALE_MS` old and at most
+ * `HEARTBEAT_FUTURE_MS` ahead of our clock (a heartbeat far in the future is forged or skewed).
+ * @param {{heartbeat_at?: unknown}} w @param {number} [nowMs]
+ */
+export function heartbeatFresh(w, nowMs = Date.now()) {
+  const at = typeof w?.heartbeat_at === 'string' ? Date.parse(w.heartbeat_at) : NaN;
+  if (!Number.isFinite(at)) return false;
+  const age = nowMs - at;
+  return age <= HEARTBEAT_STALE_MS && age >= -HEARTBEAT_FUTURE_MS;
+}
+
+/** @param {string} repoRoot @returns {{pid: number, start_time: string, run: string, heartbeat_at?: string} | null} */
 export function readWorker(repoRoot) {
   const w = readJSON(path.join(queueDir(repoRoot), WORKER_FILE));
   return w && Number.isInteger(w.pid) && w.pid > 1 && typeof w.run === 'string' ? w : null;
@@ -173,13 +227,20 @@ export function retractWorker(repoRoot, pid) {
 }
 
 /**
- * The live worker serving this queue, or null (`worker_down`): the announced pid must be alive
- * AND still have the announced start time (a recycled pid is not our worker).
- * @param {string} repoRoot
+ * The live worker serving this queue, or null (`worker_down`): the announced pid must be alive,
+ * AND either its heartbeat is fresh (B30: the worker itself wrote it within `HEARTBEAT_STALE_MS`)
+ * or it still has the announced start time (a recycled pid is not our worker). A fresh heartbeat
+ * wins over the start-time check: `ps` can fail or print another time zone's `lstart` inside a
+ * coder's sandbox, which made a live worker look down mid-review (issue #2).
+ * Accepted trade-off: a worker that is SIGKILLed (no retraction) leaves a heartbeat that stays
+ * fresh for up to `HEARTBEAT_STALE_MS` (60 s); if the OS reuses its pid inside that window, the
+ * queue reads as served until the heartbeat goes stale. A clean stop retracts the file at once.
+ * @param {string} repoRoot @param {{nowMs?: number}} [opts]
  */
-export function liveWorker(repoRoot) {
+export function liveWorker(repoRoot, opts = {}) {
   const w = readWorker(repoRoot);
   if (!w || !isAlive(w.pid)) return null;
+  if (heartbeatFresh(w, opts.nowMs)) return w;
   if (w.start_time !== UNKNOWN_START_TIME && readStartTime(w.pid) !== w.start_time) return null;
   return w;
 }