@codedology/code-forge 0.12.0 → 0.14.0

package/src/worker/queue.mjs

@@ -5,6 +5,7 @@
  *   <repo>/.code-forge/queue/<ticket>.json   the ticket (written by `review-file`, idempotent)
  *   <repo>/.code-forge/queue/<ticket>.done   the worker's done marker `{ticket, status, result}`
  *   <repo>/.code-forge/queue/worker.json     the worker's announcement `{pid, start_time, run, started_at, heartbeat_at}`
+ *   <repo>/.code-forge/workers/<pid>-<start_time>.json   each worker's private registry entry
  *   <repo>/.code-forge/reviews/<run>/<ticket>.json   the SIGNED result (run key HMAC, §8.6)
  *
  * The forbidden list (§8.4 ★) denies a coder any write under `reviews/` and to `queue/*.done`;
@@ -28,7 +29,7 @@
  */
 
 import { createHash, randomBytes } from 'node:crypto';
-import { existsSync, linkSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { chmodSync, existsSync, linkSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import { loadKey, signRow, verifyRow } from '../state/signer.mjs';
 import { isAlive, readStartTime, UNKNOWN_START_TIME } from '../util/reaper.mjs';
@@ -44,15 +45,19 @@ export const HEARTBEAT_STALE_MS = 60_000;
 
 /** @param {string} repoRoot */
 export const queueDir = (repoRoot) => path.join(repoRoot, '.code-forge', 'queue');
+/** @param {string} repoRoot */
+const workersDir = (repoRoot) => path.join(repoRoot, '.code-forge', 'workers');
+/** @param {string} repoRoot @param {{pid: number, start_time: string}} self */
+const workerPath = (repoRoot, self) => path.join(workersDir(repoRoot), `${self.pid}-${self.start_time}.json`);
 /** @param {string} repoRoot @param {string} runId */
 export const reviewsDir = (repoRoot, runId) => path.join(repoRoot, '.code-forge', 'reviews', runId);
 
-/** @param {string} file @param {string} text */
-function writeAtomic(file, text) {
+/** @param {string} file @param {string} text @param {number} [mode] */
+function writeAtomic(file, text, mode = 0o644) {
   mkdirSync(path.dirname(file), { recursive: true });
   const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
   try {
-    writeFileSync(tmp, text, { mode: 0o644 });
+    writeFileSync(tmp, text, { mode });
     renameSync(tmp, file);
   } catch (err) {
     rmSync(tmp, { force: true });
@@ -295,11 +300,13 @@ export function clearDone({ repoRoot, run, block, file, tree = null }) {
  * A crash between the drop and the caller's enqueue leaves a re-runnable ticket with no marker:
  * harmless — its ticket file is pending, and the next `review-file` queues it anyway. Synchronous.
  * @param {string} repoRoot @param {string} ticket @param {Buffer} seen
- * @param {{link?: (from: string, to: string) => void, read?: (file: string) => Buffer, rename?: (from: string, to: string) => void, remove?: (file: string) => void, write?: (file: string, text: string) => void}} [io] - for
+ * `once` records the join's one retry before releasing the claim. An existing retry marker
+ * keeps the result, including when another join has already finished the rerun.
+ * @param {{once?: boolean, link?: (from: string, to: string) => void, read?: (file: string) => Buffer, rename?: (from: string, to: string) => void, remove?: (file: string) => void, write?: (file: string, text: string) => void}} [io] - for
  *   tests (default `linkSync`, `readFileSync`, `renameSync`, `rmSync` with force, `writeFileSync`
  *   with flag `wx` and mode 0o600). A claim that cannot be read is restored the same way and
  *   refused `fresh-restore-failed`.
- * @returns {'dropped' | 'gone' | 'changed'}
+ * @returns {'dropped' | 'gone' | 'changed' | 'kept'}
  * @throws {WorkerError} `fresh-claim-failed` (the claim could not be taken, or not removed after
  *   the drop; nothing was queued) or `fresh-restore-failed` (the message says which state was left:
  *   put back, a newer marker, a refused marker, the claim kept, or nothing). A message names an
@@ -307,6 +314,8 @@ export function clearDone({ repoRoot, run, block, file, tree = null }) {
  */
 export function dropDoneIf(repoRoot, ticket, seen, io = {}) {
   const marker = path.join(queueDir(repoRoot), `${assertTicketId(ticket)}.done`);
+  const retried = path.join(queueDir(repoRoot), `${ticket}.retried`);
+  if (io.once && existsSync(retried)) return 'kept';
   const claim = `${marker}.${process.pid}.${randomBytes(4).toString('hex')}.fresh`;
   const remove = io.remove ?? ((/** @type {string} */ file) => rmSync(file, { force: true }));
   try {
@@ -324,6 +333,16 @@ export function dropDoneIf(repoRoot, ticket, seen, io = {}) {
   } catch (err) {
     readFailed = codeOf(err);
   }
+  let retryFailed = null;
+  if (same && io.once) {
+    try {
+      writeFileSync(retried, `${ticket}\n`, { flag: 'wx', mode: 0o600 });
+    } catch (err) {
+      // Restore the claim before keeping or refusing it. Only one join can spend the retry.
+      retryFailed = codeOf(err);
+      same = false;
+    }
+  }
   if (same) {
     try {
       remove(claim);
@@ -350,6 +369,8 @@ export function dropDoneIf(repoRoot, ticket, seen, io = {}) {
     // the marker is back (or a newer one won), so the ticket is done; `clearDone` removes the claim
   }
   if (readFailed !== null) throw new WorkerError('fresh-restore-failed', `${unread}; ${linkFailed === 'EEXIST' ? 'a newer marker is in place' : 'it was put back'}; nothing was queued`);
+  if (retryFailed === 'EEXIST') return 'kept';
+  if (retryFailed !== null) throw new WorkerError('fresh-claim-failed', `the retry marker of ticket ${ticket} could not be written (${retryFailed}); the done marker was restored; nothing was queued`);
   return 'changed';
 }
 
@@ -427,6 +448,36 @@ export function blockTickets({ repoRoot, run, block, tree = null }) {
     .sort((a, b) => a.enqueued_at.localeCompare(b.enqueued_at) || a.ticket.localeCompare(b.ticket));
 }
 
+/**
+ * Atomically record the join's findings handoff once. Read only this marker, never a result.
+ * @param {{repoRoot: string, block: string, file: string, ticket: string}} opts
+ * @returns {boolean} true only for the join that first delivered the findings.
+ */
+export function deliverBlockFindings({ repoRoot, block, file, ticket }) {
+  const marker = path.join(queueDir(repoRoot), `${assertTicketId(ticket)}.delivered`);
+  const expected = { block, file, ticket };
+  const delivered = () => {
+    try {
+      const seen = JSON.parse(readFileSync(marker, 'utf8'));
+      if (seen?.block === block && seen.file === file && seen.ticket === ticket) return true;
+    } catch (err) {
+      if (err.code === 'ENOENT') return false;
+    }
+    throw new WorkerError('result-unreadable', 'a findings delivery marker cannot be read');
+  };
+  if (delivered()) return false;
+  const tmp = `${marker}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
+  try {
+    writeFileSync(tmp, `${JSON.stringify(expected)}\n`, { flag: 'wx' });
+    try { linkSync(tmp, marker); } catch (err) {
+      if (err.code !== 'EEXIST') throw err;
+      if (delivered()) return false;
+      throw new WorkerError('result-unreadable', 'a findings delivery marker disappeared');
+    }
+    return true;
+  } finally { rmSync(tmp, { force: true }); }
+}
+
 /**
  * Read and re-validate a ticket (coder-writable): its id must match its content.
  * @param {string} repoRoot @param {string} id
@@ -468,7 +519,8 @@ export function writeResult({ repoRoot, runId, ticket, result, key }) {
   const signed = signRow({ ...result, run: runId, ticket: assertTicketId(ticket) }, key);
   const file = path.join(reviewsDir(repoRoot, runId), `${ticket}.json`);
   writeAtomic(file, `${JSON.stringify(signed)}\n`);
-  writeAtomic(path.join(queueDir(repoRoot), `${ticket}.done`), `${JSON.stringify({ ticket, status: signed.status, result: path.relative(repoRoot, file) })}\n`);
+  const openFindings = signed.status === 'reviewed' && signed.approved !== true && Array.isArray(signed.findings) && signed.findings.length > 0;
+  writeAtomic(path.join(queueDir(repoRoot), `${ticket}.done`), `${JSON.stringify({ ticket, status: signed.status, ...(openFindings ? { open_findings: true } : {}), result: path.relative(repoRoot, file) })}\n`);
   return signed;
 }
 
@@ -497,20 +549,25 @@ export async function verifyResult(repoRoot, runId, ticket) {
 }
 
 /**
- * Announce the serving worker in the queue directory (what `review-file` checks for liveness).
+ * Announce the serving worker in the private registry and legacy queue file.
  * @param {string} repoRoot @param {{pid: number, run: string}} who
  */
 export function announceWorker(repoRoot, { pid, run }) {
   const now = new Date().toISOString();
   const body = { pid, start_time: readStartTime(pid) ?? UNKNOWN_START_TIME, run, started_at: now, heartbeat_at: now };
-  writeAtomic(path.join(queueDir(repoRoot), WORKER_FILE), `${JSON.stringify(body)}\n`);
+  mkdirSync(workersDir(repoRoot), { recursive: true, mode: 0o700 });
+  chmodSync(workersDir(repoRoot), 0o700);
+  writeAtomic(workerPath(repoRoot, body), `${JSON.stringify(body)}\n`, 0o600);
+  const legacy = readWorker(repoRoot);
+  if (!legacy || !workerLive(legacy)) writeAtomic(path.join(queueDir(repoRoot), WORKER_FILE), `${JSON.stringify(body)}\n`);
   return body;
 }
 
 /**
- * Refresh the announcement's `heartbeat_at` (B30) while it is still THIS worker's: the file is
+ * Refresh both announcements' `heartbeat_at` (B30) while each is still THIS worker's: each file is
  * re-read right before the write and must name the same `pid` AND `start_time` the worker
- * announced. Another worker's announcement is never overwritten (the caller stops beating).
+ * announced. Another worker's announcement is never overwritten. A worker keeps beating its
+ * registry entry even when another worker owns worker.json.
  * The beat is written to a temp file, the announcement re-read and compared once more, and only
  * then renamed over it. The worker calls this from a timer of its own, so it keeps beating while
  * a long review session is awaited.
@@ -520,23 +577,23 @@ export function announceWorker(repoRoot, { pid, run }) {
  */
 export function beatWorker(repoRoot, self, now = new Date()) {
   const mine = (/** @type {Record<string, any> | null} */ w) => w?.pid === self.pid && w.start_time === self.start_time;
-  const w = readWorker(repoRoot);
-  if (!mine(w)) return false;
-  const file = path.join(queueDir(repoRoot), WORKER_FILE);
-  const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
-  try {
-    writeFileSync(tmp, `${JSON.stringify({ ...w, heartbeat_at: now.toISOString() })}\n`, { mode: 0o644 });
-    // re-read right before the rename: an announcement that changed meanwhile is never replaced
-    if (!mine(readWorker(repoRoot))) {
+  let written = false;
+  const registryFile = workerPath(repoRoot, self);
+  for (const file of [registryFile, path.join(queueDir(repoRoot), WORKER_FILE)]) {
+    const w = readJSON(file);
+    if (!mine(w)) continue;
+    const tmp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
+    try {
+      writeFileSync(tmp, `${JSON.stringify({ ...w, heartbeat_at: now.toISOString() })}\n`, { mode: file === registryFile ? 0o600 : 0o644 });
+      // Re-read before rename; another owner's announcement is never replaced.
+      if (!mine(readJSON(file))) continue;
+      renameSync(tmp, file);
+      written = true;
+    } finally {
       rmSync(tmp, { force: true });
-      return false;
     }
-    renameSync(tmp, file);
-    return true;
-  } catch (err) {
-    rmSync(tmp, { force: true });
-    throw err;
   }
+  return written;
 }
 
 /** A heartbeat further than this in the future is not fresh (clock skew tolerance). */
@@ -560,9 +617,59 @@ export function readWorker(repoRoot) {
   return w && Number.isInteger(w.pid) && w.pid > 1 && typeof w.run === 'string' ? w : null;
 }
 
-/** Remove the announcement if it is still `pid`'s. @param {string} repoRoot @param {number} pid */
-export function retractWorker(repoRoot, pid) {
-  if (readWorker(repoRoot)?.pid === pid) rmSync(path.join(queueDir(repoRoot), WORKER_FILE), { force: true });
+/** Remove only this owner's announcements. @param {string} repoRoot @param {number} pid @param {string} [startTime] */
+export function retractWorker(repoRoot, pid, startTime) {
+  const mine = (w) => w?.pid === pid && (startTime === undefined || w.start_time === startTime);
+  for (const entry of workerRegistry(repoRoot)) {
+    if (mine(entry.worker)) rmSync(entry.file, { force: true });
+  }
+  if (mine(readWorker(repoRoot))) rmSync(path.join(queueDir(repoRoot), WORKER_FILE), { force: true });
+}
+
+/** @typedef {{pid: number, start_time: string, run: string, started_at: string, heartbeat_at: string}} WorkerAnnouncement */
+
+/** Bad entries remain occupied until a human fixes them. @param {any} w @returns {w is WorkerAnnouncement} */
+function workerShape(w) {
+  return w && Number.isInteger(w.pid) && w.pid > 1 && typeof w.start_time === 'string' && w.start_time.length > 0
+    && !/[\\/]/.test(w.start_time) && typeof w.run === 'string' && w.run.length > 0
+    && typeof w.started_at === 'string' && Number.isFinite(Date.parse(w.started_at))
+    && typeof w.heartbeat_at === 'string' && Number.isFinite(Date.parse(w.heartbeat_at));
+}
+
+/** @param {{pid: number, start_time: string, heartbeat_at?: string}} w @param {number} [nowMs] */
+function workerLive(w, nowMs) {
+  if (!isAlive(w.pid)) return false;
+  if (heartbeatFresh(w, nowMs)) return true;
+  return w.start_time === UNKNOWN_START_TIME || readStartTime(w.pid) === w.start_time;
+}
+
+/** @param {{pid: number, start_time: string, heartbeat_at?: string}} w @param {number} [nowMs] */
+function workerLiveAtBoot(w, nowMs) {
+  if (!isAlive(w.pid)) return false;
+  const startTime = readStartTime(w.pid);
+  // Boot trusts orchestrator ps; sandbox readers let fresh heartbeats mask unreliable ps.
+  if (startTime !== null && w.start_time !== UNKNOWN_START_TIME) return startTime === w.start_time;
+  return heartbeatFresh(w, nowMs) || w.start_time === UNKNOWN_START_TIME;
+}
+
+/** Read all entries, keeping unreadable/bad entries occupied. Only proven stale owners may be pruned.
+ * @param {string} repoRoot @param {{prune?: boolean, nowMs?: number}} [opts]
+ * @returns {Array<{file: string, worker: WorkerAnnouncement | null, live: boolean}>}
+ */
+export function workerRegistry(repoRoot, opts = {}) {
+  let names;
+  try { names = readdirSync(workersDir(repoRoot)); }
+  catch (err) { if (err.code === 'ENOENT') return []; throw err; }
+  const entries = [];
+  for (const name of names.sort().filter((n) => n.endsWith('.json'))) {
+    const file = path.join(workersDir(repoRoot), name);
+    const w = readJSON(file);
+    const worker = workerShape(w) && name === `${w.pid}-${w.start_time}.json` ? w : null;
+    const live = worker === null || (opts.prune ? workerLiveAtBoot(worker, opts.nowMs) : workerLive(worker, opts.nowMs));
+    if (!live && opts.prune) rmSync(file, { force: true });
+    else entries.push({ file, worker, live });
+  }
+  return entries;
 }
 
 /**
@@ -574,13 +681,14 @@ export function retractWorker(repoRoot, pid) {
  * Accepted trade-off: a worker that is SIGKILLed (no retraction) leaves a heartbeat that stays
  * fresh for up to `HEARTBEAT_STALE_MS` (60 s); if the OS reuses its pid inside that window, the
  * queue reads as served until the heartbeat goes stale. A clean stop retracts the file at once.
- * @param {string} repoRoot @param {{nowMs?: number}} [opts]
+ * Registry entries take precedence; worker.json supports workers from older versions.
+ * @param {string} repoRoot @param {{nowMs?: number, pid?: number, registeredOnly?: boolean}} [opts]
  */
 export function liveWorker(repoRoot, opts = {}) {
+  const entry = workerRegistry(repoRoot, opts).find((e) => e.live && e.worker && (opts.pid === undefined || e.worker.pid === opts.pid));
+  if (entry) return entry.worker;
+  if (opts.registeredOnly) return null;
   const w = readWorker(repoRoot);
-  if (!w || !isAlive(w.pid)) return null;
-  if (heartbeatFresh(w, opts.nowMs)) return w;
-  if (w.start_time !== UNKNOWN_START_TIME && readStartTime(w.pid) !== w.start_time) return null;
-  return w;
+  return w && (opts.pid === undefined || w.pid === opts.pid) && workerLive(w, opts.nowMs) ? w : null;
 }
 

package/src/worker/watch.mjs

@@ -99,10 +99,12 @@ export async function watchTick({ repoRoot, runId, cfg, readRun: loadRun = readR
       if (unreadable) { await skip('unreadable_head'); continue; }
       await hooks.beforeQueue?.();
       for (const { file, content_hash } of hashes) {
-        const current = enqueueHash({ repoRoot, run: runId, block, file, content_hash, tree });
+        const held = earlier.some((old) => old.file === file);
+        if (!held) enqueueHash({ repoRoot, run: runId, block, file, content_hash, tree });
         for (const old of earlier) {
-          if (old.file === file && old.ticket !== current.ticket && !running.has(old.ticket) && !isDone(repoRoot, old.ticket)) supersedeTicket(repoRoot, old.ticket);
+          if (old.file === file && old.content_hash !== content_hash && !running.has(old.ticket) && !isDone(repoRoot, old.ticket)) supersedeTicket(repoRoot, old.ticket);
         }
+        if (held) await writeRow({ event: 'watch.held', block, file, content_hash });
       }
       save({ ...state, run: runId, base, tree, sha: head, skipped: [] });
     } catch {