@codedology/code-forge 0.8.0 → 0.9.0

package/src/worker/queue.mjs

@@ -12,10 +12,23 @@
  *
  * B62b: `<repo>` is always the MAIN tree's top level, where the worker runs — a block with its own
  * worktree still queues there; only the reviewed content is read in the block's tree.
+ *
+ * B71: an ANSWERS ticket (`kind: 'answers'`, from `review-file <answering file> --answers <ids>
+ * --of <source>`) asks the worker whether `by_file` resolves the open findings `answers` of
+ * `file` (the source). Its id adds `answers`, `by_file`, `by_hash` and the sorted ids to the
+ * normal ticket's parts, so it never collides with a review ticket of the same source content.
+ *
+ * B75: `review-file --fresh` drops a done marker it classified in place as re-runnable through
+ * `dropDoneIf` (an atomic claim by rename, then a byte compare), so two concurrent callers never
+ * both drop it. A counted marker the read saw is never moved; only a newer marker the worker wrote
+ * between that read and the claim is moved for a moment, then linked straight back (`changed`).
+ * B83: a claim that could not be linked back leaves a refused `fresh-restore-failed` marker in its
+ * place (or, when that write fails too, the claim itself, which `pendingTickets` treats as done),
+ * so a failed restore never hands the worker a ticket to run again; `clearDone` clears both.
  */
 
-import { randomBytes } from 'node:crypto';
-import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { existsSync, linkSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import { loadKey, signRow, verifyRow } from '../state/signer.mjs';
 import { isAlive, readStartTime, UNKNOWN_START_TIME } from '../util/reaper.mjs';
@@ -57,12 +70,27 @@ function readJSON(file) {
 }
 
 /**
- * @typedef {{ticket: string, run: string, block: string, file: string, content_hash: string, enqueued_at: string, tree?: string}} Ticket
+ * @typedef {{ticket: string, run: string, block: string, file: string, content_hash: string, enqueued_at: string, tree?: string, kind?: 'answers', by_file?: string, by_hash?: string, answers?: string[]}} Ticket
  *   `tree` (B62b): the block's worktree the content was hashed in. It is part of the ticket id,
  *   and the worker only checks it against the tree it takes from the run record (a ticket whose
  *   tree is not that one is `unavailable`, `worktree-mismatch`); it never reads content there.
  */
 
+/**
+ * The review ticket for `file`'s CURRENT content: the one rule `enqueue` and `clearDone` (B72)
+ * share — the repo-relative path, the realpath'd tree (or null), the content hash taken under the
+ * tree when given (else under `repoRoot`) and the ticket id over all of them.
+ * @param {{repoRoot: string, run: string, block: string, file: string, tree?: string | null}} opts
+ * @returns {{rel: string, root: string | null, content_hash: string, ticket: string}}
+ * @throws {WorkerError} `bad-path` for a bad file path or a tree that cannot be resolved.
+ */
+export function resolveTicket({ repoRoot, run, block, file, tree = null }) {
+  const rel = assertRowPath(file);
+  const root = tree === null ? null : realTree(tree);
+  const content_hash = contentHash(root ?? repoRoot, rel);
+  return { rel, root, content_hash, ticket: ticketId({ run, block, file: rel, content_hash, tree: root }) };
+}
+
 /**
  * Enqueue `file` (already repo-root-relative) for review. Idempotent by content hash: the same
  * content returns the same ticket and never rewrites it. The queue is under `repoRoot`; the
@@ -74,10 +102,7 @@ function readJSON(file) {
  * @throws {WorkerError} `bad-path` for a relative `tree` or one that cannot be resolved.
  */
 export function enqueue({ repoRoot, run, block, file, tree = null, now = new Date() }) {
-  const rel = assertRowPath(file);
-  const root = tree === null ? null : realTree(tree);
-  const content_hash = contentHash(root ?? repoRoot, rel);
-  const ticket = ticketId({ run, block, file: rel, content_hash, tree: root });
+  const { rel, root, content_hash, ticket } = resolveTicket({ repoRoot, run, block, file, tree });
   const target = path.join(queueDir(repoRoot), `${ticket}.json`);
   const named = root === null ? {} : { tree: root };
   if (!existsSync(target)) {
@@ -88,6 +113,67 @@ export function enqueue({ repoRoot, run, block, file, tree = null, now = new Dat
   return { ticket, status: isDone(repoRoot, ticket) ? 'done' : 'queued', file: rel, content_hash, ...named };
 }
 
+/** B71: the most finding ids one answers ticket may name. */
+export const MAX_ANSWERS = 10;
+
+/** B71: the finding-id shape (the one `block waive` and the review results use). */
+const ANSWER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
+
+/**
+ * B71: the id of an answers ticket — the normal ticket's parts (run, block, source, source hash,
+ * tree) plus `answers`, the answering file, its hash and the sorted ids, so it differs from the
+ * normal ticket of the same source content and from an answer of other ids or another file.
+ * @param {{run: string, block: string, file: string, content_hash: string, by_file: string, by_hash: string, answers: string[], tree?: string | null}} t
+ * @returns {string}
+ */
+export function answersTicketId({ run, block, file, content_hash, by_file, by_hash, answers, tree = null }) {
+  const parts = ['answers', run, block, file, content_hash, by_file, by_hash, [...answers].sort().join(','), ...(tree === null ? [] : [tree])];
+  return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 24);
+}
+
+/**
+ * B71: refuse an id list that is not 1..MAX_ANSWERS finding ids; returns them sorted and unique.
+ * @param {unknown} answers @returns {string[]}
+ * @throws {WorkerError} `usage`
+ */
+export function assertAnswers(answers) {
+  if (!Array.isArray(answers) || answers.length === 0) throw new WorkerError('usage', '--answers needs at least one finding id');
+  if (answers.some((a) => typeof a !== 'string' || !ANSWER_ID.test(a))) throw new WorkerError('usage', 'a finding id is letters, digits, . _ : - (at most 64)');
+  const ids = [...new Set(/** @type {string[]} */ (answers))].sort();
+  if (ids.length > MAX_ANSWERS) throw new WorkerError('usage', `--answers takes at most ${MAX_ANSWERS} finding ids`);
+  return ids;
+}
+
+/**
+ * @typedef {Ticket & {kind: 'answers', by_file: string, by_hash: string, answers: string[]}} AnswersTicket
+ */
+
+/**
+ * B71: enqueue an answers ticket: `byFile` answers the findings `answers` of `file` (both
+ * repo-root-relative). Both files are hashed now (under `tree` when given); idempotent like
+ * {@link enqueue}. The caller has already refused the same file and the id list.
+ * @param {{repoRoot: string, run: string, block: string, file: string, byFile: string, answers: string[], tree?: string | null, now?: Date}} opts
+ * @returns {{ticket: string, status: 'queued' | 'done', file: string, content_hash: string, by_file: string, by_hash: string, answers: string[], tree?: string}}
+ * @throws {WorkerError} `bad-path`, `usage`
+ */
+export function enqueueAnswers({ repoRoot, run, block, file, byFile, answers, tree = null, now = new Date() }) {
+  const rel = assertRowPath(file);
+  const by_file = assertRowPath(byFile);
+  const ids = assertAnswers(answers);
+  const root = tree === null ? null : realTree(tree);
+  const content_hash = contentHash(root ?? repoRoot, rel);
+  const by_hash = contentHash(root ?? repoRoot, by_file);
+  const ticket = answersTicketId({ run, block, file: rel, content_hash, by_file, by_hash, answers: ids, tree: root });
+  const target = path.join(queueDir(repoRoot), `${ticket}.json`);
+  const named = root === null ? {} : { tree: root };
+  if (!existsSync(target)) {
+    /** @type {AnswersTicket} */
+    const body = { ticket, kind: 'answers', run, block, file: rel, content_hash, by_file, by_hash, answers: ids, enqueued_at: now.toISOString(), ...named };
+    writeAtomic(target, `${JSON.stringify(body)}\n`);
+  }
+  return { ticket, status: isDone(repoRoot, ticket) ? 'done' : 'queued', file: rel, content_hash, by_file, by_hash, answers: ids, ...named };
+}
+
 /**
  * @param {string} tree @returns {string} `tree` realpath'd.
  * @throws {WorkerError} `bad-path` when it is relative or cannot be resolved.
@@ -106,9 +192,163 @@ export function isDone(repoRoot, ticket) {
   return existsSync(path.join(queueDir(repoRoot), `${assertTicketId(ticket)}.done`));
 }
 
+/** B83: a `dropDoneIf` claim of a done marker (`<ticket>.done.<pid>.<hex>.fresh`). */
+const CLAIM = /^([0-9a-f]{24})\.done\.\d+\.[0-9a-f]{8}\.fresh$/;
+
+/**
+ * B83: an fs error's code when it is a plain code word, else `unknown` (a message names the code
+ * only, never a path). @param {unknown} err @returns {string}
+ */
+function codeOf(err) {
+  const code = /** @type {{code?: unknown}} */ (err)?.code;
+  return typeof code === 'string' && /^[A-Za-z][A-Za-z0-9_-]{0,31}$/.test(code) ? code : 'unknown';
+}
+
+/**
+ * B72 (`block reopen`, C11): drop the done marker of the review ticket for the file's CURRENT
+ * content, so the next `review-file` on it runs instead of returning the cached result. The ticket
+ * id comes from `resolveTicket`, the rule `enqueue` uses; no ticket is written. Synchronous.
+ * B83: it also removes the ticket's leftover `dropDoneIf` claims (`<ticket>.done.*.fresh`), which
+ * would otherwise keep the ticket off the worker for good; `removed` still names the marker only.
+ * @param {{repoRoot: string, run: string, block: string, file: string, tree?: string | null}} opts
+ * @returns {{ticket: string, content_hash: string, removed: boolean}}
+ * @throws {WorkerError} `bad-path` for a bad file path or a tree that cannot be resolved.
+ */
+export function clearDone({ repoRoot, run, block, file, tree = null }) {
+  const { content_hash, ticket } = resolveTicket({ repoRoot, run, block, file, tree });
+  const marker = path.join(queueDir(repoRoot), `${ticket}.done`);
+  const removed = existsSync(marker);
+  rmSync(marker, { force: true });
+  let names;
+  try {
+    names = readdirSync(queueDir(repoRoot));
+  } catch {
+    names = [];
+  }
+  // accepted race: a claim a concurrent dropDoneIf still uses may go here, which can leave a refused
+  // fresh-restore-failed marker (rare, fail closed; a second `block reopen` clears it)
+  for (const name of names) if (CLAIM.exec(name)?.[1] === ticket) rmSync(path.join(queueDir(repoRoot), name), { force: true });
+  return { ticket, content_hash, removed };
+}
+
+/**
+ * B75 (`review-file --fresh`): drop the done marker of `ticket` only while it still holds `seen`,
+ * the bytes the caller already read IN PLACE and classified as re-runnable (a marker classified
+ * as counted, tampered or unknown is never passed here, so it is never moved). The queue has no
+ * lock file; the order is the guarantee:
+ *   1. the caller reads the marker in place and classifies it (a refusal touches nothing);
+ *   2. only a re-runnable marker is claimed by an atomic rename to a name of this process — of
+ *      two concurrent callers only one takes it, the other gets `gone`;
+ *   3. the claim's bytes are compared with `seen`: equal, the claim is removed (`dropped`) and the
+ *      caller enqueues; different (a newer result arrived), the marker is restored with
+ *      `linkSync` only — on EEXIST a newer marker is there and wins, the claim is removed
+ *      (`changed`); on any other link error `fresh-restore-failed` is thrown (below).
+ * B83, fail closed — no failure leaves a ticket the worker runs again by itself:
+ *   - a link-back that fails with other than EEXIST writes a refused marker in place,
+ *     `{ticket, status: 'refused', reason: 'fresh-restore-failed'}` (no-overwrite open `wx`, mode
+ *     0o600; it names no result, so `--fresh` counts it and refuses, and `block reopen` clears it).
+ *     EEXIST on that write: a newer marker is in place. Any other write error keeps the claim; a
+ *     claim (`<ticket>.done.<pid>.<hex>.fresh`) counts as done for `pendingTickets`, and
+ *     `clearDone` (`block reopen`) removes it;
+ *   - a claim that cannot be removed after a `dropped` compare is `fresh-claim-failed` (it would
+ *     hold the re-queued ticket off the worker; `block reopen` clears it); after a good link-back
+ *     the marker is back, so a claim that cannot be removed is left for `clearDone`.
+ * A crash between the drop and the caller's enqueue leaves a re-runnable ticket with no marker:
+ * harmless — its ticket file is pending, and the next `review-file` queues it anyway. Synchronous.
+ * @param {string} repoRoot @param {string} ticket @param {Buffer} seen
+ * @param {{link?: (from: string, to: string) => void, read?: (file: string) => Buffer, rename?: (from: string, to: string) => void, remove?: (file: string) => void, write?: (file: string, text: string) => void}} [io] - for
+ *   tests (default `linkSync`, `readFileSync`, `renameSync`, `rmSync` with force, `writeFileSync`
+ *   with flag `wx` and mode 0o600). A claim that cannot be read is restored the same way and
+ *   refused `fresh-restore-failed`.
+ * @returns {'dropped' | 'gone' | 'changed'}
+ * @throws {WorkerError} `fresh-claim-failed` (the claim could not be taken, or not removed after
+ *   the drop; nothing was queued) or `fresh-restore-failed` (the message says which state was left:
+ *   put back, a newer marker, a refused marker, the claim kept, or nothing). A message names an
+ *   error code, never a path.
+ */
+export function dropDoneIf(repoRoot, ticket, seen, io = {}) {
+  const marker = path.join(queueDir(repoRoot), `${assertTicketId(ticket)}.done`);
+  const claim = `${marker}.${process.pid}.${randomBytes(4).toString('hex')}.fresh`;
+  const remove = io.remove ?? ((/** @type {string} */ file) => rmSync(file, { force: true }));
+  try {
+    (io.rename ?? renameSync)(marker, claim);
+  } catch (err) {
+    if (/** @type {{code?: unknown}} */ (err)?.code === 'ENOENT') return 'gone';
+    throw new WorkerError('fresh-claim-failed', `the done marker of ticket ${ticket} could not be claimed (${codeOf(err)}); nothing was queued`);
+  }
+  /** @type {string | null} */
+  let readFailed = null;
+  /** @type {boolean} */
+  let same = false;
+  try {
+    same = (io.read ?? readFileSync)(claim).equals(seen);
+  } catch (err) {
+    readFailed = codeOf(err);
+  }
+  if (same) {
+    try {
+      remove(claim);
+    } catch (err) {
+      throw new WorkerError('fresh-claim-failed', `the done marker of ticket ${ticket} was dropped but its claim could not be removed (${codeOf(err)}); block reopen clears it; nothing was queued`);
+    }
+    return 'dropped';
+  }
+  /** @type {string | null} */
+  let linkFailed = null;
+  try {
+    (io.link ?? linkSync)(claim, marker);
+  } catch (err) {
+    linkFailed = codeOf(err);
+  }
+  const unread = `the claimed done marker of ticket ${ticket} could not be read (${readFailed})`;
+  if (linkFailed !== null && linkFailed !== 'EEXIST') {
+    const cause = readFailed !== null ? `${unread} and could not be restored (${linkFailed})` : `the done marker of ticket ${ticket} changed and could not be restored (${linkFailed})`;
+    throw new WorkerError('fresh-restore-failed', `${cause}; ${refuseInPlace(marker, claim, ticket, io, remove)}; nothing was queued`);
+  }
+  try {
+    remove(claim);
+  } catch {
+    // the marker is back (or a newer one won), so the ticket is done; `clearDone` removes the claim
+  }
+  if (readFailed !== null) throw new WorkerError('fresh-restore-failed', `${unread}; ${linkFailed === 'EEXIST' ? 'a newer marker is in place' : 'it was put back'}; nothing was queued`);
+  return 'changed';
+}
+
+/**
+ * B83: after a failed link-back, put a refused `fresh-restore-failed` marker where the done marker
+ * was (never over one), so the worker does not run the ticket; the claim is then removed when it
+ * can be (`clearDone` removes a leftover one). When the write fails the claim is kept instead.
+ * @param {string} marker @param {string} claim @param {string} ticket
+ * @param {{write?: (file: string, text: string) => void}} io @param {(file: string) => void} remove
+ * @returns {string} which state was left, for the message (codes only, no path).
+ */
+function refuseInPlace(marker, claim, ticket, io, remove) {
+  const text = `${JSON.stringify({ ticket, status: 'refused', reason: 'fresh-restore-failed' })}\n`;
+  /** @type {string | null} */
+  let writeFailed = null;
+  try {
+    (io.write ?? ((/** @type {string} */ file, /** @type {string} */ body) => writeFileSync(file, body, { flag: 'wx', mode: 0o600 })))(marker, text);
+  } catch (err) {
+    writeFailed = codeOf(err);
+  }
+  if (writeFailed === null || writeFailed === 'EEXIST') {
+    try {
+      remove(claim);
+    } catch {
+      // the marker holds the ticket off the worker; `clearDone` removes the leftover claim
+    }
+    return writeFailed === null ? 'a refused fresh-restore-failed marker was written in its place, so the worker does not run the ticket' : 'a newer marker is in place';
+  }
+  // the claim holds the ticket off the worker (`pendingTickets`); a claim that is gone cannot
+  return existsSync(claim)
+    ? `the refused marker could not be written (${writeFailed}); its claim was kept, so the worker does not run the ticket; block reopen clears it`
+    : `the refused marker could not be written (${writeFailed}) and no claim exists`;
+}
+
 /**
  * Tickets with no done marker, oldest first (enqueue time, then id). Unparseable files are
- * skipped here and refused by the worker when it reads them.
+ * skipped here and refused by the worker when it reads them. B83: a ticket whose marker is held
+ * by a `dropDoneIf` claim is not pending either (a claim kept after a failed restore fails closed).
  * @param {string} repoRoot
  * @returns {string[]} ticket ids
  */
@@ -119,8 +359,9 @@ export function pendingTickets(repoRoot) {
   } catch {
     return [];
   }
+  const claimed = new Set(names.map((n) => CLAIM.exec(n)?.[1]));
   const ids = names.filter((n) => /^[0-9a-f]{24}\.json$/.test(n)).map((n) => n.slice(0, 24));
-  const pending = ids.filter((id) => !names.includes(`${id}.done`));
+  const pending = ids.filter((id) => !names.includes(`${id}.done`) && !claimed.has(id));
   const at = (/** @type {string} */ id) => {
     const t = readJSON(path.join(queueDir(repoRoot), `${id}.json`));
     return typeof t?.enqueued_at === 'string' ? t.enqueued_at : '';
@@ -139,9 +380,23 @@ export function readTicket(repoRoot, id) {
   if (!t || typeof t !== 'object') throw new WorkerError('bad-ticket', `ticket ${id} is not readable JSON`);
   assertRowPath(t.file);
   if (t.tree !== undefined && (typeof t.tree !== 'string' || !path.isAbsolute(t.tree))) throw new WorkerError('bad-ticket', `ticket ${id} names a tree that is not an absolute path`);
-  if (typeof t.run !== 'string' || typeof t.block !== 'string' || typeof t.content_hash !== 'string' || ticketId(t) !== id) {
+  if (typeof t.run !== 'string' || typeof t.block !== 'string' || typeof t.content_hash !== 'string') {
     throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
   }
+  if (t.kind !== undefined) {
+    // B71: the only other kind is an answers ticket; its id covers the answering file and the ids
+    if (t.kind !== 'answers') throw new WorkerError('bad-ticket', `ticket ${id} has an unknown kind`);
+    assertRowPath(t.by_file);
+    let ids;
+    try {
+      ids = assertAnswers(t.answers);
+    } catch {
+      throw new WorkerError('bad-ticket', `ticket ${id} names no valid finding ids`);
+    }
+    if (typeof t.by_hash !== 'string' || ids.join(',') !== t.answers.join(',') || answersTicketId(t) !== id) throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
+    return /** @type {Ticket} */ (t);
+  }
+  if (ticketId(t) !== id) throw new WorkerError('bad-ticket', `ticket ${id} does not match its id`);
   return /** @type {Ticket} */ (t);
 }