@codedology/code-forge 0.7.1 → 0.7.3

package/src/session/spawn.mjs

@@ -64,6 +64,7 @@ import { exec } from '../util/exec.mjs';
 import { isForbidden, mergeForbidden } from '../util/forbidden.mjs';
 import { semaphore } from '../util/locks.mjs';
 import { readStartTime, registerPid, UNKNOWN_START_TIME } from '../util/reaper.mjs';
+import { asText, stderrTail } from '../util/diagnostic-tail.mjs';
 import { redact, writeSafe } from '../util/redact.mjs';
 import { currentRunRoot, pidsDir, runRoot, tmpBase, untrustedReason } from '../util/tmp.mjs';
 
@@ -152,11 +153,8 @@ export function runRootFor(runId, root) {
   return runRoot(runId, { root });
 }
 
-/** How much of a timed-out session's stderr is kept (B30). */
-export const DIAGNOSTIC_TAIL_BYTES = 4096;
-
-/** A timed-out session's stderr over this size keeps no text at all (bounds the redact cost). */
-export const STDERR_MAX_BYTES = 1024 * 1024;
+// B30's stderr tail moved to `util/diagnostic-tail.mjs` (B55 shares it); re-exported here unchanged
+export { DIAGNOSTIC_TAIL_BYTES, STDERR_MAX_BYTES, TAIL_WITHHELD_LARGE, TAIL_WITHHELD_LONG_LINE, TAIL_WITHHELD_PACKET, stderrTail } from '../util/diagnostic-tail.mjs';
 
 /** How many stdout event types a timed-out session keeps. */
 export const STDOUT_EVENTS_MAX = 20;
@@ -164,87 +162,6 @@ export const STDOUT_EVENTS_MAX = 20;
 /** The longest event type kept. */
 const EVENT_TYPE_MAX = 40;
 
-/**
- * Packet lines shorter than this are too common to tell apart from ordinary output, so they are
- * NOT filtered. Accepted risk (B30): a packet line under 16 characters (a brace, `return x;`) can
- * appear in a stderr tail; such a line carries no meaningful packet content on its own.
- */
-const PACKET_LINE_MIN = 16;
-
-/** The tail note when the packet cannot be read: no stderr text is kept at all (fail closed). */
-export const TAIL_WITHHELD_PACKET = 'withheld: packet unreadable';
-/** The tail note when stderr is over `STDERR_MAX_BYTES`. */
-export const TAIL_WITHHELD_LARGE = 'stderr too large, withheld';
-/** The tail note when the last 4 KB of stderr is one partial line. */
-export const TAIL_WITHHELD_LONG_LINE = 'one long line, withheld';
-
-/** @param {Buffer} buf @param {number} at @returns {number} the first character start at or after `at`. */
-function charStart(buf, at) {
-  let i = at;
-  while (i < buf.length && (buf[i] & 0xc0) === 0x80) i += 1;
-  return i;
-}
-
-/** @param {unknown} text @returns {string} */
-function asText(text) {
-  return Buffer.isBuffer(text) ? text.toString('utf8') : typeof text === 'string' ? text : '';
-}
-
-const PEM_BEGIN = /-----BEGIN [A-Z0-9 ]*-----/;
-const PEM_END = /-----END [A-Z0-9 ]*-----/;
-
-/**
- * The stderr tail of a timed-out session (B30), never packet content:
- *  1. no packet text (unreadable) ⇒ no text: `{tail: null, note: 'withheld: packet unreadable'}`;
- *     stderr over `STDERR_MAX_BYTES` ⇒ `{tail: null, note: 'stderr too large, withheld'}`;
- *  2. on the FULL raw text: every PEM block (`-----BEGIN` … `-----END` lines, an unterminated one
- *     to the end) and every line repeating a packet line (≥ 16 chars) is dropped;
- *  3. the FULL remaining text goes through B0 `redact`;
- *  4. over `maxBytes`, the last `maxBytes` bytes are kept, starting on a character, with the first
- *     (partial) line dropped; when that piece has no line break the tail is withheld.
- * @param {string | Buffer | null | undefined} text @param {string | null} packetText @param {number} [maxBytes]
- * @returns {{tail: string | null, note?: string}}
- */
-export function stderrTail(text, packetText, maxBytes = DIAGNOSTIC_TAIL_BYTES) {
-  if (typeof packetText !== 'string') return { tail: null, note: TAIL_WITHHELD_PACKET };
-  const raw = asText(text);
-  if (raw.length === 0) return { tail: '' };
-  if (Buffer.byteLength(raw) > STDERR_MAX_BYTES) return { tail: null, note: TAIL_WITHHELD_LARGE };
-  const packetLines = [
-    ...new Set(
-      packetText
-        .split('\n')
-        .map((l) => l.trim())
-        .filter((l) => l.length >= PACKET_LINE_MIN),
-    ),
-  ];
-  let inPem = false;
-  /** @type {string[]} */
-  const kept = [];
-  for (const line of raw.split('\n')) {
-    const t = line.trim();
-    if (inPem) {
-      if (PEM_END.test(t)) inPem = false;
-      continue;
-    }
-    if (PEM_BEGIN.test(t)) {
-      inPem = !PEM_END.test(t);
-      continue;
-    }
-    if (t.length >= PACKET_LINE_MIN && packetLines.some((p) => t.includes(p))) continue;
-    kept.push(line);
-  }
-  const safe = /** @type {string} */ (redact(kept.join('\n')));
-  const buf = Buffer.from(safe, 'utf8');
-  if (buf.length <= maxBytes) return { tail: safe };
-  const from = charStart(buf, buf.length - maxBytes);
-  const piece = buf.subarray(from).toString('utf8');
-  if (buf[from - 1] === 0x0a) return { tail: piece }; // the cut fell exactly on a line start
-  const nl = piece.indexOf('\n');
-  if (nl === -1) return { tail: '', note: TAIL_WITHHELD_LONG_LINE };
-  return { tail: piece.slice(nl + 1) };
-}
-
 /**
  * What a timed-out session's stdout says without any of its text (B30): the last
  * `STDOUT_EVENTS_MAX` event types (JSON lines with a string `type`, each cut to 40 characters and

package/src/util/diagnostic-tail.mjs

@@ -0,0 +1,95 @@
+/**
+ * The packet-free, redacted tail of a session's diagnostic text (B30): a timed-out session's
+ * stderr (`spawn.mjs`) and, since B55, a review answer that failed the schema
+ * (`review/schema-fallback.mjs`). Lives in `util/` so both layers can use the one filter.
+ */
+
+import { redact } from './redact.mjs';
+
+/** How much of a timed-out session's stderr is kept (B30). */
+export const DIAGNOSTIC_TAIL_BYTES = 4096;
+
+/** A timed-out session's stderr over this size keeps no text at all (bounds the redact cost). */
+export const STDERR_MAX_BYTES = 1024 * 1024;
+
+/**
+ * Packet lines shorter than this are too common to tell apart from ordinary output, so they are
+ * NOT filtered. Accepted risk (B30): a packet line under 16 characters (a brace, `return x;`) can
+ * appear in a stderr tail; such a line carries no meaningful packet content on its own.
+ */
+const PACKET_LINE_MIN = 16;
+
+/** The tail note when the packet cannot be read: no stderr text is kept at all (fail closed). */
+export const TAIL_WITHHELD_PACKET = 'withheld: packet unreadable';
+/** The tail note when stderr is over `STDERR_MAX_BYTES`. */
+export const TAIL_WITHHELD_LARGE = 'stderr too large, withheld';
+/** The tail note when the last 4 KB of stderr is one partial line. */
+export const TAIL_WITHHELD_LONG_LINE = 'one long line, withheld';
+
+/** @param {Buffer} buf @param {number} at @returns {number} the first character start at or after `at`. */
+function charStart(buf, at) {
+  let i = at;
+  while (i < buf.length && (buf[i] & 0xc0) === 0x80) i += 1;
+  return i;
+}
+
+/** @param {unknown} text @returns {string} */
+export function asText(text) {
+  return Buffer.isBuffer(text) ? text.toString('utf8') : typeof text === 'string' ? text : '';
+}
+
+const PEM_BEGIN = /-----BEGIN [A-Z0-9 ]*-----/;
+const PEM_END = /-----END [A-Z0-9 ]*-----/;
+
+/**
+ * The stderr tail of a timed-out session (B30), never packet content:
+ *  1. no packet text (unreadable) ⇒ no text: `{tail: null, note: 'withheld: packet unreadable'}`;
+ *     stderr over `STDERR_MAX_BYTES` ⇒ `{tail: null, note: 'stderr too large, withheld'}`;
+ *  2. on the FULL raw text: every PEM block (`-----BEGIN` … `-----END` lines, an unterminated one
+ *     to the end) and every line repeating a packet line (≥ 16 chars) is dropped;
+ *  3. the FULL remaining text goes through B0 `redact`;
+ *  4. over `maxBytes`, the last `maxBytes` bytes are kept, starting on a character, with the first
+ *     (partial) line dropped; when that piece has no line break the tail is withheld.
+ * @param {string | Buffer | null | undefined} text @param {string | null} packetText @param {number} [maxBytes]
+ * @returns {{tail: string | null, note?: string}}
+ */
+export function stderrTail(text, packetText, maxBytes = DIAGNOSTIC_TAIL_BYTES) {
+  if (typeof packetText !== 'string') return { tail: null, note: TAIL_WITHHELD_PACKET };
+  const raw = asText(text);
+  if (raw.length === 0) return { tail: '' };
+  if (Buffer.byteLength(raw) > STDERR_MAX_BYTES) return { tail: null, note: TAIL_WITHHELD_LARGE };
+  const packetLines = [
+    ...new Set(
+      packetText
+        .split('\n')
+        .map((l) => l.trim())
+        .filter((l) => l.length >= PACKET_LINE_MIN),
+    ),
+  ];
+  let inPem = false;
+  /** @type {string[]} */
+  const kept = [];
+  for (const line of raw.split('\n')) {
+    const t = line.trim();
+    if (inPem) {
+      if (PEM_END.test(t)) inPem = false;
+      continue;
+    }
+    if (PEM_BEGIN.test(t)) {
+      inPem = !PEM_END.test(t);
+      continue;
+    }
+    if (t.length >= PACKET_LINE_MIN && packetLines.some((p) => t.includes(p))) continue;
+    kept.push(line);
+  }
+  const safe = /** @type {string} */ (redact(kept.join('\n')));
+  const buf = Buffer.from(safe, 'utf8');
+  if (buf.length <= maxBytes) return { tail: safe };
+  const from = charStart(buf, buf.length - maxBytes);
+  const piece = buf.subarray(from).toString('utf8');
+  if (buf[from - 1] === 0x0a) return { tail: piece }; // the cut fell exactly on a line start
+  const nl = piece.indexOf('\n');
+  if (nl === -1) return { tail: '', note: TAIL_WITHHELD_LONG_LINE };
+  return { tail: piece.slice(nl + 1) };
+}
+

package/src/worker/engine.mjs

@@ -15,7 +15,8 @@
  *    last round reviewed and the current content), with the open findings by id;
  *  - `next: patch` (the L3 rung at the cap) ⇒ the `patch_check` round; a `retry` re-runs its round;
  *  - `stopped` (`review_cap`, `l3_patch_exhausted`, `split_required`) ⇒ no session: the result
- *    repeats the stop — only the human moves on (§4.11 rule 5).
+ *    repeats the stop — only the human moves on (§4.11 rule 5). A `split_required` result (round 1
+ *    or a recheck stop) carries `tokens_in` and `budget` (B54).
  * The round's outcome: `approved: true` ONLY when the loop is `complete` AND its signed
  * `review.approved` row (keyed `run, block, file, content_hash` = the ticket's hash) was written;
  * else `findings` = the open fix list, `next` = what the orchestrator does (`fix` at `level`,
@@ -29,6 +30,10 @@
  * block's review budget (§4.10, B19): the forecast over the block's changed owned files (tier from
  * the path floors, before S1), one `review.budget` row per block; a failure there never fails the
  * review.
+ * B55: every round gets the packet's signed schema-miss history (`ledgerSchemaHistory`: rows of
+ * this run for this file and content hash whose MAC verifies with the run key), so a packet's
+ * second `schema` miss — this round's first try after the earlier ticket's — is tried once on
+ * `review.second_levels.<level>` when one is configured.
  * The review NEVER approves an empty or failed session: that is `unavailable` (§4.2 stub guard).
  * A throw inside the round is `unavailable` too, with the state saved as `next: retry` (see
  * `fixLoopRound`): the next ticket re-runs that round, never a fresh round 1.
@@ -44,6 +49,8 @@ import { reviewFile, rulesRisk } from '../review/engine.mjs';
 import { computeFileSet, ownsFile } from '../gates/scope.mjs';
 import { extraRoundsFor, newFileState, reopenForExtraRound, runRound } from '../review/fixloop.mjs';
 import { readRun } from '../state/run.mjs';
+import { ledgerSchemaHistory } from '../review/schema-fallback.mjs';
+import { verifyRow } from '../state/signer.mjs';
 import { redact } from '../util/redact.mjs';
 import { keyedLock } from '../util/locks.mjs';
 import { blockAnchors, blockRungUsed, loadState, saveState } from './review-state.mjs';
@@ -83,6 +90,10 @@ import { assertTicketId } from './ticket.mjs';
  * @property {string | null} [trigger] - `review_stall`, `review_rounds`, `review_cap`, or null.
  * @property {string} [stopped] - the stop reason (`review_cap`, `l3_patch_exhausted`, …).
  * @property {number} [late] - late findings recorded so far for the file.
+ * @property {number} [tokens_in] - `split_required` (or `stopped: split_required`): the estimated
+ *   tokens of the diff-only packet that was over the budget (B54).
+ * @property {number} [budget] - with `tokens_in`: the packet budget it exceeded (B54).
+ * @property {string} [section] - a Markdown file's section that alone is over the budget (B54).
  */
 
 /**
@@ -302,7 +313,7 @@ async function fixLoopRound(ticket, ctx, { base, level, owned, blockKindOf, key,
     reopenForExtraRound(state, extraRounds, ctx.cfg);
 
     if (state.status === 'stopped') {
-      return { outcome: /** @type {ReviewOutcome} */ ({ ...loopFields(state, null), status: 'stopped', stopped: state.next?.reason ?? 'stopped', approved: false, engine: 'adaptive', sessions: [], findings: state.open }) };
+      return { outcome: /** @type {ReviewOutcome} */ ({ ...loopFields(state, null), status: 'stopped', stopped: state.next?.reason ?? 'stopped', ...stopSize(state), approved: false, engine: 'adaptive', sessions: [], findings: state.open }) };
     }
 
     /** @type {'full' | 'recheck' | 'patch_check'} */
@@ -350,6 +361,23 @@ async function playRound(ticket, ctx, { state, kind, where, seq, base, blockKind
   /** @type {string | null} */
   let crashed = null;
   try {
+    // B55: a packet's schema misses are counted from the run's signed rows for this run, file,
+    // content hash AND packet hash (`ledgerSchemaHistory` is asked per packet). Built inside the
+    // guarded block: a throw here is this round's `unavailable` with `next: retry`, like any other;
+    // a ledger read that throws later only disables the second-level try (with one warning line).
+    // Once per packet holds because this ticket holds the (block, file) file lock (`loop.mjs`) for
+    // the whole round — round 1, its retry, a recheck and the patch_check alike — and
+    // `schema-fallback.mjs` adds a per-packet lock around read-history → write-mark.
+    // The rows are matched on the values the worker's writeRow stamps on them (`loop.mjs`:
+    // `{...row, block, file, content_hash: ticket.content_hash}`): `ticket.file` and
+    // `ticket.content_hash` here are the same ticket's, for round 1 and rechecks alike (a recheck
+    // ticket's hash is the fixed content's). `where.key` is the run key: `loop.mjs` puts the run's
+    // `key` in `ctx.key` (`key, // signs the fix-loop state`), `reviewTicket` passes `key: ctx.key`
+    // to `fixLoopRound`, which builds `where = {…, key, rows}`. A row whose check throws is not
+    // counted (`ledgerSchemaHistory` catches per row).
+    const schemaHistory = ctx.readRows
+      ? ledgerSchemaHistory({ readRows: ctx.readRows, verify: (row) => verifyRow(row, where.key).ok, runId: ctx.runId, file: ticket.file, contentHash: ticket.content_hash })
+      : undefined;
     await runRound(
       state,
       {
@@ -361,8 +389,9 @@ async function playRound(ticket, ctx, { state, kind, where, seq, base, blockKind
         factsExcerpt,
         extraRounds,
         ...(jev ? { jev } : {}),
+        ...(schemaHistory ? { schemaHistory } : {}),
         review: async () => {
-          engineOutcome = await reviewFile({ repoRoot: ctx.repoRoot, file: ticket.file, base, cfg: ctx.cfg, kind: blockKindOf, workDir, factsExcerpt }, { spawn: ctx.spawn, writeRow });
+          engineOutcome = await reviewFile({ repoRoot: ctx.repoRoot, file: ticket.file, base, cfg: ctx.cfg, kind: blockKindOf, workDir, factsExcerpt }, { spawn: ctx.spawn, writeRow, ...(schemaHistory ? { schemaHistory } : {}) });
           return engineOutcome;
         },
         spawn: async (opts) => {
@@ -404,7 +433,7 @@ async function playRound(ticket, ctx, { state, kind, where, seq, base, blockKind
     if (!approvalWritten) return { ...head, status: 'unavailable', reason: 'approval-not-written', approved: false };
     return { ...head, status: 'reviewed', approved: true, findings: [] };
   }
-  if (status === 'stopped') return { ...head, status: 'stopped', stopped: next?.reason ?? 'stopped', approved: false, findings: state.open };
+  if (status === 'stopped') return { ...head, status: 'stopped', stopped: next?.reason ?? 'stopped', ...stopSize(state), approved: false, findings: state.open };
   return { ...head, status: 'reviewed', approved: false, reason: 'findings-open', findings: state.open };
 }
 
@@ -457,6 +486,18 @@ function loopFields(state, kind) {
   return { round: state.round, ...(kind ? { kind } : {}), level: state.level, next: state.next, trigger: state.next?.trigger ?? null, late: state.late.length };
 }
 
+/**
+ * B54: a `split_required` stop's packet size (`tokens_in`, `budget` — null when unknown — and a
+ * Markdown `section`), from the state's `next`; nothing for any other stop.
+ * @param {import('../review/fixloop.mjs').FileState} state @returns {{tokens_in?: number | null, budget?: number | null, section?: string}}
+ */
+function stopSize(state) {
+  const next = state.next;
+  if (next?.reason !== 'split_required') return {};
+  const fin = (/** @type {unknown} */ v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
+  return { tokens_in: fin(next.tokens_in), budget: fin(next.budget), ...(typeof next.section === 'string' ? { section: next.section } : {}) };
+}
+
 /**
  * S1 through `askJev` with the worker's key bound (the key stays in this process).
  * @param {string} key @returns {import('../review/triage.mjs').JevAsk}

package/src/worker/loop.mjs

@@ -159,6 +159,21 @@ export function loopResult(outcome) {
   return out;
 }
 
+/**
+ * B54: the packet size of a `split_required` outcome (round 1 `status: split_required`, or a
+ * recheck `stopped: split_required`) — and ONLY of one: `tokens_in` and `budget` always (null when
+ * not a finite number), plus the Markdown `section` that alone is over the budget when one is named. The result
+ * file and the ledger `review.result` row carry the same fields.
+ * @param {Record<string, any>} outcome @returns {Record<string, any>}
+ */
+export function packetSize(outcome) {
+  if (outcome.status !== 'split_required' && outcome.stopped !== 'split_required') return {};
+  return { tokens_in: finiteOrNull(outcome.tokens_in), budget: finiteOrNull(outcome.budget), ...(typeof outcome.section === 'string' ? { section: outcome.section } : {}) };
+}
+
+/** @param {unknown} v @returns {number | null} `v` when a finite number, else null. */
+export const finiteOrNull = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
+
 /**
  * B52: the verdict a result is recorded with (its signed file and its ledger row), always with
  * `approved` and `open`: `approved: true, open: []` only when the outcome said so AND it lists no
@@ -302,6 +317,7 @@ export async function createWorker(opts, deps = {}) {
         engine: outcome.engine,
         ...(typeof outcome.reason === 'string' ? { reason: outcome.reason } : {}),
         ...(Array.isArray(outcome.findings) ? { findings: outcome.findings } : {}),
+        ...packetSize(outcome),
         ...loopResult(outcome),
         sessions: outcome.sessions,
       };
@@ -387,6 +403,8 @@ export async function createWorker(opts, deps = {}) {
         ...(typeof result.reason === 'string' ? { reason: result.reason } : {}),
         ...(typeof result.trigger === 'string' ? { trigger: result.trigger } : {}),
         ...(typeof result.stopped === 'string' ? { stopped: result.stopped } : {}),
+        // B54: a split_required result says how far over the budget it was (the result file's fields)
+        ...packetSize(result),
       });
     } catch {
       // the signed result file stands; the gate reads it (§4.6)