iterate-plugin 3.9.2 → 3.10.0

package/src/tools/history.ts

@@ -140,8 +140,10 @@ export function registerHistoryTool(ctx: { tools: { register: (def: ReturnType<t
         type: {
           type: 'string',
           description:
-            'Optional entry-type filter: round_start, review_result, atomic_fix, architectural_fix, ' +
-            'revert, round_failed, validation, decision, report, resume.',
+            'Optional entry-type (decision kind) filter. This plugin emits round_start, review_result, ' +
+            'atomic_fix, architectural_fix, revert, round_failed, validation, decision, report, resume — ' +
+            'a LABEL list, not a closed set: any kind matching the kernel grammar ' +
+            '(`^[a-z][a-z0-9_]{0,31}$`) can be recorded and filtered here.',
         },
         since: {
           type: 'string',

package/src/tools/prune.ts

@@ -6,7 +6,6 @@
  *                   actually delete.
  *
  * Artifacts managed:
- *   - Decision-log entries older than `retainDays` (default 30, via since).
  *   - Stale checkpoint files (checkpoint.json).
  *   - Fix backups left over from old rounds (backups whose fix-id no longer
  *     appears in the registry).
@@ -18,6 +17,14 @@
  *   - Defense events older than `retainDays` (stale iteration streams that
  *     were never cleared).
  *
+ * Artifacts NOT managed (deliberately):
+ *   - The decision ledger (`.iterate/decision-log.jsonl`). It is a hash CHAIN since the
+ *     kernel flip: every entry's `prevEntryHash` is the sha256 of the previous line's
+ *     bytes, so dropping or renumbering one line invalidates every later one. Prune
+ *     therefore never rewrites it — `pruneDecisionLog` reports what a cutoff WOULD have
+ *     removed and names the two legal moves instead. The renumbering rewrite that used to
+ *     live here is deleted, not hidden behind a flag.
+ *
  * Security model:
  *   - Only operates under the resolved project `.iterate/` directory.
  *   - dryRun=true by default — the caller must explicitly opt into deletion.
@@ -30,16 +37,16 @@ import { join } from 'node:path'
 import { defineTool } from '@deepseek-ai/dsh-tools'
 import type { JsonValue } from '@deepseek-ai/dsh-util-values'
 import { resolveProjectRootForExec } from '../config-loader.ts'
-import { writeTextAtomic, writeJsonAtomic } from '../atomic-fs.ts'
+import { writeJsonAtomic } from '../atomic-fs.ts'
 import { withProjectLock } from '../file-lock.ts'
-import { readDecisionEntries, appendDecisionEntry, acquireLogLockChecked, invalidateLogCountCache } from './decision-log.ts'
+import { readDecisionEntries, appendDecisionEntry, decisionLogContestedLine } from './decision-log.ts'
 import { readRegistry, removeRecord, recomputeRoundCounts } from './fix.ts'
 import { readExperienceBank, writeExperienceBank } from './experience-store.ts'
 import { readDefenseEvents, writeDefenseEvents } from './defense-store.ts'
 import { iterateDir, fixesDir, checkpointPath, fixRegistryPath } from '../paths.ts'
 import type { DefenseEventStream, ExperienceBank, FixRegistry } from '../types.ts'
 
-/** Default retention for decision-log entries (in days). */
+/** Retention window for the age-based artifacts (checkpoints, defense events). */
 const DEFAULT_RETAIN_DAYS = 30
 const MIN_RETAIN_DAYS = 1
 const MAX_RETAIN_DAYS = 365
@@ -52,9 +59,6 @@ const MAX_RETAIN_DAYS = 365
  */
 export const MAX_EXPERIENCE_ENTRIES = 2000
 
-/** How many compare-and-append retries bound the decision-log rewrite race. */
-const MAX_LOG_REWRITE_RETRIES = 3
-
 /** Clamp retainDays to a sane range. */
 export function clampRetainDays(days: number | undefined): number {
   if (typeof days !== 'number' || !Number.isInteger(days) || days <= 0) {
@@ -163,6 +167,9 @@ export function inspectPrune(
   retainDays: number,
 ): {
   oldLogEntries: number
+  /** 1-based line of the first legacy record AFTER the chain began (a contested ledger),
+   *  or null. This is the only decision-log CONDITION prune can report but not settle. */
+  contestedLogLine: number | null
   hasCheckpoint: boolean
   checkpointStale: boolean
   staleBackups: string[]
@@ -180,9 +187,12 @@ export function inspectPrune(
 } {
   const cutoff = cutoffTimestamp(retainDays)
 
-  // 1. Decision-log entries older than retainDays.
+  // 1. Decision-log entries older than retainDays — REPORTED, never removed: the ledger
+  // is a hash chain (see `pruneDecisionLog`). `contestedLogLine` is the one ledger
+  // condition a human has to settle, so the dry-run preview surfaces it.
   const entries = readDecisionEntries(projectRoot)
   const oldLogEntries = entries.filter((e) => e.timestamp < cutoff).length
+  const contestedLogLine = decisionLogContestedLine(projectRoot)
 
   // 2. Checkpoint presence. A checkpoint is only STALE once it is older than
   // retainDays — a fresh checkpoint means a run was interrupted recently and
@@ -280,6 +290,7 @@ export function inspectPrune(
 
   return {
     oldLogEntries,
+    contestedLogLine,
     hasCheckpoint,
     checkpointStale,
     staleBackups,
@@ -300,91 +311,70 @@ export function inspectPrune(
  * Returns a detailed report of what was deleted.
  */
 /**
- * Rewrite the decision log to the entries younger than `cutoff`.
+ * What a cutoff WOULD have removed from the decision ledger — and why nothing is removed.
  *
- * The read-before-rewrite is a TOCTOU hot spot: a concurrent worker can append
- * a fresh audit line between our read and our atomic rewrite, and an atomic
- * rename would silently discard it. So instead of a single read+write we take
- * the cross-process log mutex for the whole loop AND keep a bounded
- * compare-and-append loop behind it — after each rewrite we re-read the log and
- * retry whenever the file grew (a concurrent append slipped in). The loop
- * terminates when the file is stable or after `MAX_LOG_REWRITE_RETRIES`, and a
- * failure here is surfaced as a structured error instead of truncating the log.
+ * THE RENUMBERING REWRITE IS GONE, NOT HIDDEN BEHIND A FLAG. It used to filter the log to
+ * the entries younger than `cutoff` and rewrite the whole file (temp + atomic rename, a
+ * bounded retry loop, the cross-process lock). Against the kernel's hash chain that is not
+ * a destructive-but-honest trim, it is forgery: entry i+1's `prevEntryHash` is the sha256
+ * of entry i's BYTES, so dropping line 3 makes every later line's recorded link wrong —
+ * and re-emitting the survivors with renumbered `sequence` and recomputed hashes produces
+ * a chain that verifies while proving nothing, because it now claims "these decisions were
+ * contiguous" about a file somebody deleted the middle of.
  *
- * Lock required: when the mutex cannot be taken (another process is appending
- * and does not release within the wait budget) the rewrite is REFUSED with a
- * structured error rather than proceeding unlocked — an unlocked rename can
- * drop a concurrent appender's audit line (its fd points at the pre-rename
- * inode) with no way to detect the loss afterwards.
+ * So prune reads, reports, and touches nothing. The two legal moves belong to the ledger
+ * itself and both are named in the remedy:
  *
- * `deleted` is the TOTAL across every attempt: attempt 1 may already have
- * removed entries before a retry is needed, and reporting only the final
- * attempt's slice would under-count the audit entry (the pre-fix code returned
- * `deleted: 0` on the exhaustion path even after real deletions).
+ *   1. keep appending — an all-legacy log migrates onto the chain on its next write
+ *      (verified before the swap, the original kept beside it at mode 0600) and a healthy
+ *      chain simply grows; a line is bounded by PAYLOAD_MAX_CANONICAL_LENGTH, not by age;
+ *   2. settle a CONTESTED ledger by CHOOSING archive-and-restart
+ *      (`iterate_decision_log({operation:"archive"})` / `archiveContestedDecisionLog`),
+ *      which renames the file to `…contested-<stamp>.jsonl` at mode 0600 and lets the next
+ *      append start a fresh chain. It refuses a healthy or an all-legacy ledger, so it can
+ *      never become a way to erase continuity by accident.
  *
- * @param options.afterRewrite test hook run after each successful rewrite —
- * simulates a concurrent appender landing lines mid-loop (production passes
- * no hook).
- * @param options.lock lock wait/stale budgets (tests pass short waits so the
- * lock-unavailable refusal is exercised in milliseconds, not 5s).
- * @returns the number of entries removed across all attempts, or 0 with an
- *  `error` when nothing needed removing or the rewrite was refused/failed.
+ * `cutoff` stays in the signature because the REPORT needs it: `retained` is exactly the
+ * number of lines the old code would have deleted, which is what a human weighs when
+ * deciding whether a ledger is worth settling.
  */
-export function rewriteDecisionLogKeepingRecent(
+export function pruneDecisionLog(
   projectRoot: string,
   cutoff: string,
-  options: { afterRewrite?: () => void; lock?: { waitMs?: number; staleMs?: number } } = {},
-): { deleted: number; error?: string } {
-  const logPath = join(iterateDir(projectRoot), 'decision-log.jsonl')
-  // Take the cross-process log lock for the ENTIRE loop: with the mutex held,
-  // no concurrent appender can slip a fresh line into the read-before-rewrite
-  // window, so the first attempt is nearly always the only one. (The retry
-  // loop stays as defense-in-depth for a line landing via the afterRewrite
-  // hook or an already-running appender's buffered write.)
-  const { acquired, release } = acquireLogLockChecked(projectRoot, options.lock)
-  if (!acquired) {
-    return {
-      deleted: 0,
-      error:
-        'failed to acquire the decision-log lock — refusing to rewrite the log unlocked ' +
-        '(a concurrent appender could lose an audit line). Retry after the other operation finishes.',
-    }
-  }
-  try {
-    let deletedTotal = 0
-    for (let attempt = 0; attempt < MAX_LOG_REWRITE_RETRIES; attempt++) {
-      try {
-        const entries = readDecisionEntries(projectRoot)
-        const kept = entries.filter((e) => e.timestamp >= cutoff)
-        const deleted = entries.length - kept.length
-        if (deleted === 0) return { deleted: deletedTotal }
-        writeTextAtomic(logPath, kept.map((e) => JSON.stringify(e)).join('\n') + '\n')
-        // The append-side entry-count cache is keyed by path and assumes "same
-        // byte size ⇒ same count"; the rename above replaced the file with a
-        // different number of entries, so drop the stale cache or the next
-        // append can report a wrong entryCount when the sizes coincidentally
-        // match.
-        invalidateLogCountCache(logPath)
-        // Accumulate THIS attempt's deletions before the re-read: whatever the
-        // later attempts (or the exhaustion path) report, these entries are
-        // already gone from disk and must count toward the audit total.
-        deletedTotal += deleted
-        options.afterRewrite?.()
-        // A concurrent appender (cross-process, lock-unavailable path) may have
-        // landed new lines after our read. If the log grew during the write
-        // window, re-read and prune again instead of accepting a lost audit trail.
-        const after = readDecisionEntries(projectRoot)
-        if (after.length <= kept.length) return { deleted: deletedTotal }
-      } catch (err) {
-        return { deleted: deletedTotal, error: `failed to rewrite decision log: ${String(err)}` }
-      }
-    }
-    return { deleted: deletedTotal, error: `failed to rewrite decision log after ${MAX_LOG_REWRITE_RETRIES} attempts` }
-  } finally {
-    release()
+): {
+  deleted: number
+  retained: number
+  contestedLine: number | null
+  code: string
+  error: string
+  remedy: string
+} {
+  const entries = readDecisionEntries(projectRoot)
+  const retained = entries.filter((e) => e.timestamp < cutoff).length
+  const contestedLine = decisionLogContestedLine(projectRoot)
+  return {
+    deleted: 0,
+    retained,
+    contestedLine,
+    code: 'ITERATE_E_DECISION_LOG_IMMUTABLE',
+    error:
+      `the decision ledger is a hash chain: ${retained} ` +
+      `entr${retained === 1 ? 'y' : 'ies'} older than ${cutoff} are RETAINED, not rewritten`,
+    remedy:
+      'a chain cannot be trimmed in place — every later line links to the bytes of the line before it, ' +
+      'so deleting or renumbering one forges the rest. Two legal moves: (1) keep appending: an all-legacy ' +
+      'ledger migrates onto the chain on its next write and a healthy one just grows; (2) if an append ' +
+      'reports ITERATE_E_DECISION_LOG_INTERLEAVED, a human settles it with archive-and-restart — ' +
+      'iterate_decision_log({operation:"archive"}) moves the contested ledger aside losslessly (mode 0600) ' +
+      'and the next append starts a clean chain. Prune never deletes a ledger line.',
   }
 }
 
+/**
+ * Actually prune the runtime artifacts (only called when dryRun=false).
+ * Returns a detailed report of what was deleted.
+ */
+
 /**
  * Execute the prune: apply every cleanable item reported by `inspectPrune`.
  * Every step is individually hedged so a single disk failure degrades to a
@@ -396,6 +386,10 @@ export function executePrune(
   report: ReturnType<typeof inspectPrune>,
 ): {
   deletedLogEntries: number
+  /** Lines older than the cutoff the ledger KEEPS (a hash chain is not age-trimmed). */
+  retainedLogEntries: number
+  /** The immutable-ledger refusal: code + what was not done + the two legal moves. */
+  decisionLog: { code: string; error: string; remedy: string }
   deletedCheckpoint: boolean
   deletedBackups: string[]
   deletedTemps: string[]
@@ -405,8 +399,13 @@ export function executePrune(
   errors: string[]
 } {
   const cutoff = cutoffTimestamp(retainDays)
+  const ledger = pruneDecisionLog(projectRoot, cutoff)
   const result = {
+    // Always 0: a ledger line is never deleted by prune. The field stays because the
+    // prune summary (and the audit entry it logs) has always carried it.
     deletedLogEntries: 0,
+    retainedLogEntries: ledger.retained,
+    decisionLog: { code: ledger.code, error: ledger.error, remedy: ledger.remedy },
     deletedCheckpoint: false,
     deletedBackups: [] as string[],
     deletedTemps: [] as string[],
@@ -416,15 +415,15 @@ export function executePrune(
     errors: [] as string[],
   }
 
-  // 1. Rewrite the decision log, keeping only recent entries. Atomic
-  // (temp + rename) under the cross-process log mutex — REFUSED with a
-  // structured error when the lock cannot be taken (an unlocked rename could
-  // drop a concurrent appender's audit line); the bounded compare-and-append
-  // loop behind it guards the remaining read-before-rewrite window.
-  {
-    const { deleted, error } = rewriteDecisionLogKeepingRecent(projectRoot, cutoff)
-    result.deletedLogEntries = deleted
-    if (error) result.errors.push(error)
+  // 1. The decision ledger is READ-ONLY here: nothing is deleted, nothing is rewritten
+  // (see `pruneDecisionLog`). The refusal is always reported; a CONTESTED ledger is
+  // additionally escalated into `errors`, because that is a broken state only a human can
+  // settle — archive-and-restart is offered, never taken.
+  if (ledger.contestedLine !== null) {
+    result.errors.push(
+      `line ${ledger.contestedLine} of the decision ledger is a legacy record appended after the chain ` +
+        `began (ITERATE_E_DECISION_LOG_INTERLEAVED). ${ledger.remedy}`,
+    )
   }
 
   // 2. Remove checkpoint — ONLY when it is actually stale (older than
@@ -548,9 +547,12 @@ export function registerPruneTool(ctx: { tools: { register: (def: ReturnType<typ
       description:
         'Inspect or clean up old iterate runtime artifacts (.iterate/). ' +
         'Defaults to dry-run (report-only, no deletion). Pass `dryRun: false` to actually prune. ' +
-        'Manages: old decision-log entries, stale checkpoints, orphaned fix backups, empty fix rounds, ' +
+        'Manages: stale checkpoints, orphaned fix backups, empty fix rounds, ' +
         'stray temp files left by crashed atomic writes, experience-bank entries beyond the entry cap, ' +
         'and stale defense events. ' +
+        'The decision ledger is NEVER trimmed: it is a hash chain, so a removed or renumbered line ' +
+        'would invalidate every later one. Prune reports how many lines a cutoff would have removed ' +
+        'and names the two legal moves (keep appending / human-chosen archive-and-restart). ' +
         'A non-dry-run appends ONE aggregate summary entry (counts per artifact class) to the decision log.',
       parameters: {
         dryRun: {
@@ -588,7 +590,10 @@ export function registerPruneTool(ctx: { tools: { register: (def: ReturnType<typ
             const registryError = report?.registryError as string | null | undefined
             const lines = [
               `[dry-run] prune report (retainDays=${value.retainDays}):`,
-              `  Decision-log entries to remove: ${report?.oldLogEntries ?? '?'} (of ${report?.totalLogEntries ?? '?'})`,
+              `  Decision-log entries older than retainDays: ${report?.oldLogEntries ?? '?'} (of ${report?.totalLogEntries ?? '?'}) — RETAINED, a hash chain is never trimmed`,
+              report?.contestedLogLine
+                ? `  CONTESTED ledger: line ${report.contestedLogLine} is a legacy record after the chain — settle it with iterate_decision_log({operation:"archive"})`
+                : '  Ledger continuity: no interleaved legacy records',
               `  Checkpoint to delete: ${report?.checkpointStale ? 'yes (stale)' : report?.hasCheckpoint ? 'no (fresh — resume point, kept)' : 'none'}`,
               registryError
                 ? `  Stale backups to delete: none — ${registryError}`
@@ -604,7 +609,7 @@ export function registerPruneTool(ctx: { tools: { register: (def: ReturnType<typ
           }
           const lines = [
             `Prune complete (retainDays=${value.retainDays}):`,
-            `  Deleted ${result?.deletedLogEntries ?? 0} old log entries.`,
+            `  Decision-log lines deleted: ${result?.deletedLogEntries ?? 0} — ${result?.retainedLogEntries ?? 0} older than the cutoff RETAINED (${(result?.decisionLog as { code?: string } | undefined)?.code ?? 'hash chain'}: never renumbered).`,
             `  Checkpoint deleted: ${result?.deletedCheckpoint ? 'yes' : 'no'}`,
             `  Deleted ${(result?.deletedBackups as string[] | undefined)?.length ?? 0} stale backups.`,
             `  Deleted ${(result?.deletedTemps as string[] | undefined)?.length ?? 0} stray temp files.`,
@@ -652,6 +657,10 @@ export function registerPruneTool(ctx: { tools: { register: (def: ReturnType<typ
             action: 'prune',
             retainDays,
             deletedLogEntries: result.deletedLogEntries,
+            retainedLogEntries: result.retainedLogEntries,
+            // The ledger itself was not touched; record WHY, so an audit reader can tell
+            // "prune deleted nothing" apart from "prune was refused by an immutable chain".
+            decisionLogRefusal: result.decisionLog.code,
             deletedCheckpoint: result.deletedCheckpoint,
             deletedBackups: result.deletedBackups.length,
             trimmedEmptyRounds: result.trimmedEmptyRounds,

package/src/types.ts

@@ -53,21 +53,50 @@ export interface IterateConfig {
   personalization?: Record<string, unknown>
 }
 
-/** One entry in the append-only decision log */
+/**
+ * The decision kinds THIS plugin's own producers write.
+ *
+ * DISPLAY AND DOCUMENTATION ONLY — never a validity rule. Membership in this list
+ * used to be the gate on `iterate_decision_log` appends, and it disagreed with the
+ * Python harness's own list (which rejected `round_failed` and `resume` outright):
+ * two shells, two closed sets, one shared ledger. The disagreement WAS the bug.
+ * Validity is now the kernel's kind GRAMMAR (`DECISION_KIND_PATTERN`,
+ * `^[a-z][a-z0-9_]{0,31}$`) — see `src/tools/decision-log.ts` `isDecisionKind` —
+ * deliberately open so adding a kind does not need a kernel release, and so a kind
+ * produced by either shell is readable by the other.
+ */
+export const KNOWN_DECISION_KINDS = [
+  'round_start',
+  'review_result',
+  'atomic_fix',
+  'architectural_fix',
+  'revert',
+  'validation',
+  'decision',
+  'report',
+  'round_failed',
+  'resume',
+] as const
+
+/** One of the kinds this plugin emits itself (a label, not an exhaustive set). */
+export type KnownDecisionKind = (typeof KNOWN_DECISION_KINDS)[number]
+
+/**
+ * One entry in the append-only decision log — the loop's RECORD shape.
+ *
+ * This is what a producer hands to `appendDecisionEntry` and what every reader sees
+ * back. It is NOT the shape on disk: the ledger stores the kernel's canonical
+ * hash-chained entry, and `decision-log.ts` transcribes on the way in
+ * (`type`→`kind`, `data`→`payload`, `timestamp`→`createdAt`) and projects back out
+ * on the way read (`projectDecisionRecord`).
+ *
+ * `type` is an open string (kernel grammar) — see {@link KNOWN_DECISION_KINDS} for
+ * what this plugin actually emits.
+ */
 export interface DecisionLogEntry {
   timestamp: string
   round: number
-  type:
-    | 'round_start'
-    | 'review_result'
-    | 'atomic_fix'
-    | 'architectural_fix'
-    | 'revert'
-    | 'validation'
-    | 'decision'
-    | 'report'
-    | 'round_failed'
-    | 'resume'
+  type: string
   data: Record<string, unknown>
 }