failproofai 1.0.2-beta.3 → 1.0.2-beta.5

package/src/audit/cache.ts

@@ -10,35 +10,38 @@
  * sessions live in a SQLite DB rather than a file with a stable mtime).
  */
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync, chmodSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync, chmodSync, openSync, readSync, closeSync } from "node:fs";
 import { join } from "node:path";
 import { BUILTIN_POLICIES } from "../hooks/builtin-policies";
-import { readInstalledPacks } from "../hooks/pack-manifest";
 import { AUDIT_DETECTORS } from "./detectors";
-import type { TranscriptAuditResult } from "./types";
+import type { TranscriptAuditResult, DetectorSessionState } from "./types";
 import { auditCacheDir } from "../hooks/fp-home";
 
 let cachedEngineVersion: string | null = null;
 let cachedDetectorVersion: string | null = null;
 
 /**
- * Hash of every builtin policy's name + function body, PLUS the identity of every
- * installed pack. Changes when policy code changes, invalidating downstream
- * caches.
+ * Hash of every builtin policy's name and function body. Changes when policy
+ * code changes, invalidating downstream caches.
  *
- * Packs belong in this key because they change what a machine would have caught,
- * which is exactly what a cached audit result claims to know. They are folded in
- * by `id|version|sha256` rather than by source text: the pack's bytes are already
- * digest-pinned, and hashing the loaded source instead would drag in the
- * per-load temporary filename the loader rewrites into every import specifier —
- * which changes on every single run and would cold-rescan the whole history each
- * time.
+ * Nothing else is in it, and specifically not installed packs. The audit
+ * evaluates the implementations compiled into this build and only those —
+ * `replay.ts` iterates `BUILTIN_POLICIES` and never reads the installed packs —
+ * so a pack cannot move an audit result, and keying on one meant every install
+ * or removal cold-rescanned the entire history to reproduce the answers it
+ * already had. The key it replaced folded packs in by `id|version|sha256`; the
+ * rationale was that packs "change what a machine would have caught", which is
+ * true of enforcement and was never true of this replay.
  *
- * **A machine with no packs must hash byte-identically to a build with no pack
- * support at all.** Otherwise merely shipping this feature invalidates every
- * user's audit cache and forces a ~104-second cold rescan on upgrade, for a
- * capability they are not using. Hence the empty pack set contributes nothing —
- * not an empty line, not a separator.
+ * **A machine with no packs hashes byte-identically to the old key**, because
+ * that one contributed nothing for an empty pack set — not an empty line, not a
+ * separator. So this change costs a rescan only to machines that had a pack
+ * installed, and costs it exactly once.
+ *
+ * It is deliberately NOT the source text of anything loaded at runtime: the
+ * loader rewrites a per-load temporary filename into every import specifier, so
+ * hashing loaded source would change on every single run and cold-rescan the
+ * whole history each time.
  */
 function getEngineVersion(): string {
   if (cachedEngineVersion) return cachedEngineVersion;
@@ -46,19 +49,21 @@ function getEngineVersion(): string {
     .map((p) => `${p.name}|${p.fn.toString()}`)
     .sort()
     .join("\n");
-  // Never throws by contract, but this runs inside the audit's own try/catch-free
-  // path and a cache key is not worth taking the run down for.
-  let packBlob = "";
-  try {
-    packBlob = readInstalledPacks()
-      .packs.map((pack) => `${pack.id}|${pack.version}|${pack.sha256}`)
-      .sort()
-      .join("\n");
-  } catch {
-    packBlob = "";
-  }
-  const blob = packBlob ? `${builtinBlob}\n${packBlob}` : builtinBlob;
-  cachedEngineVersion = createHash("sha1").update(blob).digest("hex").slice(0, 16);
+  // Packs are NOT in this key, and that is the whole point of the audit's design
+  // rather than an omission.
+  //
+  // `replay.ts` evaluates `BUILTIN_POLICIES` and nothing else — it never reads
+  // `readInstalledPacks()`, and third-party pack policies are never replayed. So
+  // an installed pack cannot change an audit result, and folding its identity in
+  // here invalidated every cached transcript on a change that provably produces
+  // the same answers: a full rescan of thousands of transcripts to arrive back
+  // where it started. That was survivable while packs were rare; policies ARE
+  // packs now, so installing or removing one is routine.
+  //
+  // If audit is ever extended to evaluate what a pack would actually have
+  // caught, this key has to carry pack identity again — and that change belongs
+  // in the same commit as the one that makes it true.
+  cachedEngineVersion = createHash("sha1").update(builtinBlob).digest("hex").slice(0, 16);
   return cachedEngineVersion;
 }
 
@@ -105,7 +110,29 @@ function getCachePathFor(transcriptPath: string): string {
  * accidentally on some paths) so we reject them outright and force a
  * re-scan.
  */
-export const CACHE_SCHEMA_VERSION = 3;
+/**
+ * v4: entries carry `bytesScanned`, `anchorHash` and `detectorState` so a
+ * transcript that GREW can be resumed from where the last scan stopped
+ * instead of being read from byte zero. v3 entries have none of the three;
+ * they are rejected rather than treated as "scanned 0 bytes", which would
+ * re-scan everything anyway but silently, and rather than as "scanned it
+ * all", which would drop every event appended since.
+ */
+export const CACHE_SCHEMA_VERSION = 4;
+
+/**
+ * How much of the head and of the resume boundary is hashed to decide the
+ * bytes already scanned are still the same bytes.
+ *
+ * A transcript is append-only in normal operation, so a grown file is
+ * ordinarily the same file plus more. "Ordinarily" is not a guarantee: a
+ * compaction can rewrite one in place. Re-reading the whole prefix to be sure
+ * would cost exactly what this change exists to avoid, so two cheap windows
+ * stand in for it — the first bytes of the file, and the last bytes before the
+ * offset being resumed from. A rewrite that leaves both intact and diverges
+ * only in between is not a shape any of these writers produce.
+ */
+const ANCHOR_WINDOW = 4096;
 
 /** Hard expiry: a cached transcript result is rejected on read once it's
  *  this old. Keeps the audit pipeline honest about long-lived results that
@@ -136,39 +163,163 @@ interface CacheEntry {
   engineVersion: string;
   detectorVersion: string;
   result: TranscriptAuditResult;
+  /**
+   * How far into the file the recorded result actually accounts for, at a LINE
+   * boundary. Not the same as `sizeBytes`: a transcript being appended to while
+   * the audit reads it ends in a partial line, which is not parsed and must not
+   * be counted as scanned or the events in it are lost forever.
+   */
+  bytesScanned?: number;
+  /** Head + boundary fingerprint of the bytes already scanned. */
+  anchorHash?: string;
+  /** Stateful detectors' session state at `bytesScanned`, Map-aware encoded. */
+  detectorState?: string;
 }
 
-export function readCachedTranscriptResult(
+/** What a cache lookup can say. */
+export type CacheLookup =
+  | { kind: "hit"; result: TranscriptAuditResult }
+  | {
+      kind: "resume";
+      result: TranscriptAuditResult;
+      fromByte: number;
+      detectorState: DetectorSessionState;
+    };
+
+/**
+ * `Map` does not survive `JSON.stringify` — it encodes as `{}`, silently. The
+ * one stateful detector keeps its state in one, so a plain round-trip would
+ * hand back an empty countdown and quietly change what the detector finds
+ * across a resume boundary. Encoded generically rather than per-detector so a
+ * detector added later cannot forget to do it.
+ */
+function encodeDetectorState(state: DetectorSessionState): string {
+  return JSON.stringify(state, (_k, v) =>
+    v instanceof Map ? { __map: [...v.entries()] } : v,
+  );
+}
+
+function decodeDetectorState(raw: string | undefined): DetectorSessionState {
+  if (!raw) return {};
+  try {
+    return JSON.parse(raw, (_k, v) =>
+      v && typeof v === "object" && Array.isArray((v as { __map?: unknown }).__map)
+        ? new Map((v as { __map: [unknown, unknown][] }).__map)
+        : v,
+    ) as DetectorSessionState;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * A fingerprint of the head of the file and of the bytes immediately before
+ * `upTo`, which together stand in for "the prefix has not been rewritten".
+ * Returns null when the file cannot be read that far — which is itself an
+ * answer: do not resume.
+ */
+export function anchorHashFor(transcriptPath: string, upTo: number): string | null {
+  if (upTo <= 0) return null;
+  let fd: number | null = null;
+  try {
+    fd = openSync(transcriptPath, "r");
+    const head = Buffer.alloc(Math.min(ANCHOR_WINDOW, upTo));
+    readSync(fd, head, 0, head.length, 0);
+    const tailLen = Math.min(ANCHOR_WINDOW, upTo);
+    const tail = Buffer.alloc(tailLen);
+    readSync(fd, tail, 0, tailLen, upTo - tailLen);
+    return createHash("sha1")
+      .update(String(upTo))
+      .update(head)
+      .update(tail)
+      .digest("hex")
+      .slice(0, 32);
+  } catch {
+    return null;
+  } finally {
+    if (fd !== null) {
+      try { closeSync(fd); } catch { /* nothing useful to do */ }
+    }
+  }
+}
+
+/**
+ * What the cache can offer for this transcript: everything, the part before it
+ * grew, or nothing.
+ *
+ * The exact-match case is unchanged and still the common one. What is new is
+ * the middle answer. Keying validity on `(mtime, size)` alone made the cache
+ * all-or-nothing per FILE, so a session gaining a single line was re-parsed and
+ * re-replayed from byte zero — and the files that gain lines are the long-lived
+ * ones, which are also the largest. The cost of an audit tracked the size of
+ * whatever was still being written to, rather than the amount actually written
+ * since last time.
+ */
+export function readCachedTranscript(
   transcriptPath: string,
   mtimeMs: number,
   sizeBytes: number,
-): TranscriptAuditResult | null {
-  if (sizeBytes === 0) return null; // OpenCode and other DB-backed sources
+): CacheLookup | null {
+  if (sizeBytes === 0) return null; // DB-backed sources have no file to key on
   const cachePath = getCachePathFor(transcriptPath);
   if (!existsSync(cachePath)) return null;
   try {
     const raw = readFileSync(cachePath, "utf-8");
     const entry = JSON.parse(raw) as Partial<CacheEntry>;
     if (entry.schemaVersion !== CACHE_SCHEMA_VERSION) return null;
-    if (entry.mtimeMs !== mtimeMs) return null;
-    if (entry.sizeBytes !== sizeBytes) return null;
     if (entry.engineVersion !== getEngineVersion()) return null;
     if (entry.detectorVersion !== getDetectorVersion()) return null;
     // Number.isFinite (not typeof) so a malformed JSON `Infinity` /
     // `NaN` is rejected — those would otherwise bypass the TTL check
     // and pin a stale entry as valid forever.
     if (!Number.isFinite(entry.cachedAt) || Date.now() - (entry.cachedAt as number) > CACHE_TTL_MS) return null;
-    return entry.result ?? null;
+    if (!entry.result) return null;
+
+    if (entry.mtimeMs === mtimeMs && entry.sizeBytes === sizeBytes) {
+      return { kind: "hit", result: entry.result };
+    }
+
+    // Grew. Everything below has to hold before any of the recorded result can
+    // be trusted, and each one failing means a full re-scan rather than a
+    // guess.
+    const from = entry.bytesScanned;
+    if (!Number.isFinite(from) || (from as number) <= 0) return null;
+    if ((from as number) > sizeBytes) return null; // truncated, or rewritten shorter
+    if (!entry.anchorHash) return null;
+    if (anchorHashFor(transcriptPath, from as number) !== entry.anchorHash) return null;
+
+    return {
+      kind: "resume",
+      result: entry.result,
+      fromByte: from as number,
+      detectorState: decodeDetectorState(entry.detectorState),
+    };
   } catch {
     return null;
   }
 }
 
+/**
+ * The exact-match half on its own, for callers that cannot resume.
+ *
+ * Kept because "is this transcript already fully accounted for" is a question
+ * worth being able to ask without also being handed a partial answer to merge.
+ */
+export function readCachedTranscriptResult(
+  transcriptPath: string,
+  mtimeMs: number,
+  sizeBytes: number,
+): TranscriptAuditResult | null {
+  const found = readCachedTranscript(transcriptPath, mtimeMs, sizeBytes);
+  return found?.kind === "hit" ? found.result : null;
+}
+
 export function writeCachedTranscriptResult(
   transcriptPath: string,
   mtimeMs: number,
   sizeBytes: number,
   result: TranscriptAuditResult,
+  resume?: { bytesScanned: number; detectorState: DetectorSessionState },
 ): void {
   if (sizeBytes === 0) return;
   const cachePath = getCachePathFor(transcriptPath);
@@ -183,6 +334,18 @@ export function writeCachedTranscriptResult(
       detectorVersion: getDetectorVersion(),
       result,
     };
+    if (resume && resume.bytesScanned > 0) {
+      const anchor = anchorHashFor(transcriptPath, resume.bytesScanned);
+      // No anchor means no way to prove the prefix later, so the entry is
+      // written WITHOUT resume data: still a valid exact-match entry, just not
+      // a resumable one. Recording an offset we cannot verify would be worse
+      // than recording none.
+      if (anchor) {
+        entry.bytesScanned = resume.bytesScanned;
+        entry.anchorHash = anchor;
+        entry.detectorState = encodeDetectorState(resume.detectorState);
+      }
+    }
     // Set 0o600 at file-creation time so there's no window where the file
     // exists with the umask default (typically 0o644). The chmodSync below is
     // a belt-and-suspenders pass for the case where the file already existed.
@@ -193,8 +356,6 @@ export function writeCachedTranscriptResult(
   }
 }
 
-/** Test helper: reset memoized version hashes (so they recompute after a
- *  detector or policy is monkey-patched in a test). */
 export function _resetVersionCache(): void {
   cachedEngineVersion = null;
   cachedDetectorVersion = null;

package/src/audit/cli-adapters/claude.ts

@@ -5,7 +5,7 @@
  * transcripts under <sessionId>/subagents/) via lib/claude-sessions.ts, then
  * parses each via lib/log-entries.ts.
  */
-import { readFile } from "node:fs/promises";
+import { readFile, open, type FileHandle } from "node:fs/promises";
 import {
   listClaudeProjects,
   listClaudeTranscripts,
@@ -14,6 +14,7 @@ import {
 import { parseLogContent, type LogSource } from "../../../lib/log-entries";
 import type { NormalizedToolEvent, TranscriptMetadata } from "../types";
 import { logEntriesToEvents } from "./shared";
+import type { IncrementalEvents } from "./index";
 
 export interface ListOpts {
   /** Restrict to sessions whose decoded cwd matches one of these paths. */
@@ -95,3 +96,112 @@ export async function streamClaudeEvents(
     cwd,
   });
 }
+
+/**
+ * The events after `fromByte`, and nothing before it.
+ *
+ * Claude writes session transcripts as append-only JSONL, which is what makes
+ * this sound: the bytes before `fromByte` were parsed on an earlier run and
+ * have not moved. The caller has already established that — see
+ * `anchorHashFor` in the audit cache — so this reads from the offset and does
+ * not re-verify it.
+ *
+ * Two details carry the correctness:
+ *
+ *  - The read starts at `fromByte`, which the caller only ever sets to a
+ *    position it previously reported through `bytesConsumed`, so it always
+ *    lands at the start of a line.
+ *  - It stops at the last NEWLINE, not at end-of-file. A transcript is
+ *    frequently being written while the audit reads it, so the tail is often
+ *    half a line; parsing it would drop or mangle that event, and reporting
+ *    the file size as consumed would mean the next run started after it and
+ *    lost it for good.
+ */
+export async function streamClaudeEventsFrom(
+  meta: TranscriptMetadata,
+  fromByte: number,
+): Promise<IncrementalEvents | null> {
+  if (!Number.isFinite(fromByte) || fromByte < 0) return null;
+  let handle: FileHandle | null = null;
+  let chunk: string;
+  try {
+    handle = await open(meta.transcriptPath, "r");
+    const { size } = await handle.stat();
+    if (fromByte > size) return null; // truncated under us; caller re-scans
+    if (fromByte === size) {
+      return { events: [], bytesConsumed: fromByte };
+    }
+    const buf = Buffer.alloc(size - fromByte);
+    await handle.read(buf, 0, buf.length, fromByte);
+    chunk = buf.toString("utf-8");
+  } catch {
+    return null;
+  } finally {
+    if (handle) {
+      try { await handle.close(); } catch { /* nothing useful to do */ }
+    }
+  }
+
+  // Everything up to the last newline is unambiguously complete. What follows
+  // it is either a line still being written, or the last line of a file that
+  // simply does not end in a newline — and those need opposite treatment:
+  // dropping the first is required, dropping the second loses the final event
+  // of every completed transcript.
+  //
+  // The only signal available in the bytes is whether the remainder PARSES. A
+  // half-written JSONL record does not (its braces are unclosed); a whole one
+  // missing only its newline does. So a parseable tail is taken and counted,
+  // and an unparseable one is left for the next run to pick up whole — which
+  // is also what stops it being counted twice when it arrives.
+  const lastNewline = chunk.lastIndexOf("\n");
+  const head = lastNewline === -1 ? "" : chunk.slice(0, lastNewline + 1);
+  const tail = chunk.slice(lastNewline + 1);
+  const tailIsWholeLine = tail.trim().length > 0 && (() => {
+    try {
+      JSON.parse(tail);
+      return true;
+    } catch {
+      return false;
+    }
+  })();
+  const complete = tailIsWholeLine ? head + tail : head;
+  if (complete.length === 0) {
+    // Nothing whole to parse, and the offset does not move.
+    return { events: [], bytesConsumed: fromByte };
+  }
+  // Byte length, not string length — a multi-byte character in the tail would
+  // otherwise put the next resume offset mid-character.
+  const bytesConsumed = fromByte + Buffer.byteLength(complete, "utf-8");
+
+  let entries;
+  try {
+    entries = await parseLogContent(complete, "session" as LogSource);
+  } catch {
+    return null;
+  }
+
+  let cwd = "";
+  for (const line of complete.split("\n", 50)) {
+    if (!line.trim()) continue;
+    try {
+      const parsed = JSON.parse(line) as { cwd?: unknown };
+      if (typeof parsed.cwd === "string" && parsed.cwd.length > 0) {
+        cwd = parsed.cwd;
+        break;
+      }
+    } catch {
+      // skip malformed lines
+    }
+  }
+
+  return {
+    events: logEntriesToEvents(entries, {
+      cli: "claude",
+      sessionId: meta.sessionId,
+      transcriptPath: meta.transcriptPath,
+      cwd,
+    }),
+    bytesConsumed,
+    cwd: cwd || undefined,
+  };
+}

package/src/audit/cli-adapters/index.ts

@@ -11,7 +11,7 @@ import type { IntegrationType } from "../../hooks/types";
 import type { NormalizedToolEvent, TranscriptMetadata } from "../types";
 import type { ListOpts } from "./claude";
 
-import { listClaudeTranscriptMetadata, streamClaudeEvents } from "./claude";
+import { listClaudeTranscriptMetadata, streamClaudeEvents, streamClaudeEventsFrom } from "./claude";
 import { listCodexTranscriptMetadata, streamCodexEvents } from "./codex";
 import { listCopilotTranscriptMetadata, streamCopilotEvents } from "./copilot";
 import { listCursorTranscriptMetadata, streamCursorEvents } from "./cursor";
@@ -26,10 +26,39 @@ import { listGooseTranscriptMetadata, streamGooseEvents } from "./goose";
 
 export type { ListOpts };
 
+/** What an append-only source can hand back when asked to resume. */
+export interface IncrementalEvents {
+  events: NormalizedToolEvent[];
+  /**
+   * How far into the file `events` accounts for, at a LINE boundary — which is
+   * NOT the file size when a transcript is being appended to as it is read. The
+   * trailing partial line is left unparsed and uncounted so the next run picks
+   * it up whole; recording the file size instead would skip it permanently.
+   */
+  bytesConsumed: number;
+  /** The session cwd, when the resumed region carried one. */
+  cwd?: string;
+}
+
 export interface CliAdapter {
   cli: IntegrationType;
   listTranscripts: (opts?: ListOpts) => Promise<TranscriptMetadata[]>;
   streamEvents: (meta: TranscriptMetadata) => Promise<NormalizedToolEvent[]>;
+  /**
+   * Read only the events after `fromByte`, for sources whose transcripts are
+   * append-only files.
+   *
+   * OPTIONAL, and deliberately a separate method rather than a parameter on
+   * `streamEvents`. A parameter an adapter ignored would return the WHOLE
+   * transcript to a caller that is about to merge it into an existing partial
+   * result — every event counted twice, quietly, on exactly the sources that
+   * had not been updated. Absence of this method is what tells the caller it
+   * cannot resume, and there is no way to be wrong about that by omission.
+   */
+  streamEventsFrom?: (
+    meta: TranscriptMetadata,
+    fromByte: number,
+  ) => Promise<IncrementalEvents | null>;
 }
 
 export const ADAPTERS: Record<IntegrationType, CliAdapter> = {
@@ -37,6 +66,7 @@ export const ADAPTERS: Record<IntegrationType, CliAdapter> = {
     cli: "claude",
     listTranscripts: listClaudeTranscriptMetadata,
     streamEvents: streamClaudeEvents,
+    streamEventsFrom: streamClaudeEventsFrom,
   },
   codex: {
     cli: "codex",