failproofai 1.0.2-beta.3 → 1.0.2-beta.5

package/src/audit/index.ts

@@ -15,7 +15,7 @@ import { INTEGRATION_TYPES, type IntegrationType } from "../hooks/types";
 import { ADAPTERS } from "./cli-adapters";
 import { AUDIT_DETECTORS } from "./detectors";
 import { severityForBuiltin } from "./features";
-import { readCachedTranscriptResult, writeCachedTranscriptResult } from "./cache";
+import { readCachedTranscript, writeCachedTranscriptResult } from "./cache";
 import { initReplay, replayEvent, restoreReplay } from "./replay";
 import {
   AUDIT_EXAMPLE_MAX_CHARS,
@@ -87,7 +87,24 @@ function parseSinceOpt(since: string | undefined): number | undefined {
   throw new Error(`Invalid --since value: "${since}" (expected e.g. "7d", "30d", or "2026-04-01")`);
 }
 
-async function scanOneTranscript(meta: TranscriptMetadata): Promise<TranscriptAuditResult> {
+/**
+ * What a scan covered, alongside the result — so the caller can record a resume
+ * point without re-deriving it.
+ *
+ * `bytesScanned` is 0 when the source cannot be resumed at all (a database, or
+ * an adapter with no `streamEventsFrom`), which is what stops a resume point
+ * being written for something that has no bytes to resume from.
+ */
+interface ScanOutcome {
+  result: TranscriptAuditResult;
+  bytesScanned: number;
+  detectorState: DetectorSessionState;
+}
+
+async function scanOneTranscript(
+  meta: TranscriptMetadata,
+  resume?: { fromByte: number; detectorState: DetectorSessionState },
+): Promise<ScanOutcome> {
   const empty: TranscriptAuditResult = {
     transcriptPath: meta.transcriptPath,
     cli: meta.cli,
@@ -102,17 +119,54 @@ async function scanOneTranscript(meta: TranscriptMetadata): Promise<TranscriptAu
     rangeByName: {},
   };
 
+  const adapter = ADAPTERS[meta.cli];
   // Stream failures must propagate so the orchestrator counts them in
   // `errors` rather than silently returning an empty hits map.
-  const events = await ADAPTERS[meta.cli].streamEvents(meta);
-  if (events.length === 0) return empty;
+  let events: NormalizedToolEvent[];
+  let bytesScanned = 0;
+  // A resumed scan carries the stateful detectors' state forward across the
+  // boundary. Starting them empty would silently change what they find — the
+  // re-read detector's countdown spans tool calls, so an edit before the offset
+  // and its wasteful re-read after it would stop being a pair.
+  const sessionState: DetectorSessionState = resume?.detectorState ?? {};
+
+  if (adapter.streamEventsFrom) {
+    // A full scan is a resume from zero, deliberately — not a separate branch.
+    //
+    // The separate branch is what got this wrong: it recorded `meta.sizeBytes`
+    // as scanned, which is the whole FILE, including a trailing partial line
+    // that was never parsed. Transcripts are appended to while the audit reads
+    // them, so that tail is routinely half a line — and the next run then
+    // resumed past an event nobody had read. Only the reader knows where the
+    // last complete line ended, so only the reader gets to say.
+    const scan = await adapter.streamEventsFrom(meta, resume?.fromByte ?? 0);
+    // Null means it could not be read from there after all — truncated,
+    // unreadable, unparseable. A thrown error would count as a scan failure,
+    // so fall back to a full read instead and keep the transcript scannable.
+    if (!scan) {
+      events = await adapter.streamEvents(meta);
+      bytesScanned = 0;
+    } else {
+      events = scan.events;
+      bytesScanned = scan.bytesConsumed;
+      empty.cwd = scan.cwd ?? "";
+    }
+  } else {
+    // A source with no byte offsets to speak of — a database. Scanned whole,
+    // every time, and given no resume point to mislead the next run with.
+    events = await adapter.streamEvents(meta);
+    bytesScanned = 0;
+  }
+
+  if (events.length === 0) {
+    return { result: empty, bytesScanned, detectorState: sessionState };
+  }
 
   const result = empty;
   result.eventsScanned = events.length;
   // Capture the session's cwd from the first event that carried one — every
   // event in a single transcript shares the same cwd by construction.
-  result.cwd = events[0].cwd || "";
-  const sessionState: DetectorSessionState = {};
+  result.cwd = result.cwd || events[0].cwd || "";
 
   for (const event of events) {
     // Run audit detectors first (stateful, must see every event).
@@ -146,7 +200,7 @@ async function scanOneTranscript(meta: TranscriptMetadata): Promise<TranscriptAu
     }
   }
 
-  return result;
+  return { result, bytesScanned, detectorState: sessionState };
 }
 
 function formatPolicyExample(_policyName: string, event: NormalizedToolEvent): string {
@@ -159,6 +213,55 @@ function formatPolicyExample(_policyName: string, event: NormalizedToolEvent): s
   return `${event.toolName}`;
 }
 
+/**
+ * Fold a tail-scan into the result the cache already held for the same
+ * transcript.
+ *
+ * Mirrors `recordHit` exactly, and has to: counts add, examples stay capped at
+ * the same limit, ranges widen. The one asymmetry is `cwd` — the cached value
+ * came from the FIRST event in the file and the tail's came from the first
+ * event after the offset, so the older one wins.
+ */
+function mergeIncremental(
+  cached: TranscriptAuditResult,
+  tail: TranscriptAuditResult,
+): TranscriptAuditResult {
+  const out: TranscriptAuditResult = {
+    ...cached,
+    mtimeMs: tail.mtimeMs,
+    sizeBytes: tail.sizeBytes,
+    cwd: cached.cwd || tail.cwd || "",
+    eventsScanned: (cached.eventsScanned ?? 0) + (tail.eventsScanned ?? 0),
+    hitsByName: { ...cached.hitsByName },
+    examplesByName: {},
+    rangeByName: { ...cached.rangeByName },
+  };
+  for (const [name, list] of Object.entries(cached.examplesByName)) {
+    out.examplesByName[name] = [...list];
+  }
+  for (const [name, count] of Object.entries(tail.hitsByName)) {
+    out.hitsByName[name] = (out.hitsByName[name] ?? 0) + count;
+  }
+  for (const [name, list] of Object.entries(tail.examplesByName)) {
+    const exs = out.examplesByName[name] ?? [];
+    for (const ex of list) {
+      if (exs.length >= AUDIT_MAX_EXAMPLES_PER_NAME) break;
+      exs.push(ex);
+    }
+    out.examplesByName[name] = exs;
+  }
+  for (const [name, range] of Object.entries(tail.rangeByName)) {
+    const existing = out.rangeByName[name];
+    if (!existing) {
+      out.rangeByName[name] = { ...range };
+    } else {
+      if (range.first < existing.first) existing.first = range.first;
+      if (range.last > existing.last) existing.last = range.last;
+    }
+  }
+  return out;
+}
+
 function recordHit(
   result: TranscriptAuditResult,
   name: string,
@@ -300,14 +403,31 @@ async function runAuditInner(opts: RunAuditOptions, startedAt: number): Promise<
   let skipped = 0;
   let errors = 0;
   const tasks = allTranscripts.map((meta) => async (): Promise<TranscriptAuditResult> => {
+    let resume: { fromByte: number; detectorState: DetectorSessionState } | undefined;
+    let cachedPrefix: TranscriptAuditResult | null = null;
     if (!opts.noCache) {
-      const cached = readCachedTranscriptResult(meta.transcriptPath, meta.mtimeMs, meta.sizeBytes);
-      if (cached) return cached;
+      const found = readCachedTranscript(meta.transcriptPath, meta.mtimeMs, meta.sizeBytes);
+      if (found?.kind === "hit") return found.result;
+      resume = found?.kind === "resume"
+        ? { fromByte: found.fromByte, detectorState: found.detectorState }
+        : undefined;
+      cachedPrefix = found?.kind === "resume" ? found.result : null;
     }
     try {
-      const fresh = await scanOneTranscript(meta);
+      const scan = await scanOneTranscript(meta, resume);
+      // A resumed scan produced hits for the TAIL only; the cached result holds
+      // everything before it. Merging is what makes the two halves one answer.
+      const fresh = cachedPrefix ? mergeIncremental(cachedPrefix, scan.result) : scan.result;
       if (!opts.noCache) {
-        writeCachedTranscriptResult(meta.transcriptPath, meta.mtimeMs, meta.sizeBytes, fresh);
+        writeCachedTranscriptResult(
+          meta.transcriptPath,
+          meta.mtimeMs,
+          meta.sizeBytes,
+          fresh,
+          scan.bytesScanned > 0
+            ? { bytesScanned: scan.bytesScanned, detectorState: scan.detectorState }
+            : undefined,
+        );
       }
       return fresh;
     } catch {

package/src/audit/replay.ts

@@ -13,19 +13,13 @@
  * execSync against live git, so they never fire on PreToolUse/PostToolUse
  * replay — no explicit skip needed.
  */
-import { resolve } from "node:path";
-import { existsSync } from "node:fs";
 import type { EvaluationResult } from "../hooks/policy-evaluator";
 import { evaluatePolicies } from "../hooks/policy-evaluator";
 import { BUILTIN_POLICIES, registerBuiltinPolicies } from "../hooks/builtin-policies";
-import { PACK_ENTRY_ASSET, bundledPackDir } from "../hooks/pack-store";
-import { loadCustomHooks } from "../hooks/custom-hooks-loader";
-import { clearCustomHooks } from "../hooks/custom-hooks-registry";
 import {
   clearPolicies,
   getAllPolicies,
   normalizePolicyName,
-  registerPolicy,
   setAllPolicies,
 } from "../hooks/policy-registry";
 import type { RegisteredPolicy } from "../hooks/policy-types";
@@ -46,10 +40,20 @@ let initialized = false;
  *  (e.g. the Next.js dashboard) doesn't wipe any prior policy registrations. */
 let savedSnapshot: RegisteredPolicy[] | null = null;
 
-/** Register every builtin policy (regardless of user config) so the replay
- *  shows what *could* be caught, not just what's currently enabled. Called
- *  once per `runAudit` invocation. Snapshots the existing registry so it can
- *  be restored by `restoreReplay()` once the audit is done. */
+/**
+ * Register every builtin policy (regardless of user config) so the replay shows
+ * what *could* be caught, not just what is currently enabled. Called once per
+ * `runAudit` invocation. Snapshots the existing registry so `restoreReplay()`
+ * can put it back.
+ *
+ * The implementations compiled into this build, and ONLY those. The audit used
+ * to prefer a vendored `policy-pack/` copy where one existed, which was
+ * meaningful while the package shipped that directory; it no longer ships it, so
+ * that branch could not fire in any published build and existed only to be
+ * misread as "the audit scores against installed packs". It does not, and must
+ * not: an audit is a fixed yardstick, and one that changed shape with whatever
+ * pack a machine happened to have could not be compared against its own history.
+ */
 export async function initReplay(): Promise<void> {
   if (initialized) return;
   savedSnapshot = getAllPolicies();
@@ -57,90 +61,10 @@ export async function initReplay(): Promise<void> {
   const enabled = BUILTIN_POLICIES
     .map((p) => p.name)
     .filter((n) => !SKIP_POLICIES.has(normalizePolicyName(n)));
-  if (!(await registerFromVendoredPack(enabled))) registerBuiltinPolicies(enabled);
+  registerBuiltinPolicies(enabled);
   initialized = true;
 }
 
-/**
- * Register the audit's policies from the pack that ships inside the package,
- * rather than from implementations compiled into this build.
- *
- * The audit scores by RUNNING the policies — three of its four penalty buckets
- * are replay hits — so the day the builtins stop being compiled in, this is
- * what keeps `failproofai audit` reporting the same findings and the same score
- * for the same transcripts. The pack carries the identical functions:
- * `builtin-pack-conformance.test.ts` replays a corpus through both and asserts
- * the verdicts do not diverge.
- *
- * Two things it must get right, or the audit quietly changes:
- *
- * - The pack cannot carry `block-failproofai-commands`, because `alwaysOn` is
- *   refused by the pack loader by design — a downloaded file that no local
- *   command can switch off. It is registered from the compiled side, so the
- *   replayed set is the same 39 either way.
- * - The skip list still applies. `warn-repeated-tool-calls` writes a per-session
- *   sidecar into the user's real transcript directory on every evaluation.
- *
- * Returns false when there is no vendored pack to read — a source checkout that
- * has not run `build:pack`, or a tarball packed without it — and the caller
- * falls back to the compiled implementations. An audit that silently scored on
- * fewer policies would be worse than either.
- */
-async function registerFromVendoredPack(enabled: string[]): Promise<boolean> {
-  const dir = bundledPackDir();
-  if (!dir) return false;
-  const entry = resolve(dir, PACK_ENTRY_ASSET);
-  if (!existsSync(entry)) return false;
-
-  const wanted = new Set(enabled);
-  let hooks;
-  try {
-    clearCustomHooks();
-    hooks = await loadCustomHooks(entry, { strict: true });
-  } catch {
-    return false;
-  } finally {
-    clearCustomHooks();
-  }
-  if (hooks.length === 0) return false;
-  const fromPack = new Map(hooks.map((hook) => [hook.name, hook]));
-
-  // Registered in CATALOG ORDER, taking the pack's function where it has one and
-  // the compiled one for the guard the pack may not carry.
-  //
-  // Order is not cosmetic here. `getPoliciesForEvent` sorts by priority only and
-  // V8's sort is stable, so registration order decides EVALUATION order, and
-  // `evaluatePolicies` stops at the first deny — which means the order decides
-  // which policy is CREDITED for a hit. Appending the always-on guard after the
-  // pack instead of at its catalog position moved it from index 11 to last, and
-  // re-attributed 3 events in a 23,477-event corpus: 456 -> 453 for the guard,
-  // with the denies it used to shadow surfacing as block-force-push and
-  // block-gh-pipeline. The audit's per-policy counts are what a user reads, so
-  // that is a changed audit, not a changed internal.
-  let registered = 0;
-  for (const policy of BUILTIN_POLICIES) {
-    if (!wanted.has(policy.name)) continue;
-    const packHook = fromPack.get(policy.name);
-    if (policy.alwaysOn || !packHook) {
-      registerPolicy(policy.name, policy.description, policy.fn, policy.match, 0, policy.params);
-      continue;
-    }
-    registerPolicy(
-      packHook.name,
-      packHook.description ?? policy.description,
-      packHook.fn,
-      packHook.match ?? policy.match,
-      0,
-      policy.params,
-    );
-    registered += 1;
-  }
-  // Nothing came from the pack: it loaded but carried none of the names the
-  // audit replays. Falling back is more honest than scoring on the compiled set
-  // while claiming the pack lane ran.
-  return registered > 0;
-}
-
 /** Restore the registry to whatever was there before `initReplay()`. Safe to
  *  call when not initialized (no-op). Always paired with `initReplay()` in a
  *  try/finally inside `runAudit()`. */

package/src/hooks/configure-wizard.ts

@@ -32,7 +32,6 @@ import { dirname, resolve, sep } from "node:path";
 
 import {
   selectOne,
-  multiSelect,
   promptText,
   intro,
   outro,
@@ -223,7 +222,6 @@ export function buildAgentChoices(scope: HookScope, cwd: string): MultiChoice<In
   });
 }
 
-const ALL_CLIS = "__all_clis__";
 
 const DIM_NOTE = "(auto-loaded)";
 
@@ -938,54 +936,24 @@ export async function runConfigureWizard(io: WizardIO = {}): Promise<WizardResul
   // An "Everything available" row protects every supported CLI (detected +
   // set-up-ahead); when ticked it wins over the individual boxes.
   //
-  // Always asked, never inferred. This is the one decision setup genuinely
-  // cannot make for you: which agents on this machine should be guarded. The
-  // detected ones are pre-ticked, so the common answer is a single keypress.
+  // Every supported agent, detected or not — setup asks nothing about this.
   //
-  // No `allowBack`: with the policy step and the scope fork both gone, there is
-  // nothing inside setup to return to.
-  let clisSel: string[];
-  {
-    const picked = await multiSelect<string>({
-      message: "Which harnesses should it protect?",
-      choices: [
-        {
-          label: "Everything available",
-          value: ALL_CLIS,
-          // Counts only what this scope can actually take — expanding to all 12
-          // under project scope is what crashed the apply on Hermes.
-          hint: `protect all ${clisSupportingScope(primaryScope).length} CLIs configurable here`,
-          // A selector, not a harness. Counting it gave "13 harnesses" for
-          // the 12 supported CLIs, and listed "Everything available" among them.
-          summaryExclude: true,
-        },
-        ...buildAgentChoices(primaryScope, cwd),
-      ],
-      minSelected: 1,
-      summaryNoun: "harnesses",
-      hint: "detected CLIs are pre-selected · space toggles · ctrl+a all · ↵ confirm",
-      stdin,
-      stdout,
-    });
-    if (picked === null) return cancel();
-    clisSel = picked;
-  }
-  // Filter to what the chosen scopes support in BOTH branches: "Everything
-  // available" must not expand to CLIs that cannot take any selected scope,
-  // and a locked row can't be ticked but belt-and-braces keeps the invariant
-  // local to the one place `clis` is built.
+  // It used to be the one question left, on the reasoning that which agents to
+  // guard is a real choice. It stopped being one when this package stopped
+  // shipping policies: hooks alone enforce nothing, so wiring them everywhere
+  // costs a config entry and changes no behaviour until a pack arrives. What it
+  // buys is that an agent installed NEXT WEEK is guarded from its first tool
+  // call, instead of running unguarded until somebody remembers to re-run setup
+  // — and nobody remembers, because nothing tells them to.
+  //
+  // ALL twelve, not the detected ones, for exactly that reason. The cost is
+  // honest and worth naming: hook config appears under `~/.cursor/`,
+  // `~/.factory/` and the rest for agents that may never be installed.
   //
-  // The union across scopes, not the intersection: under "Both", a user-scope-
-  // only gateway like Hermes is still installable via the user half, and
-  // dropping it because project scope cannot take it would silently protect
-  // less than the user asked for. `installHooks` is called per scope below and
-  // skips what a given scope cannot take.
-  const supported = new Set(scopes.flatMap((s) => clisSupportingScope(s)));
-  const clis: IntegrationType[] = (
-    clisSel.includes(ALL_CLIS)
-      ? [...INTEGRATION_TYPES]
-      : (clisSel.filter((v) => v !== ALL_CLIS) as IntegrationType[])
-  ).filter((id) => supported.has(id));
+  // Which policies run, and on which of these agents, is chosen at
+  // `failproofai policies add` — where the user is looking at a real list
+  // instead of answering in the abstract.
+  const clis: IntegrationType[] = [...clisSupportingScope(primaryScope)];
 
   // 4 — Connect this machine? Last, because by this point the user has decided
   // what to protect, so "would you like to see it in a dashboard?" follows

package/src/hooks/handler.ts

@@ -425,9 +425,16 @@ export async function evaluateHookEvent(
         if (!cloudManaged && policyId && disabledCustomPolicies.has(policyId)) continue;
         // Same rule for a session pause — it suspends local policy, never cloud.
         if (!cloudManaged && activePause) continue;
+        // A pack narrowed to particular agents does not fire on the others.
+        // Setup wires hooks into every supported CLI — hooks alone enforce
+        // nothing — so the choice of which agents a pack guards is made when the
+        // pack is installed, and this is where it takes effect. `clis: null`
+        // means every agent, which is what an install with no narrowing writes
+        // and what every pack installed before this field existed reads as.
+        if (pack?.clis && !pack.clis.includes(cli)) continue;
         // A pack's artifact registers everything it contains; the user may have
         // taken only some of it. `enabled: null` means the whole pack, which is
-        // what `pack add` writes when no selection was made.
+        // what `policies add` writes when no selection was made.
         if (pack?.enabled && !pack.enabled.includes(hook.name)) continue;
         // A pack policy whose name is an ENABLED BUILTIN is the same guard
         // twice. The two register under different keys — `failproofai/block-sudo`

package/src/hooks/manager.ts

@@ -9,12 +9,12 @@ import { execSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { resolve, basename } from "node:path";
 import { platform, arch, release, hostname } from "node:os";
-import {
+import { INTEGRATION_TYPES,
   HOOK_SCOPES,
   type HookScope,
   type IntegrationType,
 } from "./types";
-import { claudeCode, getIntegration, settingsPathsFor } from "./integrations";
+import { claudeCode, getIntegration, settingsPathsFor, type Integration } from "./integrations";
 import { promptPolicySelection } from "./install-prompt";
 import { configuredCustomPolicyPaths, readMergedHooksConfig, readScopedHooksConfig, writeScopedHooksConfig, syncConventionPolicies, findProjectConfigDir } from "./hooks-config";
 import type { HooksConfig, ConventionPolicyRecord } from "./policy-types";
@@ -256,8 +256,77 @@ function deduplicateScopes(scopes: readonly HookScope[], cwd?: string): HookScop
   });
 }
 
+/**
+ * Is ANY agent CLI wired to call failproofai at this scope?
+ *
+ * Asked Claude Code and only Claude Code, which made it wrong for every machine
+ * guarded through one of the other eleven. A user with hooks in `~/.codex/` was
+ * told nothing was installed — quietly, while this only tinted a subtitle, and
+ * loudly once the listing started warning that every policy shown was inert.
+ * Reported from a real machine set up for codex.
+ *
+ * Any one integration answering yes is enough: the question is whether
+ * enforcement can reach this machine at all, not whether a particular agent is
+ * covered. Which agents specifically is a different question, and the per-CLI
+ * rows of the listing already answer it.
+ */
 export function hooksInstalledInSettings(scope: HookScope, cwd?: string): boolean {
-  return claudeCode.hooksInstalledInSettings(scope, cwd);
+  return integrationsInstalledAt(scope, cwd).length > 0;
+}
+
+/**
+ * Which integrations are wired at EXACTLY this scope — the honest form of the
+ * question above, and the one the multi-scope warning needs.
+ *
+ * Two ways an integration answers yes for a scope it is not actually installed
+ * at, both of which made `failproofai policies` warn about "hooks in multiple
+ * scopes" on a machine whose hooks are in exactly one file:
+ *
+ *  - It does not SUPPORT the scope. Hermes and OpenClaw are user-scope only
+ *    (`HERMES_HOOK_SCOPES = ["user"]`) and their `getSettingsPath` ignores the
+ *    scope argument entirely, so they hand back the user file — and report it
+ *    installed — for `project` and `local` alike. `scopes` already declares
+ *    this; nothing was reading it.
+ *  - Its project path RESOLVES to its user path. Run this from `$HOME` and
+ *    `<cwd>/.claude/settings.json` is `~/.claude/settings.json`: one file,
+ *    counted as two scopes. Cheap to detect and impossible to get right by
+ *    asking each integration separately.
+ */
+export function integrationsInstalledAt(scope: HookScope, cwd?: string): IntegrationType[] {
+  return INTEGRATION_TYPES.filter((id) => {
+    try {
+      const integration = getIntegration(id);
+      if (!integration.scopes.includes(scope)) return false;
+      if (!integration.hooksInstalledInSettings(scope, cwd)) return false;
+      // A narrower scope that lands on the same file as a wider one is that
+      // wider one, seen twice. Attribute it to the widest scope that resolves
+      // there so exactly one of them counts.
+      const here = integration.getSettingsPath(scope, cwd);
+      return !HOOK_SCOPES.some(
+        (other) =>
+          other !== scope &&
+          HOOK_SCOPES.indexOf(other) < HOOK_SCOPES.indexOf(scope) &&
+          integration.scopes.includes(other) &&
+          safeSettingsPath(integration, other, cwd) === here,
+      );
+    } catch {
+      // An integration whose settings file is unreadable is not evidence that
+      // nothing is installed — keep asking the rest.
+      return false;
+    }
+  });
+}
+
+function safeSettingsPath(
+  integration: Integration,
+  scope: HookScope,
+  cwd?: string,
+): string | null {
+  try {
+    return integration.getSettingsPath(scope, cwd);
+  } catch {
+    return null;
+  }
 }
 
 export interface InstallHooksOptions {