failproofai 1.0.1 → 1.0.2-beta.0

package/scripts/translate-docs/translator.ts

@@ -44,6 +44,24 @@ const MAX_ATTEMPTS =
     ? parsedMaxAttempts
     : 3;
 
+// How far past the source an output may run before a max_tokens stop reads as a
+// repetition loop rather than a large page. Translations of the verbose targets
+// (hi, vi, ja) sit near 2x the source's token estimate; 6x is far outside that
+// band and was ~16x in the observed failure.
+// Validated the same way MAX_TOKENS and MAX_ATTEMPTS above are, and for a
+// sharper reason: `parseInt(...) || 6` accepts a NEGATIVE, and a negative ratio
+// makes `output > source * ratio` true for every response — so a genuinely
+// oversized page would be classified as a runaway and burn all three attempts
+// to arrive exactly where it started.
+const parsedRunawayRatio = Number.parseInt(
+  process.env.TRANSLATE_RUNAWAY_RATIO ?? "",
+  10,
+);
+const RUNAWAY_RATIO =
+  Number.isInteger(parsedRunawayRatio) && parsedRunawayRatio > 0
+    ? parsedRunawayRatio
+    : 6;
+
 function getClient(): Anthropic {
   if (!client) {
     // Default 5 retries (up from SDK default of 2) so transient
@@ -151,10 +169,32 @@ export async function translateContent(
   // consolidate publish step. If a page ever legitimately needs more than
   // MAX_TOKENS, raise TRANSLATE_MAX_TOKENS or split the source.
   if (response.stop_reason === "max_tokens") {
-    throw new Error(
+    // TWO different failures wear this stop_reason, and only one of them is
+    // "the page is too big". A 16 KB source whose Vietnamese translation
+    // emitted 64 000 output tokens — observed on the box, 2026-08-18,
+    // reference/cloud-cli.mdx [vi] — is a DEGENERATE SAMPLE: the model fell
+    // into a repetition loop. Resampling fixes that; splitting the source does
+    // not, and treating it as a size problem sent one page's hiccup on to fail
+    // the whole nightly run.
+    //
+    // The tell is the ratio. Source bytes / 4 is a rough token count, and a
+    // faithful translation lands within a small multiple of it — even for the
+    // verbose target languages. Far beyond that is the model talking to
+    // itself, so the error is marked RETRYABLE and translateValidated spends
+    // an attempt on it. A source that genuinely approaches the ceiling keeps
+    // the old fail-loud behaviour, because retrying it would burn the budget
+    // three times to reach the same place.
+    const sourceTokensApprox = Math.max(1, Math.ceil(content.length / 4));
+    const runaway = response.usage.output_tokens > sourceTokensApprox * RUNAWAY_RATIO;
+    const err = new Error(
       `translation truncated at max_tokens=${MAX_TOKENS} ` +
-        `(output ${response.usage.output_tokens} tokens) — source too large to translate in one request`,
+        `(output ${response.usage.output_tokens} tokens, source ~${sourceTokensApprox}) — ` +
+        (runaway
+          ? "output dwarfs the source, which is a runaway sample rather than an oversized page"
+          : "source too large to translate in one request"),
     );
+    (err as Error & { retryable?: boolean }).retryable = runaway;
+    throw err;
   }
 
   const translated =
@@ -211,15 +251,30 @@ export async function translateValidated(opts: {
         ? { attempt, maxAttempts: MAX_ATTEMPTS, error: lastError }
         : undefined;
 
-    // No try/catch: a transport/auth/max_tokens throw is not a validity
-    // failure — let it propagate so it is never silently retried as one.
-    const result = await translateContent(
-      opts.source,
-      opts.lang,
-      opts.langName,
-      opts.model,
-      feedback,
-    );
+    // Transport/auth throws still propagate untouched — they are not validity
+    // failures and must never be silently retried as one. The ONE exception is
+    // a truncation flagged `retryable` (a runaway sample, see translateContent):
+    // that is a bad draw, and a bad draw is exactly what this loop is for.
+    let result: Awaited<ReturnType<typeof translateContent>>;
+    try {
+      result = await translateContent(
+        opts.source,
+        opts.lang,
+        opts.langName,
+        opts.model,
+        feedback,
+      );
+    } catch (e) {
+      const err = e as Error & { retryable?: boolean };
+      if (!err.retryable || attempt === MAX_ATTEMPTS) throw err;
+      lastError =
+        "The previous attempt ran past the output limit by repeating itself. " +
+        "Translate the document once, completely, and stop.";
+      console.warn(
+        `  ${opts.label} -> attempt ${attempt}/${MAX_ATTEMPTS} overran the output limit; retrying`,
+      );
+      continue;
+    }
     inputTokens += result.inputTokens;
     outputTokens += result.outputTokens;
 

package/src/audit/cache.ts

@@ -13,6 +13,7 @@ import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync, chmodSync } from "node:fs";
 import { join } from "node:path";
 import { BUILTIN_POLICIES } from "../hooks/builtin-policies";
+import { readInstalledPacks } from "../hooks/pack-manifest";
 import { AUDIT_DETECTORS } from "./detectors";
 import type { TranscriptAuditResult } from "./types";
 import { auditCacheDir } from "../hooks/fp-home";
@@ -20,14 +21,43 @@ import { auditCacheDir } from "../hooks/fp-home";
 let cachedEngineVersion: string | null = null;
 let cachedDetectorVersion: string | null = null;
 
-/** Hash of every builtin policy's name + function body. Changes when policy
- *  code changes, invalidating downstream caches. */
+/**
+ * Hash of every builtin policy's name + function body, PLUS the identity of every
+ * installed pack. Changes when policy code changes, invalidating downstream
+ * caches.
+ *
+ * Packs belong in this key because they change what a machine would have caught,
+ * which is exactly what a cached audit result claims to know. They are folded in
+ * by `id|version|sha256` rather than by source text: the pack's bytes are already
+ * digest-pinned, and hashing the loaded source instead would drag in the
+ * per-load temporary filename the loader rewrites into every import specifier —
+ * which changes on every single run and would cold-rescan the whole history each
+ * time.
+ *
+ * **A machine with no packs must hash byte-identically to a build with no pack
+ * support at all.** Otherwise merely shipping this feature invalidates every
+ * user's audit cache and forces a ~104-second cold rescan on upgrade, for a
+ * capability they are not using. Hence the empty pack set contributes nothing —
+ * not an empty line, not a separator.
+ */
 function getEngineVersion(): string {
   if (cachedEngineVersion) return cachedEngineVersion;
-  const blob = BUILTIN_POLICIES
+  const builtinBlob = BUILTIN_POLICIES
     .map((p) => `${p.name}|${p.fn.toString()}`)
     .sort()
     .join("\n");
+  // Never throws by contract, but this runs inside the audit's own try/catch-free
+  // path and a cache key is not worth taking the run down for.
+  let packBlob = "";
+  try {
+    packBlob = readInstalledPacks()
+      .packs.map((pack) => `${pack.id}|${pack.version}|${pack.sha256}`)
+      .sort()
+      .join("\n");
+  } catch {
+    packBlob = "";
+  }
+  const blob = packBlob ? `${builtinBlob}\n${packBlob}` : builtinBlob;
   cachedEngineVersion = createHash("sha1").update(blob).digest("hex").slice(0, 16);
   return cachedEngineVersion;
 }
@@ -43,6 +73,13 @@ function getDetectorVersion(): string {
   return cachedDetectorVersion;
 }
 
+/** Exposed for `engine-version-packs.test.ts`, which pins the upgrade-cost
+ *  guarantee: a machine with no packs must hash exactly as it did before packs
+ *  existed. Nothing else should call this. */
+export function getEngineVersionForTest(): string {
+  return getEngineVersion();
+}
+
 function getCachePathFor(transcriptPath: string): string {
   const root = auditCacheDir();
   const key = createHash("sha1").update(transcriptPath).digest("hex");

package/src/audit/cli.ts

@@ -58,7 +58,7 @@ export const EXIT_AUDIT_ALREADY_RUNNING = 75;
 export const AUDIT_STAGES: ReadonlyArray<{ label: string; detail: string }> = [
   { label: "discovering transcripts", detail: "walking ~/.claude, ~/.codex, ~/.cursor, …" },
   { label: "parsing session logs", detail: "reading JSONL + sqlite session stores" },
-  { label: "running policy checks", detail: "replaying through 30 builtin policies" },
+  { label: "running policy checks", detail: "replaying every policy against each tool call" },
   { label: "aggregating results", detail: "counting hits, ranking by frequency" },
 ];
 

package/src/audit/index.ts

@@ -261,7 +261,7 @@ function aggregateResults(
 
 export async function runAudit(opts: RunAuditOptions = {}): Promise<AuditResult> {
   const startedAt = Date.now();
-  initReplay();
+  await initReplay();
   try {
     return await runAuditInner(opts, startedAt);
   } finally {

package/src/audit/replay.ts

@@ -13,13 +13,19 @@
  * execSync against live git, so they never fire on PreToolUse/PostToolUse
  * replay — no explicit skip needed.
  */
+import { resolve } from "node:path";
+import { existsSync } from "node:fs";
 import type { EvaluationResult } from "../hooks/policy-evaluator";
 import { evaluatePolicies } from "../hooks/policy-evaluator";
 import { BUILTIN_POLICIES, registerBuiltinPolicies } from "../hooks/builtin-policies";
+import { PACK_ENTRY_ASSET, bundledPackDir } from "../hooks/pack-store";
+import { loadCustomHooks } from "../hooks/custom-hooks-loader";
+import { clearCustomHooks } from "../hooks/custom-hooks-registry";
 import {
   clearPolicies,
   getAllPolicies,
   normalizePolicyName,
+  registerPolicy,
   setAllPolicies,
 } from "../hooks/policy-registry";
 import type { RegisteredPolicy } from "../hooks/policy-types";
@@ -44,17 +50,97 @@ let savedSnapshot: RegisteredPolicy[] | null = null;
  *  shows what *could* be caught, not just what's currently enabled. Called
  *  once per `runAudit` invocation. Snapshots the existing registry so it can
  *  be restored by `restoreReplay()` once the audit is done. */
-export function initReplay(): void {
+export async function initReplay(): Promise<void> {
   if (initialized) return;
   savedSnapshot = getAllPolicies();
   clearPolicies();
   const enabled = BUILTIN_POLICIES
     .map((p) => p.name)
     .filter((n) => !SKIP_POLICIES.has(normalizePolicyName(n)));
-  registerBuiltinPolicies(enabled);
+  if (!(await registerFromVendoredPack(enabled))) registerBuiltinPolicies(enabled);
   initialized = true;
 }
 
+/**
+ * Register the audit's policies from the pack that ships inside the package,
+ * rather than from implementations compiled into this build.
+ *
+ * The audit scores by RUNNING the policies — three of its four penalty buckets
+ * are replay hits — so the day the builtins stop being compiled in, this is
+ * what keeps `failproofai audit` reporting the same findings and the same score
+ * for the same transcripts. The pack carries the identical functions:
+ * `builtin-pack-conformance.test.ts` replays a corpus through both and asserts
+ * the verdicts do not diverge.
+ *
+ * Two things it must get right, or the audit quietly changes:
+ *
+ * - The pack cannot carry `block-failproofai-commands`, because `alwaysOn` is
+ *   refused by the pack loader by design — a downloaded file that no local
+ *   command can switch off. It is registered from the compiled side, so the
+ *   replayed set is the same 39 either way.
+ * - The skip list still applies. `warn-repeated-tool-calls` writes a per-session
+ *   sidecar into the user's real transcript directory on every evaluation.
+ *
+ * Returns false when there is no vendored pack to read — a source checkout that
+ * has not run `build:pack`, or a tarball packed without it — and the caller
+ * falls back to the compiled implementations. An audit that silently scored on
+ * fewer policies would be worse than either.
+ */
+async function registerFromVendoredPack(enabled: string[]): Promise<boolean> {
+  const dir = bundledPackDir();
+  if (!dir) return false;
+  const entry = resolve(dir, PACK_ENTRY_ASSET);
+  if (!existsSync(entry)) return false;
+
+  const wanted = new Set(enabled);
+  let hooks;
+  try {
+    clearCustomHooks();
+    hooks = await loadCustomHooks(entry, { strict: true });
+  } catch {
+    return false;
+  } finally {
+    clearCustomHooks();
+  }
+  if (hooks.length === 0) return false;
+  const fromPack = new Map(hooks.map((hook) => [hook.name, hook]));
+
+  // Registered in CATALOG ORDER, taking the pack's function where it has one and
+  // the compiled one for the guard the pack may not carry.
+  //
+  // Order is not cosmetic here. `getPoliciesForEvent` sorts by priority only and
+  // V8's sort is stable, so registration order decides EVALUATION order, and
+  // `evaluatePolicies` stops at the first deny — which means the order decides
+  // which policy is CREDITED for a hit. Appending the always-on guard after the
+  // pack instead of at its catalog position moved it from index 11 to last, and
+  // re-attributed 3 events in a 23,477-event corpus: 456 -> 453 for the guard,
+  // with the denies it used to shadow surfacing as block-force-push and
+  // block-gh-pipeline. The audit's per-policy counts are what a user reads, so
+  // that is a changed audit, not a changed internal.
+  let registered = 0;
+  for (const policy of BUILTIN_POLICIES) {
+    if (!wanted.has(policy.name)) continue;
+    const packHook = fromPack.get(policy.name);
+    if (policy.alwaysOn || !packHook) {
+      registerPolicy(policy.name, policy.description, policy.fn, policy.match, 0, policy.params);
+      continue;
+    }
+    registerPolicy(
+      packHook.name,
+      packHook.description ?? policy.description,
+      packHook.fn,
+      packHook.match ?? policy.match,
+      0,
+      policy.params,
+    );
+    registered += 1;
+  }
+  // Nothing came from the pack: it loaded but carried none of the names the
+  // audit replays. Falling back is more honest than scoring on the compiled set
+  // while claiming the pack lane ran.
+  return registered > 0;
+}
+
 /** Restore the registry to whatever was there before `initReplay()`. Safe to
  *  call when not initialized (no-op). Always paired with `initReplay()` in a
  *  try/finally inside `runAudit()`. */
@@ -87,7 +173,7 @@ export interface ReplayHit {
  *  is reported too, so sanitize policies that emit informational notes still
  *  surface in the audit. */
 export async function replayEvent(event: NormalizedToolEvent): Promise<ReplayHit[]> {
-  if (!initialized) initReplay();
+  if (!initialized) await initReplay();
 
   const session: SessionMetadata = {
     sessionId: event.sessionId,

package/src/audit/report.ts

@@ -263,16 +263,16 @@ export function formatMarkdown(result: AuditResult): string {
   out.push("## TL;DR");
   out.push("");
   if (totalHits === 0) {
-    out.push("Clean run — the AI coding agent didn't do anything `failproofai` catches in this window.");
+    out.push("Clean run — the agent didn't do anything `failproofai` catches in this window.");
   } else {
     out.push(
-      `Over ${result.transcripts.scanned} sessions, my AI coding agent did **${totalHits} things \`failproofai\` would have stopped**: ` +
-      `${totalProtected} were already blocked in real time by my current config; ` +
-      `**${totalSlipping} slipped through** (would've been caught if more policies were on).`,
+      `Over ${result.transcripts.scanned} sessions, my agent did **${totalHits} things \`failproofai\` caught**: ` +
+      `${totalProtected} were blocked in real time by my current config; ` +
+      `**${totalSlipping} slipped through** and were only caught by the audit.`,
     );
   }
   out.push("");
-  out.push("> [failproofai](https://github.com/FailproofAI/failproofai) is a hook-based policy engine for Claude Code, Codex, Copilot, Cursor, OpenCode, and Pi. The `audit` command replays past agent sessions through every builtin policy to surface patterns that were (or could've been) stopped.");
+  out.push("> [failproofai](https://github.com/FailproofAI/failproofai) is a hook-based policy engine for 12 agent harnesses — coding CLIs, chat gateways, and self-hosted assistants. The `audit` command replays past agent sessions through every builtin policy to surface patterns that were (or could've been) stopped.");
   out.push("");
 
   if (totalHits === 0) return out.join("\n");

package/src/audit/schedule-cli.ts

@@ -37,8 +37,13 @@ import {
   ANSI_RESET,
   brandAnsi,
   colorsEnabled,
+  optsFor,
   outro,
+  printBlock,
+  rows as kitRows,
+  stack,
   step,
+  title,
 } from "../hooks/tui";
 
 /** Mirrors `fp-config`'s own bounds so the error can name them before writing. */
@@ -62,30 +67,6 @@ const green = (s: string) => wrap(brandAnsi("guide"), s);
 const dim = (s: string) => wrap(ANSI_DIM, s);
 const bold = (s: string) => wrap(ANSI_BOLD, s);
 
-/**
- * The readout's left margin.
- *
- * Deliberately NOT the `│` spine the sign-in uses. A spine means "a flow is
- * happening, with a beginning and an end"; `--status` is a snapshot of a
- * machine, and hanging one off a frame that never opened reads as an unfinished
- * wizard. Alignment does the work here instead.
- */
-const rail = () => " ";
-
-/**
- * One labelled row of the `--status` readout.
- *
- * A fixed label column so the values line up into a second column that can be
- * read straight down — the whole point of this command is answering "what is
- * this machine doing" at a glance, and a ragged left edge makes four facts read
- * as four sentences.
- */
-const LABEL_WIDTH = 15;
-function row(label: string, value: string, note?: string): string {
-  const gap = " ".repeat(Math.max(1, LABEL_WIDTH - label.length));
-  return `${rail()}  ${dim(label)}${gap}${value}${note ? `  ${dim(note)}` : ""}`;
-}
-
 /**
  * Turn scheduled audits on, signing in first if needed.
  *
@@ -219,62 +200,77 @@ export function runScheduleStatus(): void {
   const daemon = daemonServiceStatus();
 
   const on = config.audit.auto;
-  const out: string[] = [""];
+  const opts = optsFor(process.stdout);
+  const detail: Array<[string, string]> = [];
 
-  // The state first and alone, in the accent that matches it — everything below
-  // is detail about a machine that is either doing this or not, and reading the
-  // detail first is reading the answer to a question nobody asked yet.
-  out.push(
-    `${rail()}  ${bold("scheduled audit")}   ${on ? green("on") : dim("off")}` +
-      (on ? dim(`  every ${config.audit.intervalDays} days`) : ""),
-  );
-  out.push(rail());
+  // The state first, because everything below is detail about a machine that is
+  // either doing this or not, and reading the detail first answers a question
+  // nobody has asked yet. It is a ROW like the rest, though: the headline used
+  // to be hand-padded to a different column than `row()` used, which is why this
+  // readout printed its first value at column 21 and every other at 18.
+  detail.push([
+    "scheduled audit",
+    on
+      ? `${green("on")}${dim(`  every ${config.audit.intervalDays} days`)}`
+      : dim("off"),
+  ]);
 
   // A session whose refresh window has closed cannot mint another access token,
   // so it is a destination in name only. Showing the address for one would tell
   // somebody their digests are going somewhere they are not.
   const live = auth && auth.refresh_expires_at * 1000 > Date.now() ? auth : null;
-  out.push(row("reports to", live ? live.user.email : dim("— signed out")));
+  detail.push(["reports to", live ? live.user.email : dim("— signed out")]);
   if (on && !live) {
     // The state the reporter surfaces as "signed-out". Named here for the same
     // reason the settings panel names it: the scans keep running, so silence
     // about the digests would look like the feature failing.
-    out.push(row("", pink("scans continue; digests are paused until you sign in")));
+    detail.push(["", pink("scans continue; digests are paused until you sign in")]);
   } else if (on && config.audit.reportsConsentedAt === undefined) {
     // Signed in, scheduled, and still not sending: this machine set `audit.auto`
     // when it only meant "scan locally", so nothing has consented to the digest
     // leaving the box. Without this row the status screen would show a healthy
     // schedule and a live address and still mail nothing, with no explanation
     // anywhere the user can see.
-    out.push(
-      row("", pink("scans continue; digests need a fresh opt-in — run `--schedule` to turn them on")),
-    );
+    detail.push([
+      "",
+      pink("scans continue; digests need a fresh opt-in — run `--schedule` to turn them on"),
+    ]);
   }
 
-  out.push(row("daemon", describeDaemon(daemon)));
+  detail.push(["daemon", describeDaemon(daemon)]);
 
   if (sched?.nextDueAtMs != null && on) {
-    out.push(row("next scan", untilPhrase(sched.nextDueAtMs)));
+    detail.push(["next scan", untilPhrase(sched.nextDueAtMs)]);
   }
   if (sched?.lastRunAtMs != null) {
     const exit = sched.lastExitCode;
-    out.push(
-      row(
-        "last scheduled",
-        agoPhrase(sched.lastRunAtMs),
-        exit != null && exit !== 0 && exit !== 75 ? pink(`exit ${exit}`) : undefined,
-      ),
-    );
+    const failed = exit != null && exit !== 0 && exit !== 75;
+    detail.push([
+      "last scheduled",
+      agoPhrase(sched.lastRunAtMs) + (failed ? `  ${pink(`exit ${exit}`)}` : ""),
+    ]);
   }
-  out.push(
-    row("last result", meta?.cachedAt ? agoPhrase(Date.parse(meta.cachedAt)) : dim("none yet")),
-  );
+  detail.push([
+    "last result",
+    meta?.cachedAt ? agoPhrase(Date.parse(meta.cachedAt)) : dim("none yet"),
+  ]);
   if (machine?.last_reported_at) {
-    out.push(row("last reported", agoPhrase(Date.parse(machine.last_reported_at))));
+    detail.push(["last reported", agoPhrase(Date.parse(machine.last_reported_at))]);
   }
-  out.push("");
 
-  process.stdout.write(out.join("\n"));
+  printBlock(
+    process.stdout,
+    stack(
+      // The meta carries a fact, not the word "status" — a heading that repeats
+      // the command name tells the reader nothing they did not just type.
+      title(
+        "failproofai audit",
+        on ? `scheduled · every ${config.audit.intervalDays} days` : "not scheduled",
+        opts,
+      ),
+      kitRows(detail, opts),
+    ),
+  );
 }
 
 function describeDaemon(status: ReturnType<typeof daemonServiceStatus>): string {