failproofai 1.0.3 → 1.0.4-beta.2

package/src/audit/index.ts

@@ -16,6 +16,11 @@ import { INTEGRATION_TYPES, type IntegrationType } from "../hooks/types";
 import { ADAPTERS } from "./cli-adapters";
 import { AUDIT_DETECTORS } from "./detectors";
 import { severityForBuiltin } from "./features";
+import { findSecrets, flattenToolInput } from "./leak-scan";
+import { fingerprintSecret, fingerprintId } from "./leak-fingerprint";
+import { readLeakIdentity, readLeakRecord, writeLeakRecord } from "./leak-store";
+import { upsertFinding, describeMechanism } from "./leak-record";
+import { shortenPaths } from "./redact-example";
 import { readCachedTranscript, writeCachedTranscriptResult } from "./cache";
 import { initReplay, replayEvent, restoreReplay } from "./replay";
 import {
@@ -183,19 +188,23 @@ async function scanOneTranscript(
   result.cwd = result.cwd || events[0].cwd || "";
 
   for (const event of events) {
-    // Run audit detectors first (stateful, must see every event).
-    for (const detector of AUDIT_DETECTORS) {
-      const hit = detector.detect(event, sessionState);
-      if (!hit) continue;
-      recordHit(
-        result,
-        detector.name,
-        event.timestamp,
-        event.cwd,
-        truncateExample(hit.example),
-      );
-    }
-    // Then replay through every builtin policy.
+    // The 8 behavioural detectors are switched off with the rest of the old
+    // audit — see the header of `src/audit/scoring.ts`. The modules and their
+    // unit tests are untouched; nothing calls them. `sessionState` is still
+    // threaded through so a resumed scan keeps its shape and restoring this
+    // loop needs no plumbing.
+    // for (const detector of AUDIT_DETECTORS) {
+    //   const hit = detector.detect(event, sessionState);
+    //   if (!hit) continue;
+    //   recordHit(
+    //     result,
+    //     detector.name,
+    //     event.timestamp,
+    //     event.cwd,
+    //     truncateExample(hit.example),
+    //   );
+    // }
+    // Replay through every builtin policy.
     let replayHits;
     try {
       replayHits = await replayEvent(event);
@@ -212,11 +221,117 @@ async function scanOneTranscript(
         truncateExample(example),
       );
     }
+
+    // Credential VALUES, which the replay above cannot report: it returns a
+    // decision per policy, never the text that matched. Scanned separately so
+    // the leak report can name WHICH key, and so detection can be tuned for
+    // precision without dragging the redactor's recall down with it.
+    recordLeaks(result, event);
   }
 
   return { result, bytesScanned, detectorState: sessionState, resumed };
 }
 
+/**
+ * Fingerprint every credential in one event and attach the sightings.
+ *
+ * The raw value exists only inside this function. It is read out of the event,
+ * hashed into a salted id, rendered into a mask, and dropped — nothing that
+ * leaves here can reconstruct it.
+ *
+ * Input and result are scanned separately and tagged, because they are
+ * different exposures: an input is what the agent SENT (deniable at PreToolUse
+ * on 12/12 CLIs, and 39.5% of measured appearances) while a result is what it
+ * RECEIVED (blockable on 2/12, and only after the fact).
+ */
+function recordLeaks(result: TranscriptAuditResult, event: NormalizedToolEvent): void {
+  const salt = leakSalt();
+  if (!salt) return;
+
+  const filePath = (event.toolInput as { file_path?: unknown }).file_path;
+  const shortPath = typeof filePath === "string" ? shortenPaths(filePath) : null;
+
+  const scan = (text: string, direction: "input" | "result"): void => {
+    for (const match of findSecrets(text)) {
+      (result.leaks ??= []).push({
+        id: fingerprintId(match.value, salt),
+        fingerprint: fingerprintSecret(match.value, match.rule),
+        name: match.name,
+        rule: match.rule,
+        shaped: match.shaped,
+        timestamp: event.timestamp,
+        cwd: event.cwd,
+        toolName: event.toolName,
+        direction,
+        path: shortPath,
+      });
+    }
+  };
+
+  scan(flattenToolInput(event.toolInput), "input");
+  if (event.toolResultText !== undefined) scan(event.toolResultText, "result");
+}
+
+/**
+ * The machine's HMAC salt, read once per process.
+ *
+ * Cached because `scanOneTranscript` runs this per EVENT — 340,918 of them on
+ * the measured corpus — and re-reading a file that many times would dominate
+ * the scan. `null` when the store is unreadable, which disables leak recording
+ * rather than failing the audit: a scan that cannot fingerprint is still a scan
+ * that can count.
+ */
+let cachedSalt: string | null | undefined;
+function leakSalt(): string | null {
+  if (cachedSalt === undefined) {
+    try {
+      cachedSalt = readLeakIdentity().salt;
+    } catch {
+      cachedSalt = null;
+    }
+  }
+  return cachedSalt;
+}
+
+/**
+ * Merge this scan's sightings into `~/.failproofai/audit/leaks.json`.
+ *
+ * Returns the ids seen for the FIRST time, which is the set worth telling a
+ * human about. Never throws: a scan that finds leaks it cannot write down has
+ * still found them, and the caller reports on the in-memory result either way.
+ */
+function persistLeaks(perTranscript: TranscriptAuditResult[]): string[] {
+  const fresh: string[] = [];
+  try {
+    const record = readLeakRecord();
+    for (const t of perTranscript) {
+      for (const leak of t.leaks ?? []) {
+        const { isNew } = upsertFinding(record, {
+          id: leak.id,
+          fingerprint: leak.fingerprint,
+          name: leak.name,
+          rule: leak.rule,
+          // A vendor shape is what licenses an alert to name a console; a
+          // name-only match knows something leaked, not who issued it.
+          confidence: leak.shaped ? "doc-verified" : "name-only",
+          sighting: {
+            cli: t.cli,
+            sessionId: t.sessionId,
+            cwd: shortenPaths(leak.cwd),
+            at: leak.timestamp,
+            mechanism: describeMechanism(leak.toolName, leak.direction, leak.path),
+          },
+        });
+        if (isNew) fresh.push(leak.id);
+      }
+    }
+    writeLeakRecord(record);
+  } catch {
+    // Bookkeeping failure must not fail the audit — see leak-store.ts.
+  }
+  return fresh;
+}
+
 function formatPolicyExample(_policyName: string, event: NormalizedToolEvent): string {
   if (event.toolName === "Bash") {
     const command = (event.toolInput as { command?: unknown }).command;
@@ -527,8 +642,21 @@ async function runAuditInner(opts: RunAuditOptions, startedAt: number): Promise<
     eventsScanned += t.eventsScanned ?? 0;
   }
 
+  // Fold every sighting into the persistent record. This is the ONLY place the
+  // leak record is written: `scanOneTranscript` runs 8-way concurrent and a
+  // per-transcript write would be eight processes racing one file — the exact
+  // shape that lost updates when it was measured (2 concurrent writers, 5/5
+  // trials, the loser's entry gone permanently).
+  //
+  // `newFindingIds` is what a notice keys on: an id already in the record must
+  // not alert again however many fresh sightings it accumulates, and a
+  // credential seen for the first time must alert even though its rule has
+  // fired a thousand times before.
+  const newFindingIds = persistLeaks(perTranscript);
+
   const auditResult: AuditResult = {
     version: 2,
+    newLeakIds: newFindingIds,
     scannedAt: new Date(startedAt).toISOString(),
     scope: {
       cli: clis,

package/src/audit/leak-fingerprint.ts

@@ -0,0 +1,200 @@
+/**
+ * How a leaked credential is NAMED once it leaves the machine.
+ *
+ * Every other redactor in this codebase exists to destroy a value. This one
+ * exists to describe one, because the leak report has the opposite job: a
+ * finding the reader cannot tie to a row in a vendor console is a finding they
+ * cannot act on. `[REDACTED: GitHub token]` tells you nothing you can revoke.
+ *
+ * The rule is: **enough to identify, never enough to use.**
+ *
+ * ## Why the mask character matters, which is not obvious
+ *
+ * The three most natural masking characters — `X`, `x` and `0` — RE-MATCH every
+ * vendor prefix pattern we ship. `AKIA` + sixteen `X`s satisfies
+ * `AKIA[A-Z0-9]{16}`, so a masked AWS key is reported by failproofai as a live
+ * AWS key on the next scan. The same holds for a masked connection string,
+ * whose pattern keys on `://…@` and does not care what sits between. So the
+ * mask is `•` (U+2022), which is in no credential charset anywhere, and the
+ * unit tests assert that every masked form fails to re-match.
+ *
+ * That is not a hypothetical: this product measured its own audit output
+ * feeding back into its own corpus, one leaked credential becoming seven
+ * reported findings across four sessions.
+ *
+ * ## Why the last four, and why not always
+ *
+ * Last-4 is what vendor consoles display next to a key, so it is the token that
+ * lets a human find the right row and press revoke. But it is only safe when
+ * what remains is genuinely unguessable. For a 40-character GitHub token,
+ * showing `ghp_` and four trailing characters leaves ~32 unknown base62
+ * characters — no help to anyone. For a 13-character human-chosen password,
+ * last-4 is nearly a third of the secret, and the corpus's one confirmed-live
+ * password was exactly 13 characters at 3.19 bits per character.
+ *
+ * So the tail is shown only when the value clears BOTH bars: a recognised
+ * public vendor prefix (which proves it is a minted token, not a typed
+ * password) AND enough residual length. Everything else is described by class
+ * and length alone.
+ */
+
+/** The mask glyph. Never `X`, `x` or `0` — see the module header. */
+const MASK_CHAR = "•";
+
+/** Longest run of mask glyphs rendered, so a 200-char token stays readable. */
+const MAX_MASK_RUN = 8;
+
+/**
+ * Minimum total length before any trailing characters are shown.
+ *
+ * 24 is chosen so that even the shortest minted token we recognise keeps well
+ * over 100 bits of residual entropy after the prefix and tail are removed.
+ * Human-chosen passwords essentially never reach it, which is the point.
+ */
+const MIN_LEN_FOR_TAIL = 24;
+
+/** How many trailing characters a vendor console typically shows. */
+const TAIL_CHARS = 4;
+
+/**
+ * Public, minted prefixes — the part of a credential that carries no secret.
+ *
+ * Each of these is published by its issuer and identical across every key they
+ * mint, so showing it discloses nothing while telling the reader which console
+ * to open. Ordered longest-first, so `sk-ant-api03-` wins over `sk-ant-` and
+ * `sk-`; the first match is the one rendered.
+ *
+ * Deliberately NOT the full detection vocabulary. This list only has to be
+ * right about what is safe to PRINT — a prefix missing from it costs a less
+ * specific label, never a disclosure.
+ */
+const PUBLIC_PREFIXES: ReadonlyArray<readonly [string, string]> = [
+  ["sk-ant-api03-", "Anthropic API key"],
+  ["sk-ant-oat01-", "Anthropic OAuth token"],
+  ["sk-ant-", "Anthropic key"],
+  ["sk-proj-", "OpenAI project key"],
+  ["sk-svcacct-", "OpenAI service-account key"],
+  ["sk-admin-", "OpenAI admin key"],
+  ["sk-or-v1-", "OpenRouter key"],
+  ["github_pat_", "GitHub fine-grained token"],
+  ["ghp_", "GitHub personal access token"],
+  ["gho_", "GitHub OAuth token"],
+  ["ghu_", "GitHub user-to-server token"],
+  ["ghs_", "GitHub server-to-server token"],
+  ["ghr_", "GitHub refresh token"],
+  ["glpat-", "GitLab personal access token"],
+  ["xoxb-", "Slack bot token"],
+  ["xoxp-", "Slack user token"],
+  ["xapp-", "Slack app-level token"],
+  ["sk_live_", "Stripe live secret key"],
+  ["sk_test_", "Stripe test secret key"],
+  ["rk_live_", "Stripe restricted key"],
+  ["AKIA", "AWS access key ID"],
+  ["ASIA", "AWS temporary access key ID"],
+  ["AIza", "Google API key"],
+  ["hf_", "Hugging Face token"],
+  ["gsk_", "Groq API key"],
+  ["r8_", "Replicate token"],
+  ["xai-", "xAI key"],
+  ["npm_", "npm access token"],
+  ["dop_v1_", "DigitalOcean token"],
+  ["shpat_", "Shopify access token"],
+  ["sbp_", "Supabase token"],
+];
+
+export interface LeakFingerprint {
+  /** What the reader sees: `ghp_••••••••4f2a` or `[13-char password]`. */
+  display: string;
+  /** Human label for the class, e.g. "GitHub personal access token". */
+  label: string;
+  /** Total character length of the original value. */
+  length: number;
+  /** True when a minted public prefix was recognised — i.e. we can name a
+   *  console to revoke at. False means "we know it leaked, not who issued it". */
+  attributed: boolean;
+}
+
+/** The recognised public prefix for a value, longest match first. */
+function matchPublicPrefix(value: string): readonly [string, string] | null {
+  for (const entry of PUBLIC_PREFIXES) {
+    if (value.startsWith(entry[0])) return entry;
+  }
+  return null;
+}
+
+/**
+ * Render a credential as something a human can recognise and nobody can use.
+ *
+ * `fallbackLabel` names the class when no public prefix is recognised — pass
+ * the detecting rule's own label (e.g. "assigned secret", "database password")
+ * so an unattributed finding still says what KIND of thing leaked.
+ */
+export function fingerprintSecret(value: string, fallbackLabel = "secret"): LeakFingerprint {
+  const length = value.length;
+  const matched = matchPublicPrefix(value);
+
+  if (!matched) {
+    // No minted prefix: this is a password, a first-party key, or a format we
+    // do not recognise. Nothing but the class and the length is safe to print,
+    // and the length is what lets the owner recognise their own value.
+    return {
+      display: `[${length}-char ${fallbackLabel}]`,
+      label: fallbackLabel,
+      length,
+      attributed: false,
+    };
+  }
+
+  const [prefix, label] = matched;
+  const bodyLength = length - prefix.length;
+  const run = MASK_CHAR.repeat(Math.max(1, Math.min(MAX_MASK_RUN, bodyLength)));
+
+  // The tail is the actionable half — but only when what stays hidden is
+  // genuinely unguessable. See the module header.
+  if (length >= MIN_LEN_FOR_TAIL && bodyLength > TAIL_CHARS * 2) {
+    return {
+      display: `${prefix}${run}${value.slice(-TAIL_CHARS)}`,
+      label,
+      length,
+      attributed: true,
+    };
+  }
+
+  return { display: `${prefix}${run}`, label, length, attributed: true };
+}
+
+/**
+ * A stable identity for one distinct credential, for deduping across scans.
+ *
+ * Salted with a per-machine secret so the digest cannot be used as a lookup
+ * oracle: without the salt, an attacker holding a guessable candidate (and the
+ * corpus is full of dictionary passwords) could confirm it by comparing
+ * hashes. With it, the id is meaningful only on the machine that produced it.
+ *
+ * The salt therefore forecloses cross-machine dedupe until an org-scoped key
+ * exists — a deliberate trade recorded here so nobody reads a per-machine salt
+ * as an oversight.
+ */
+/**
+ * Every id `fingerprintId` can produce, and nothing else.
+ *
+ * Ids become FILENAMES — one marker file per notified finding, one queue file
+ * per pending macOS banner — so an id is a path component, and a path component
+ * built from unvalidated input is a directory traversal. Measured: passing
+ * `"../../../../tmp/PWNED"` to `markLeakNoticeDelivered` created that file.
+ *
+ * Ids are HMAC hex today, so nothing in a normal run can carry a `/`. That is
+ * not the same as safe: the record is JSON read off disk, and `leaks.json` is a
+ * file a user (or anything running as them) can edit. The guard is one regex
+ * and it makes the property structural instead of incidental.
+ */
+export function isFindingId(id: string): boolean {
+  return /^[0-9a-f]{16}$/.test(id);
+}
+
+export function fingerprintId(value: string, machineSalt: string): string {
+  // Lazily required so this module stays importable in the browser bundle,
+  // where the dashboard renders findings but never computes ids.
+  const { createHmac } = require("node:crypto") as typeof import("node:crypto");
+  return createHmac("sha256", machineSalt).update(value).digest("hex").slice(0, 16);
+}

package/src/audit/leak-notice.ts

@@ -0,0 +1,161 @@
+/**
+ * Deciding whether a finding still owes the user a notice — exactly once.
+ *
+ * ## Why this is not a field on the record
+ *
+ * The obvious design is `notifiedAt` on each finding, written back after the
+ * notice goes out. Measured against this repo's real `writeJsonAtomically`
+ * semantics, that fails 100% of the time under the conditions it actually runs
+ * in: two concurrent sessions with the same finding produced two notices, four
+ * produced four, eight produced eight. Worse, two concurrent sessions with
+ * DIFFERENT findings produced two notices and a surviving watermark containing
+ * only one of them — the loser's mark permanently gone, so it re-notified
+ * forever afterwards. `lib/atomic-write.ts` says so in its own header: atomic
+ * per FILE, last-writer-wins per CONTENT, i.e. a lost update on a map.
+ *
+ * Running several agents at once in one project is not an edge case here; it is
+ * how the tool is used.
+ *
+ * So the claim is one marker FILE per finding, created with `wx` — O_EXCL,
+ * which the kernel makes atomic across processes. Eight concurrent claimants
+ * produce exactly one winner, verified 3/3 trials. Concurrent claims on
+ * different findings all succeed and all survive, because they touch different
+ * files.
+ *
+ * ## Claim ordering, and which way to fail
+ *
+ * The marker is written BEFORE the notice reaches the stream, and the caller
+ * only keeps it if the notice was actually shaped. The alternative orderings
+ * were both measured: emit-then-mark nags forever if the process dies between
+ * them (5 sessions → 5 notices), and mark-then-never-emit swallows the alert
+ * permanently with nothing on disk saying so (total delivered: 0).
+ *
+ * Between "told twice" and "never told", a security tool picks told twice.
+ */
+import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync, rmSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { auditDir } from "../hooks/fp-home";
+import { readLeakRecord, activeFindings } from "./leak-store";
+import { isFindingId } from "./leak-fingerprint";
+
+/**
+ * Which way the user was told. Each channel claims independently.
+ *
+ * They are NOT one guarantee. The desktop banner and the in-CLI notice reach a
+ * person at different moments and fail in unrelated ways — and the banner has a
+ * failure it cannot detect at all: on a locked screen the shell accepts the
+ * call and returns an id while nobody sees anything. Letting a successful
+ * `Notify` claim the finding would then silence the in-session notice too, and
+ * the only channel that actually reached a human is the one we suppressed.
+ *
+ * So each channel gets its own marker directory and its own once-per-finding
+ * guarantee. A user at a desktop sees the banner AND the next session's notice;
+ * that is a bounded, deliberate duplicate, and it is the right side of the
+ * trade this file already made once (told twice beats never told).
+ */
+export type NoticeChannel = "cli" | "desktop";
+
+/** One empty file per claimed finding, per channel. Cheap: a few hundred
+ *  zero-byte files at the very worst, and the record itself is capped at 500
+ *  findings. "cli" keeps the original directory name so an existing machine's
+ *  claims survive the upgrade — renaming it would re-notify for every finding
+ *  already delivered. */
+const NOTICE_DIR = (home?: string, channel: NoticeChannel = "cli") =>
+  resolve(auditDir(home), channel === "cli" ? "notified" : `notified-${channel}`);
+
+/** Markers older than this are swept, so the directory cannot grow forever on a
+ *  machine that has been running for years. Comfortably longer than the
+ *  record's own 90-day TTL, so a marker never outlives its finding. */
+const MARKER_TTL_MS = 120 * 86_400_000;
+
+export interface PendingNotice {
+  /** Findings that have never been notified about and are not dismissed. */
+  ids: string[];
+  count: number;
+}
+
+/**
+ * Findings that still owe the user a notice.
+ *
+ * Reads the record and subtracts what has already been claimed. Never throws:
+ * on any failure it returns "nothing pending", because a hook that cannot read
+ * its bookkeeping must stay silent rather than guess.
+ */
+export function pendingLeakNotice(home?: string, channel: NoticeChannel = "cli"): PendingNotice {
+  try {
+    const dir = NOTICE_DIR(home, channel);
+    const claimed = new Set(existsSync(dir) ? readdirSync(dir) : []);
+    const ids = activeFindings(readLeakRecord(home), home)
+      .map((f) => f.id)
+      .filter((id) => !claimed.has(id));
+    return { ids, count: ids.length };
+  } catch {
+    return { ids: [], count: 0 };
+  }
+}
+
+/**
+ * Claim these findings, atomically and per-finding.
+ *
+ * Returns the ids this process actually won. A caller that loses a race simply
+ * gets a shorter list — it never blocks, and it never overwrites a claim made
+ * by a concurrent session.
+ */
+export function markLeakNoticeDelivered(
+  ids: string[],
+  home?: string,
+  channel: NoticeChannel = "cli",
+): string[] {
+  const won: string[] = [];
+  try {
+    const dir = NOTICE_DIR(home, channel);
+    mkdirSync(dir, { recursive: true });
+    for (const id of ids) {
+      // An id is about to become a filename. Anything that is not the exact
+      // shape `fingerprintId` mints is refused rather than resolved — see
+      // `isFindingId`. Refusing also means NOT reporting it as won, so a caller
+      // treats it as unclaimed and never records it as delivered.
+      if (!isFindingId(id)) continue;
+      try {
+        // `wx` is O_EXCL: it throws EEXIST if another process got here first.
+        // That throw IS the lock — there is no read-then-write window for two
+        // processes to both pass through.
+        writeFileSync(resolve(dir, id), "", { flag: "wx", mode: 0o600 });
+        won.push(id);
+      } catch {
+        // Already claimed by a concurrent session. Correct outcome: the user
+        // is told once, by whichever session got there first.
+      }
+    }
+  } catch {
+    // Cannot claim at all — the caller has already decided to emit, and being
+    // told twice beats never being told.
+  }
+  return won;
+}
+
+/** Drop markers for findings that no longer exist, and very old ones. */
+export function pruneNoticeMarkers(home?: string, nowMs = Date.now()): void {
+  try {
+    const live = new Set(readLeakRecord(home).findings.map((f) => f.id));
+    for (const channel of ["cli", "desktop"] as const) {
+      const dir = NOTICE_DIR(home, channel);
+      if (!existsSync(dir)) continue;
+      for (const name of readdirSync(dir)) {
+        const path = resolve(dir, name);
+        let stale = !live.has(name);
+        if (!stale) {
+          try {
+            stale = nowMs - statSync(path).mtimeMs > MARKER_TTL_MS;
+          } catch {
+            stale = false;
+          }
+        }
+        if (stale) rmSync(path, { force: true });
+      }
+    }
+  } catch {
+    // Housekeeping only.
+  }
+}