failproofai 1.0.3 → 1.0.4-beta.2

package/src/audit/leak-record.ts

@@ -0,0 +1,232 @@
+/**
+ * The per-credential record — what the audit knows that `AuditCount` cannot say.
+ *
+ * `AuditCount` aggregates BY RULE: it can report that `sanitize-api-keys` fired
+ * 47 times and show three 80-character examples. It cannot enumerate 47 distinct
+ * keys, say which of them is the same key seen twice, or answer "which CLI, in
+ * which project, at what time, and how did it get there". Every surface the
+ * narrowed product needs — the emailed digest, the dashboard's leak view, and
+ * the once-per-finding notification guard — asks exactly those questions, so
+ * none of them can be built on the rule-level record.
+ *
+ * ## Grouped by VALUE, not by occurrence — this is the whole bloat story
+ *
+ * The obvious shape is one row per sighting. Measured against the real corpus
+ * that is 10,301 rows, against roughly 266 distinct values, against about 25
+ * genuinely real credentials. Two orders of magnitude of duplication, because
+ * one pasted key fans out across every session that touched it — the single
+ * confirmed-live password in the corpus appeared in 11 files and 22 occurrences,
+ * partly because Claude's own checkpoint records replay the prompt that carried
+ * it.
+ *
+ * So the unit is the DISTINCT VALUE, with a bounded list of sightings attached
+ * as evidence. A finding seen a thousand times costs the same on disk as one
+ * seen twice: `MAX_SIGHTINGS` entries, plus a count. At the corpus's own numbers
+ * that is roughly 130 KB for the whole file, which is why this is one file and
+ * not a directory — a per-finding file would spend more on inodes and directory
+ * entries than on the findings.
+ *
+ * ## What is deliberately NOT stored
+ *
+ * The credential itself, ever. The record holds a salted id (for dedupe across
+ * scans) and a fingerprint (for a human to recognise the key in a console) and
+ * nothing that could reconstruct the value. `~/.failproofai/` is a directory the
+ * user did not ask for and cannot audit, so a leak scanner that caches the leaks
+ * has made the problem worse rather than better.
+ */
+import type { IntegrationType } from "../hooks/types";
+import type { LeakFingerprint } from "./leak-fingerprint";
+
+/** Sightings kept per distinct value. Enough to answer 5W1H for the first and
+ *  most recent exposure without storing the long tail nobody reads. */
+export const MAX_SIGHTINGS = 5;
+
+/** Findings not seen in this many days are dropped on the next write. A key
+ *  whose transcripts have aged out of the scan window is not evidence of
+ *  anything current, and keeping it forever grows the file without bound. */
+export const FINDING_TTL_DAYS = 90;
+
+/** Cap on distinct findings retained, newest-first. A machine with more than
+ *  this is not going to triage them, and the census measured a name-only rule
+ *  emitting thousands — the cap is what stops a bad matcher filling the disk. */
+export const MAX_FINDINGS = 500;
+
+/**
+ * How a credential came to be where it was found — the "how" of 5W1H.
+ *
+ * Derived from the tool event rather than authored, so it is always specific:
+ * "read from ~/…/.env" beats "detected in a file operation". The path is
+ * home-shortened by the caller before it reaches here.
+ */
+export interface LeakMechanism {
+  /** Short past-tense phrase, e.g. "read from ~/…/.env". */
+  summary: string;
+  /** Canonical tool name (Bash, Read, Write, Edit…). */
+  toolName: string;
+  /** Where the value appeared: the tool's INPUT (the agent sent it) or its
+   *  RESULT (the agent received it). Different exposures with different
+   *  remedies — an input is deniable at PreToolUse, a result is not. */
+  direction: "input" | "result";
+}
+
+/** One exposure of one credential: the where/when/who of 5W1H. */
+export interface LeakSighting {
+  /** WHO — the harness whose transcript carried it. */
+  cli: IntegrationType;
+  /** WHERE — session and project. `cwd` is already home-shortened. */
+  sessionId: string;
+  cwd: string;
+  /** WHEN — ISO-8601, from the transcript line. */
+  at: string;
+  /** HOW. */
+  mechanism: LeakMechanism;
+}
+
+/** One distinct credential, however many times it was seen. */
+export interface LeakFinding {
+  /** Salted, machine-local, stable across scans. Never derived from the
+   *  fingerprint, which is not unique. */
+  id: string;
+  /** WHAT — the masked rendering, its class, and its length. */
+  fingerprint: LeakFingerprint;
+  /** The identifier it was assigned to, when it came from an assignment
+   *  (`COMPOSIO_API_KEY`). For the 230-of-237 first-party class this is the
+   *  only actionable field there is, since no vendor console exists. */
+  name: string | null;
+  /** Which rule found it, for explaining and for suppression. */
+  rule: string;
+  /** How much we trust the attribution — mirrors the pattern census's tiers. */
+  confidence: "doc-verified" | "source-verified" | "partner-corroborated" | "observed-only" | "name-only";
+  firstSeen: string;
+  lastSeen: string;
+  /** Total exposures, which is NOT `sightings.length` — see MAX_SIGHTINGS. */
+  occurrences: number;
+  /** Bounded evidence, oldest first. */
+  sightings: LeakSighting[];
+  /** Set when the user dismisses it. Dismissed findings are retained (so they
+   *  do not re-alert) but never notified about again. */
+  dismissedAt?: string;
+  /** Set when a notice for this finding has been delivered, so the next scan
+   *  stays quiet. The delivery guard itself is a separate O_EXCL marker; this
+   *  is the human-readable record of it. */
+  notifiedAt?: string;
+}
+
+export interface LeakRecord {
+  schemaVersion: number;
+  /** Per-machine HMAC salt for `fingerprintId`. Generated once, never sent. */
+  salt: string;
+  updatedAt: string;
+  findings: LeakFinding[];
+}
+
+export const LEAK_RECORD_SCHEMA_VERSION = 1;
+
+/**
+ * Fold one new sighting into the record.
+ *
+ * Returns whether this is a **newly seen** credential, which is what the
+ * notification guard keys on: a finding whose id is already present must not
+ * alert again, however many new sightings it accumulates. Getting this backwards
+ * is the failure the census measured — keying on the rule-level examples, which
+ * `mergeIncremental` freezes at the first three, silently misses the arrival of
+ * a brand-new key while re-alerting for old ones.
+ */
+export function upsertFinding(
+  record: LeakRecord,
+  incoming: Omit<LeakFinding, "firstSeen" | "lastSeen" | "occurrences" | "sightings"> & {
+    sighting: LeakSighting;
+  },
+): { record: LeakRecord; isNew: boolean } {
+  const { sighting, ...rest } = incoming;
+  const existing = record.findings.find((f) => f.id === rest.id);
+
+  if (!existing) {
+    record.findings.push({
+      ...rest,
+      firstSeen: sighting.at,
+      lastSeen: sighting.at,
+      occurrences: 1,
+      sightings: [sighting],
+    });
+    return { record, isNew: true };
+  }
+
+  existing.occurrences += 1;
+  if (sighting.at > existing.lastSeen) existing.lastSeen = sighting.at;
+  if (sighting.at < existing.firstSeen) existing.firstSeen = sighting.at;
+
+  // Keep the FIRST sightings rather than the most recent. The first exposure is
+  // the one that explains how the credential got into the transcript at all;
+  // later ones are usually the same value being replayed, and a checkpoint
+  // record replaying a prompt is not new information.
+  if (existing.sightings.length < MAX_SIGHTINGS) {
+    const seen = existing.sightings.some(
+      (s) => s.sessionId === sighting.sessionId && s.at === sighting.at,
+    );
+    if (!seen) existing.sightings.push(sighting);
+  }
+  return { record, isNew: false };
+}
+
+/**
+ * Drop what is no longer worth keeping, so the file cannot grow without bound.
+ *
+ * Order matters: age out first, then cap. Capping first would let a burst of
+ * stale findings evict fresh ones.
+ */
+export function pruneRecord(record: LeakRecord, nowMs: number): LeakRecord {
+  const cutoff = new Date(nowMs - FINDING_TTL_DAYS * 86_400_000).toISOString();
+  const live = record.findings.filter((f) => f.lastSeen >= cutoff);
+  live.sort((a, b) => (a.lastSeen < b.lastSeen ? 1 : a.lastSeen > b.lastSeen ? -1 : 0));
+  record.findings = live.slice(0, MAX_FINDINGS);
+  return record;
+}
+
+/** An empty record, for a machine that has never scanned. */
+export function emptyRecord(salt: string, nowIso: string): LeakRecord {
+  return {
+    schemaVersion: LEAK_RECORD_SCHEMA_VERSION,
+    salt,
+    updatedAt: nowIso,
+    findings: [],
+  };
+}
+
+/**
+ * Describe how a credential came to be in a tool event.
+ *
+ * Specific by construction: the path or command is what makes a finding
+ * recognisable to the person who caused it, and a generic phrase ("detected in
+ * tool input") is the kind of alert people learn to ignore.
+ */
+export function describeMechanism(
+  toolName: string,
+  direction: "input" | "result",
+  shortPath: string | null,
+): LeakMechanism {
+  const where = shortPath ? ` ${shortPath}` : "";
+  let summary: string;
+  if (direction === "result") {
+    summary = shortPath ? `appeared in the output of reading${where}` : "appeared in tool output";
+  } else {
+    switch (toolName) {
+      case "Read":
+        summary = `read from${where || " a file"}`;
+        break;
+      case "Write":
+        summary = `written to${where || " a file"}`;
+        break;
+      case "Edit":
+      case "NotebookEdit":
+        summary = `edited into${where || " a file"}`;
+        break;
+      case "Bash":
+        summary = "passed in a shell command";
+        break;
+      default:
+        summary = `sent to the ${toolName} tool`;
+    }
+  }
+  return { summary, toolName, direction };
+}

package/src/audit/leak-scan.ts

@@ -0,0 +1,292 @@
+/**
+ * Pulling credential VALUES out of a tool event.
+ *
+ * The audit already replays every event through the policy engine, but
+ * `ReplayHit` carries `{policyName, decision, reason}` and no matched text —
+ * the engine answers "did a rule fire", not "on what". That is the right answer
+ * for enforcement and the wrong one for a leak report, which has to say *which*
+ * key, so this walks the event itself.
+ *
+ * Running beside the replay rather than inside it is also what the pattern
+ * census concluded independently: detection wants a vocabulary tuned for
+ * PRECISION (a false positive becomes an email to a customer about a key that
+ * never leaked), while the redactor's vocabulary is tuned for RECALL (a false
+ * positive costs a few characters in a digest). Sharing one list would drag one
+ * of them to the other's error costs — measured, importing the detector's names
+ * into the redactor stopped 305 of 646 currently-masked names from being masked.
+ *
+ * ## Where it looks, and why both directions matter
+ *
+ * Tool INPUT and tool RESULT are separate exposures with separate remedies. An
+ * input is what the agent SENT — deniable at PreToolUse on all 12 CLIs, and
+ * 39.5% of measured credential appearances live there. A result is what the
+ * agent RECEIVED, blockable on 2 of 12 and only after the fact. The record keeps
+ * them apart so the report can tell a user which of the two happened.
+ */
+import { SECRET_PATTERNS } from "../hooks/builtin-policies";
+import { isSecretName } from "./redact-example";
+
+/** One credential value found in one event. */
+export interface SecretMatch {
+  /** The raw value. Held in memory only — the caller fingerprints it and must
+   *  never persist or transmit this. */
+  value: string;
+  /** The identifier it was assigned to, when it came from an assignment. For
+   *  the first-party class (230 of 237 secret-named assignments match no vendor
+   *  pattern) this is the only actionable label there is. */
+  name: string | null;
+  /** Which rule found it — a vendor label, or "assigned secret". */
+  rule: string;
+  /** Whether the value carried a recognised vendor shape. Drives the record's
+   *  confidence, and therefore whether an alert may name a console. */
+  shaped: boolean;
+}
+
+/**
+ * `NAME=value` in its four real spellings.
+ *
+ * Kept in step with `redact-example.ts`'s ASSIGNMENT_RE — same four shapes, same
+ * scheme guard — because a value the DETECTOR can report but the REDACTOR
+ * cannot mask is a leak in the report itself. The redactor may be broader; it
+ * must never be narrower. `__tests__/audit/leak-scan.test.ts` asserts that
+ * direction explicitly.
+ */
+/**
+ * The identifier is BOUNDED, and the bound is load-bearing rather than tidy.
+ *
+ * Unbounded (`[A-Za-z0-9_]*`), this is quadratic on a long unbroken token: the
+ * engine starts at every one of n positions, consumes the whole run, then
+ * backtracks a character at a time looking for the `=` or `:` that never comes.
+ * Measured on a plain 300 KB base64 blob — `"A".repeat(300000)` — it ran for
+ * over 20 SECONDS on one string. That is not an exotic input: agent transcripts
+ * carry base64 images, minified bundles and whole file contents as single
+ * lines, and the audit reads every one of them. A scheduled scan hitting a few
+ * of these would stall for minutes with nobody watching.
+ *
+ * 128 characters is far past any real environment-variable or field name, so
+ * bounding it costs nothing that exists and makes the scan linear in n.
+ */
+const ASSIGNMENT_RE =
+  /(["']?)\b([A-Za-z_][A-Za-z0-9_]{0,127})\1([ \t]*(?::(?!\/\/)|=)[ \t]*)("[^"]*"|'[^']*'|[^\s;|&"']+)/g;
+
+/** Shortest value worth reporting. Below this it is a flag, a boolean or a
+ *  placeholder far more often than a credential. */
+const MIN_VALUE_LEN = 8;
+
+/**
+ * Values that are structurally incapable of being a leaked credential.
+ *
+ * Deliberately structural, never entropy: the corpus's one confirmed-live
+ * password measures 3.19 bits per character while 1.3 million UUIDs in the same
+ * corpus sit at 3.72, so any entropy threshold that catches the password also
+ * catches every UUID. These tests are decisive instead — a `$VAR` reference
+ * contains no secret by construction, and `process.env.X` is the CORRECT secure
+ * form, which the redactor currently flags 7,870 times.
+ */
+function isNotACredential(value: string): boolean {
+  if (value.length < MIN_VALUE_LEN) return true;
+  // Shell/template indirection: the value is a reference, not a secret.
+  if (/^[$<{(`%]/.test(value)) return true;
+  if (value.includes("${") || value.includes("{{") || value.includes("%(")) return true;
+  // A code expression — `process.env.DISCORD_TOKEN` is how you SHOULD do it.
+  if (/^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)+$/.test(value)) return true;
+  // Already masked, by us or by anyone else.
+  if (value.includes("[REDACTED") || /^[•*#.]+$/.test(value)) return true;
+  if (/(.)\1{5,}/.test(value)) return true;          // xxxxxxxx, 00000000
+  if (/^(true|false|null|undefined|none)$/i.test(value)) return true;
+  if (/^-?\d+(\.\d+)?$/.test(value)) return true;    // pure number
+  if (/^\/|^~\/|^\.\.?\//.test(value)) return true;  // a path
+  return false;
+}
+
+/** Strip one layer of surrounding quotes. */
+function unquote(v: string): string {
+  return (v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))
+    ? v.slice(1, -1)
+    : v;
+}
+
+/**
+ * SHA-256 of credentials the vendors themselves publish in their documentation.
+ *
+ * The single highest-value refuter in the whole detector, and the only one that
+ * is decisive rather than heuristic. Measured on the real corpus: **456 of 457
+ * `AKIA` matches were AWS's own documentation literal** — 99.8% of that
+ * pattern's entire output, and 232 findings collapsing to zero. Across the
+ * pattern census, 44 of 113 patterns with five or more hits were at least 85%
+ * one literal, and 30 fired on nothing else at all.
+ *
+ * Stored as hashes so this file does not itself contain the values — a
+ * denylist written in plaintext is a file our own scanner flags, and one that
+ * lands in the corpus of every machine that installs us.
+ *
+ * A hash comparison cannot false-positive: either the value is byte-identical
+ * to a published sample or it is not. That is what makes this worth more than
+ * any amount of pattern tuning.
+ */
+const DOCS_LITERALS = new Set([
+  "1a5d44a2dca19669d72edf4c4f1c27c4c1ca4b4408fbb17f6ce4ad452d78ddb3", // AWS access key ID
+  "78314b11be2e581549ac1c4f616563fad3fdf0c3b71678f6e2299182080e0598", // AWS secret access key
+  "cb20d34a7adda2f5d2043836051e242fd4c2b233e02c80c91fe9d5af9b100574", // AWS temporary key ID
+  "3543c50b0cb9cea8e55eb1f529b79f043cb250d485fde7b66cabbb5183add7da", // GitHub PAT sample
+  "1fae28c1b317300acfd90078c973567c8159015822b0c8b81508a5ec40517d20", // Google API key sample
+  "23219833092a3136d7ad178ccb62ebd6ac8cba54ff24c2cfcd37b84aba9e95cf", // SendGrid key sample
+  "872db7f4419592e397ef45230f910e06b48c36ed9970cf1f9e4fb896e69f129c", // Slack bot token sample
+  "78a08441f4314f0a2833cfe58c62e555a162264206982cda39f6804f5048f570", // Stripe live key sample
+  "2cafc0970149a84f3b9e62eaf169f36f59907a3b3e31f7b82e68c69cd27f7326", // Stripe test key sample
+  "b363ade84d56c43bebbc9339b4971e340e3f13f7f8d45580b53373d2763e9e33", // Twilio account SID sample
+]);
+
+/** True when the value is a credential a vendor published on purpose. */
+export function isDocsLiteral(value: string): boolean {
+  // Required lazily so this module stays importable from the browser bundle.
+  const { createHash } = require("node:crypto") as typeof import("node:crypto");
+  return DOCS_LITERALS.has(createHash("sha256").update(value).digest("hex"));
+}
+
+/**
+ * Text that is failproofai's OWN output being replayed back at us.
+ *
+ * Auditing for secrets writes secrets into the corpus the next audit reads.
+ * That is measured, not theoretical: the single largest organic cluster in
+ * 1.6 GB of transcripts was one file — a previous secret-audit run quoting back
+ * the credentials it had found (123 Atlassian tokens, 26 Telegram, 25 Slack,
+ * 18 GitHub) — and 108 of the 161 patterns that matched anything matched ONLY
+ * inside the investigation's own transcripts.
+ *
+ * Left unhandled the count climbs every scan, each report inflating the next.
+ * These markers appear in our own rendered output and essentially nowhere else.
+ */
+const SELF_OUTPUT_MARKERS = [
+  "[REDACTED:",
+  "[redacted:",
+  "failproofai audit",
+  "assigned secret]",
+];
+
+/** True when this text is our own report, not a transcript of real work. */
+export function isSelfOutput(text: string): boolean {
+  return SELF_OUTPUT_MARKERS.some((m) => text.includes(m));
+}
+
+/**
+ * Literal prefixes that must appear before any vendor regex is worth running.
+ *
+ * A prefilter, and it is not an optimisation — it is what makes the pattern set
+ * shippable at all. The census measured the cost as LINEAR in pattern count:
+ * ~0.095s per pattern per 6 MB, so 288 patterns over a 1.6 GB corpus is roughly
+ * two hours, while the handful that ever match take five seconds. Scanning is
+ * already the audit's slowest phase and the whole point of adding vendors is
+ * that most of them never fire on any given machine.
+ *
+ * So: one cheap pass over the text asking "does any prefix appear at all", and
+ * the regexes only run when one does. Derived from `SECRET_PATTERNS` at module
+ * load rather than hand-maintained, because a hand-copied list that drifts from
+ * its source silently stops matching — the exact failure class this repo already
+ * has a tripwire test for elsewhere.
+ *
+ * Patterns with no literal prefix (a bare JWT, a connection string, the Telegram
+ * `\d{8,10}:` shape) yield no gate token and are always run; they are few, and
+ * a pattern that cannot be prefiltered is one whose cost has to be paid.
+ */
+function literalPrefixOf(source: string): string | null {
+  // Walk the regex source until the first construct that is not a plain
+  // literal. Anything before that is text the input must contain verbatim.
+  let out = "";
+  for (let i = 0; i < source.length; i++) {
+    const c = source[i];
+    if (c === "\\") {
+      // An escape: only a handful are literal characters we can rely on.
+      const next = source[i + 1];
+      if (next === undefined) break;
+      if (/[dwsSDWbB]/.test(next)) break; // a class, not a literal
+      out += next;
+      i++;
+      continue;
+    }
+    if ("[](){}|?*+^$.".includes(c)) break;
+    out += c;
+  }
+  // Two characters is the shortest gate worth having: shorter than that and the
+  // prefilter matches nearly every input and buys nothing.
+  return out.length >= 2 ? out : null;
+}
+
+/** Built once. Each entry is `[pattern, label, gate]`; a null gate always runs. */
+const GATED_PATTERNS: ReadonlyArray<readonly [RegExp, string, string | null]> =
+  SECRET_PATTERNS.map(([re, label]) => [re, label, literalPrefixOf(re.source)] as const);
+
+/** Patterns worth running against this text. */
+function applicablePatterns(text: string): ReadonlyArray<readonly [RegExp, string, string | null]> {
+  return GATED_PATTERNS.filter(([, , gate]) => gate === null || text.includes(gate));
+}
+
+/**
+ * Every credential value in one blob of text.
+ *
+ * Vendor SHAPES first, then secret-NAMED assignments, deduped by value so one
+ * key matched by both layers is reported once — under its vendor label, which
+ * is the one that can carry a revocation link. The two layers are near-disjoint
+ * in practice: 55.9% of real vendor-shaped credentials in the corpus have no
+ * secret-ish word within 60 characters, and 230 of 237 secret-named assignments
+ * match no vendor pattern at all.
+ *
+ * Both refuters run before anything is reported: our own replayed output is
+ * dropped whole, and every published documentation sample is dropped by hash.
+ */
+export function findSecrets(text: string): SecretMatch[] {
+  if (!text) return [];
+  // Our own report replayed into a transcript is not a leak — see isSelfOutput.
+  if (isSelfOutput(text)) return [];
+  const byValue = new Map<string, SecretMatch>();
+
+  for (const [pattern, label] of applicablePatterns(text)) {
+    const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g");
+    for (const m of text.matchAll(re)) {
+      const value = m[0];
+      if (isDocsLiteral(value)) continue;
+      if (!byValue.has(value)) byValue.set(value, { value, name: null, rule: label, shaped: true });
+    }
+  }
+
+  for (const m of text.matchAll(ASSIGNMENT_RE)) {
+    const name = m[2];
+    const value = unquote(m[4]);
+    if (!isSecretName(name) || isNotACredential(value) || isDocsLiteral(value)) continue;
+    const existing = byValue.get(value);
+    if (existing) {
+      // A vendor-shaped value that also has a name: keep the vendor rule (it
+      // can name a console) and gain the name (it says which of the user's
+      // variables to change).
+      existing.name ??= name;
+      continue;
+    }
+    byValue.set(value, { value, name, rule: "assigned secret", shaped: false });
+  }
+
+  return [...byValue.values()];
+}
+
+/**
+ * Serialize a tool input for scanning.
+ *
+ * Every string leaf, not just `command`: the measured surface includes file
+ * bodies on Write, replacement text on Edit, headers on WebFetch and arbitrary
+ * MCP payloads. `JSON.stringify` would work but escapes quotes, which breaks
+ * the assignment grammar on exactly the JSON-config shape that carries most
+ * first-party keys — so the leaves are joined raw.
+ */
+export function flattenToolInput(input: unknown, depth = 0): string {
+  if (depth > 6 || input == null) return "";
+  if (typeof input === "string") return input;
+  if (typeof input === "number" || typeof input === "boolean") return String(input);
+  if (Array.isArray(input)) return input.map((v) => flattenToolInput(v, depth + 1)).join("\n");
+  if (typeof input === "object") {
+    return Object.entries(input as Record<string, unknown>)
+      // Keep the key adjacent to the value so `{"api_key": "…"}` still reads as
+      // an assignment to the grammar above.
+      .map(([k, v]) => `${k}=${flattenToolInput(v, depth + 1)}`)
+      .join("\n");
+  }
+  return "";
+}