failproofai 1.0.4-beta.7 → 1.0.4

package/src/audit/leak-record.ts

@@ -1,248 +0,0 @@
-/**
- * The per-credential record — what the audit knows that `AuditCount` cannot say.
- *
- * `AuditCount` aggregates BY RULE: it can report that `sanitize-api-keys` fired
- * 47 times and show three 80-character examples. It cannot enumerate 47 distinct
- * keys, say which of them is the same key seen twice, or answer "which CLI, in
- * which project, at what time, and how did it get there". Every surface the
- * narrowed product needs — the emailed digest, the dashboard's leak view, and
- * the once-per-finding notification guard — asks exactly those questions, so
- * none of them can be built on the rule-level record.
- *
- * ## Grouped by VALUE, not by occurrence — this is the whole bloat story
- *
- * The obvious shape is one row per sighting. Measured against the real corpus
- * that is 10,301 rows, against roughly 266 distinct values, against about 25
- * genuinely real credentials. Two orders of magnitude of duplication, because
- * one pasted key fans out across every session that touched it — the single
- * confirmed-live password in the corpus appeared in 11 files and 22 occurrences,
- * partly because Claude's own checkpoint records replay the prompt that carried
- * it.
- *
- * So the unit is the DISTINCT VALUE, with a bounded list of sightings attached
- * as evidence. A finding seen a thousand times costs the same on disk as one
- * seen twice: `MAX_SIGHTINGS` entries, plus a count. At the corpus's own numbers
- * that is roughly 130 KB for the whole file, which is why this is one file and
- * not a directory — a per-finding file would spend more on inodes and directory
- * entries than on the findings.
- *
- * ## What is deliberately NOT stored
- *
- * The credential itself, ever. The record holds a salted id (for dedupe across
- * scans) and a fingerprint (for a human to recognise the key in a console) and
- * nothing that could reconstruct the value. `~/.failproofai/` is a directory the
- * user did not ask for and cannot audit, so a leak scanner that caches the leaks
- * has made the problem worse rather than better.
- */
-import type { IntegrationType } from "../hooks/types";
-import type { LeakFingerprint } from "./leak-fingerprint";
-
-/** Sightings kept per distinct value. Enough to answer 5W1H for the first and
- *  most recent exposure without storing the long tail nobody reads. */
-export const MAX_SIGHTINGS = 5;
-
-/** Findings not seen in this many days are dropped on the next write. A key
- *  whose transcripts have aged out of the scan window is not evidence of
- *  anything current, and keeping it forever grows the file without bound. */
-export const FINDING_TTL_DAYS = 90;
-
-/** Cap on distinct findings retained, newest-first. A machine with more than
- *  this is not going to triage them, and the census measured a name-only rule
- *  emitting thousands — the cap is what stops a bad matcher filling the disk. */
-export const MAX_FINDINGS = 500;
-
-/**
- * How a credential came to be where it was found — the "how" of 5W1H.
- *
- * Derived from the tool event rather than authored, so it is always specific:
- * "read from ~/…/.env" beats "detected in a file operation". The path is
- * home-shortened by the caller before it reaches here.
- */
-export interface LeakMechanism {
-  /** Short past-tense phrase, e.g. "read from ~/…/.env". */
-  summary: string;
-  /** Canonical tool name (Bash, Read, Write, Edit…). */
-  toolName: string;
-  /** Where the value appeared: the tool's INPUT (the agent sent it) or its
-   *  RESULT (the agent received it). Different exposures with different
-   *  remedies — an input is deniable at PreToolUse, a result is not. */
-  direction: "input" | "result";
-}
-
-/** One exposure of one credential: the where/when/who of 5W1H. */
-export interface LeakSighting {
-  /** WHO — the harness whose transcript carried it. */
-  cli: IntegrationType;
-  /** WHERE — session and project. `cwd` is already home-shortened. */
-  sessionId: string;
-  cwd: string;
-  /** WHEN — ISO-8601, from the transcript line. */
-  at: string;
-  /** HOW. */
-  mechanism: LeakMechanism;
-}
-
-/** One distinct credential, however many times it was seen. */
-export interface LeakFinding {
-  /** Salted, machine-local, stable across scans. Never derived from the
-   *  fingerprint, which is not unique. */
-  id: string;
-  /** WHAT — the masked rendering, its class, and its length. */
-  fingerprint: LeakFingerprint;
-  /** The identifier it was assigned to, when it came from an assignment
-   *  (`COMPOSIO_API_KEY`). For the 230-of-237 first-party class this is the
-   *  only actionable field there is, since no vendor console exists. */
-  name: string | null;
-  /** Which rule found it, for explaining and for suppression. */
-  rule: string;
-  /** How much we trust the attribution — mirrors the pattern census's tiers. */
-  confidence: "doc-verified" | "source-verified" | "partner-corroborated" | "observed-only" | "name-only";
-  firstSeen: string;
-  lastSeen: string;
-  /** Total exposures, which is NOT `sightings.length` — see MAX_SIGHTINGS. */
-  occurrences: number;
-  /** Bounded evidence, oldest first. */
-  sightings: LeakSighting[];
-  /** Set when the user dismisses it. Dismissed findings are retained (so they
-   *  do not re-alert) but never notified about again. */
-  dismissedAt?: string;
-  /** Set when a notice for this finding has been delivered, so the next scan
-   *  stays quiet. The delivery guard itself is a separate O_EXCL marker; this
-   *  is the human-readable record of it. */
-  notifiedAt?: string;
-}
-
-export interface LeakRecord {
-  schemaVersion: number;
-  /** Per-machine HMAC salt for `fingerprintId`. Generated once, never sent. */
-  salt: string;
-  updatedAt: string;
-  findings: LeakFinding[];
-}
-
-export const LEAK_RECORD_SCHEMA_VERSION = 1;
-
-/**
- * Fold one new sighting into the record.
- *
- * Returns whether this is a **newly seen** credential, which is what the
- * notification guard keys on: a finding whose id is already present must not
- * alert again, however many new sightings it accumulates. Getting this backwards
- * is the failure the census measured — keying on the rule-level examples, which
- * `mergeIncremental` freezes at the first three, silently misses the arrival of
- * a brand-new key while re-alerting for old ones.
- */
-export function upsertFinding(
-  record: LeakRecord,
-  incoming: Omit<LeakFinding, "firstSeen" | "lastSeen" | "occurrences" | "sightings"> & {
-    sighting: LeakSighting;
-  },
-): { record: LeakRecord; isNew: boolean } {
-  const { sighting, ...rest } = incoming;
-  const existing = record.findings.find((f) => f.id === rest.id);
-
-  if (!existing) {
-    record.findings.push({
-      ...rest,
-      firstSeen: sighting.at,
-      lastSeen: sighting.at,
-      occurrences: 1,
-      sightings: [sighting],
-    });
-    return { record, isNew: true };
-  }
-
-  // Keep the earliest sighting AND the most recent evidence. The report/email
-  // promises the CLI and timestamp of the latest exposure; keeping only the
-  // first five made those fields stale forever after a busy credential crossed
-  // the cap. Middle sightings are the expendable ones.
-  const seen = existing.sightings.some(
-    (s) =>
-      s.sessionId === sighting.sessionId &&
-      s.at === sighting.at &&
-      s.mechanism.direction === sighting.mechanism.direction &&
-      s.mechanism.summary === sighting.mechanism.summary,
-  );
-  // Cached transcript results are folded through this function again. An
-  // identical sighting is therefore not a new exposure and must not inflate
-  // the count each time the same cache entry is reused.
-  if (seen) return { record, isNew: false };
-
-  existing.occurrences += 1;
-  if (sighting.at > existing.lastSeen) existing.lastSeen = sighting.at;
-  if (sighting.at < existing.firstSeen) existing.firstSeen = sighting.at;
-
-  existing.sightings.push(sighting);
-  existing.sightings.sort((a, b) => a.at.localeCompare(b.at));
-  if (existing.sightings.length > MAX_SIGHTINGS) {
-    existing.sightings.splice(1, existing.sightings.length - MAX_SIGHTINGS);
-  }
-  return { record, isNew: false };
-}
-
-/**
- * Drop what is no longer worth keeping, so the file cannot grow without bound.
- *
- * Order matters: age out first, then cap. Capping first would let a burst of
- * stale findings evict fresh ones.
- */
-export function pruneRecord(record: LeakRecord, nowMs: number): LeakRecord {
-  const cutoff = new Date(nowMs - FINDING_TTL_DAYS * 86_400_000).toISOString();
-  // A missing or malformed timestamp cannot be aged out safely. Keep it until
-  // the bounded-cap pass below rather than silently deleting a finding that the
-  // reporting path deliberately surfaces as having an unknown date.
-  const live = record.findings.filter(
-    (f) => !f.lastSeen || !Number.isFinite(Date.parse(f.lastSeen)) || f.lastSeen >= cutoff,
-  );
-  live.sort((a, b) => (a.lastSeen < b.lastSeen ? 1 : a.lastSeen > b.lastSeen ? -1 : 0));
-  record.findings = live.slice(0, MAX_FINDINGS);
-  return record;
-}
-
-/** An empty record, for a machine that has never scanned. */
-export function emptyRecord(salt: string, nowIso: string): LeakRecord {
-  return {
-    schemaVersion: LEAK_RECORD_SCHEMA_VERSION,
-    salt,
-    updatedAt: nowIso,
-    findings: [],
-  };
-}
-
-/**
- * Describe how a credential came to be in a tool event.
- *
- * Specific by construction: the path or command is what makes a finding
- * recognisable to the person who caused it, and a generic phrase ("detected in
- * tool input") is the kind of alert people learn to ignore.
- */
-export function describeMechanism(
-  toolName: string,
-  direction: "input" | "result",
-  shortPath: string | null,
-): LeakMechanism {
-  const where = shortPath ? ` ${shortPath}` : "";
-  let summary: string;
-  if (direction === "result") {
-    summary = shortPath ? `appeared in the output of reading${where}` : "appeared in tool output";
-  } else {
-    switch (toolName) {
-      case "Read":
-        summary = `read from${where || " a file"}`;
-        break;
-      case "Write":
-        summary = `written to${where || " a file"}`;
-        break;
-      case "Edit":
-      case "NotebookEdit":
-        summary = `edited into${where || " a file"}`;
-        break;
-      case "Bash":
-        summary = "passed in a shell command";
-        break;
-      default:
-        summary = `sent to the ${toolName} tool`;
-    }
-  }
-  return { summary, toolName, direction };
-}

package/src/audit/leak-scan.ts

@@ -1,268 +0,0 @@
-/**
- * Pulling credential VALUES out of a tool event.
- *
- * The audit already replays every event through the policy engine, but
- * `ReplayHit` carries `{policyName, decision, reason}` and no matched text —
- * the engine answers "did a rule fire", not "on what". That is the right answer
- * for enforcement and the wrong one for a leak report, which has to say *which*
- * key, so this walks the event itself.
- *
- * Running beside the replay rather than inside it is also what the pattern
- * census concluded independently: detection wants a vocabulary tuned for
- * PRECISION (a false positive becomes an email to a customer about a key that
- * never leaked), while the redactor's vocabulary is tuned for RECALL (a false
- * positive costs a few characters in a digest). Sharing one list would drag one
- * of them to the other's error costs — measured, importing the detector's names
- * into the redactor stopped 305 of 646 currently-masked names from being masked.
- *
- * ## Where it looks, and why both directions matter
- *
- * Tool INPUT and tool RESULT are separate exposures with separate remedies. An
- * input is what the agent SENT — deniable at PreToolUse on all 12 CLIs, and
- * 39.5% of measured credential appearances live there. A result is what the
- * agent RECEIVED, blockable on 2 of 12 and only after the fact. The record keeps
- * them apart so the report can tell a user which of the two happened.
- */
-import { SECRET_PATTERNS } from "../hooks/builtin-policies";
-
-/**
- * Cache key for the credential detector.
- *
- * The transcript cache used to key only on builtin policies and behavioural
- * audit detectors. Changing THIS scanner therefore kept serving the old
- * `result.leaks` forever, which made a precision fix appear to do nothing on a
- * warm machine. Bump this whenever matching or suppression semantics change.
- */
-export const LEAK_SCAN_VERSION = 6;
-
-/** One credential value found in one event. */
-export interface SecretMatch {
-  /** The raw value. Held in memory only — the caller fingerprints it and must
-   *  never persist or transmit this. */
-  value: string;
-  /** The identifier it was assigned to, when a recognized shaped value also
-   *  appeared in an assignment. Used only as an actionable label. */
-  name: string | null;
-  /** Which recognizable credential shape found it. */
-  rule: string;
-  /** Whether the value carried a recognised vendor shape. Drives the record's
-   *  confidence, and therefore whether an alert may name a console. */
-  shaped: boolean;
-}
-
-/**
- * `NAME=value` in its four real spellings.
- *
- * Kept in step with `redact-example.ts`'s ASSIGNMENT_RE — same four shapes, same
- * scheme guard — because a value the DETECTOR can report but the REDACTOR
- * cannot mask is a leak in the report itself. The redactor may be broader; it
- * must never be narrower. `__tests__/audit/leak-scan.test.ts` asserts that
- * direction explicitly.
- */
-/**
- * The identifier is BOUNDED, and the bound is load-bearing rather than tidy.
- *
- * Unbounded (`[A-Za-z0-9_]*`), this is quadratic on a long unbroken token: the
- * engine starts at every one of n positions, consumes the whole run, then
- * backtracks a character at a time looking for the `=` or `:` that never comes.
- * Measured on a plain 300 KB base64 blob — `"A".repeat(300000)` — it ran for
- * over 20 SECONDS on one string. That is not an exotic input: agent transcripts
- * carry base64 images, minified bundles and whole file contents as single
- * lines, and the audit reads every one of them. A scheduled scan hitting a few
- * of these would stall for minutes with nobody watching.
- *
- * 128 characters is far past any real environment-variable or field name, so
- * bounding it costs nothing that exists and makes the scan linear in n.
- */
-const ASSIGNMENT_RE =
-  /(["']?)\b([A-Za-z_][A-Za-z0-9_]{0,127})\1([ \t]*(?::(?!\/\/)|=)[ \t]*)("[^"]*"|'[^']*'|[^\s;|&"']+)/g;
-
-/** Strip one layer of surrounding quotes. */
-function unquote(v: string): string {
-  return (v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))
-    ? v.slice(1, -1)
-    : v;
-}
-
-/**
- * SHA-256 of credentials the vendors themselves publish in their documentation.
- *
- * The single highest-value refuter in the whole detector, and the only one that
- * is decisive rather than heuristic. Measured on the real corpus: **456 of 457
- * `AKIA` matches were AWS's own documentation literal** — 99.8% of that
- * pattern's entire output, and 232 findings collapsing to zero. Across the
- * pattern census, 44 of 113 patterns with five or more hits were at least 85%
- * one literal, and 30 fired on nothing else at all.
- *
- * Stored as hashes so this file does not itself contain the values — a
- * denylist written in plaintext is a file our own scanner flags, and one that
- * lands in the corpus of every machine that installs us.
- *
- * A hash comparison cannot false-positive: either the value is byte-identical
- * to a published sample or it is not. That is what makes this worth more than
- * any amount of pattern tuning.
- */
-const DOCS_LITERALS = new Set([
-  "1a5d44a2dca19669d72edf4c4f1c27c4c1ca4b4408fbb17f6ce4ad452d78ddb3", // AWS access key ID
-  "78314b11be2e581549ac1c4f616563fad3fdf0c3b71678f6e2299182080e0598", // AWS secret access key
-  "cb20d34a7adda2f5d2043836051e242fd4c2b233e02c80c91fe9d5af9b100574", // AWS temporary key ID
-  "3543c50b0cb9cea8e55eb1f529b79f043cb250d485fde7b66cabbb5183add7da", // GitHub PAT sample
-  "1fae28c1b317300acfd90078c973567c8159015822b0c8b81508a5ec40517d20", // Google API key sample
-  "23219833092a3136d7ad178ccb62ebd6ac8cba54ff24c2cfcd37b84aba9e95cf", // SendGrid key sample
-  "872db7f4419592e397ef45230f910e06b48c36ed9970cf1f9e4fb896e69f129c", // Slack bot token sample
-  "78a08441f4314f0a2833cfe58c62e555a162264206982cda39f6804f5048f570", // Stripe live key sample
-  "2cafc0970149a84f3b9e62eaf169f36f59907a3b3e31f7b82e68c69cd27f7326", // Stripe test key sample
-  "b363ade84d56c43bebbc9339b4971e340e3f13f7f8d45580b53373d2763e9e33", // Twilio account SID sample
-]);
-
-/** True when the value is a credential a vendor published on purpose. */
-export function isDocsLiteral(value: string): boolean {
-  // Required lazily so this module stays importable from the browser bundle.
-  const { createHash } = require("node:crypto") as typeof import("node:crypto");
-  return DOCS_LITERALS.has(createHash("sha256").update(value).digest("hex"));
-}
-
-/**
- * Text that is failproofai's OWN output being replayed back at us.
- *
- * Auditing for secrets writes secrets into the corpus the next audit reads.
- * That is measured, not theoretical: the single largest organic cluster in
- * 1.6 GB of transcripts was one file — a previous secret-audit run quoting back
- * the credentials it had found (123 Atlassian tokens, 26 Telegram, 25 Slack,
- * 18 GitHub) — and 108 of the 161 patterns that matched anything matched ONLY
- * inside the investigation's own transcripts.
- *
- * Left unhandled the count climbs every scan, each report inflating the next.
- * These markers appear in our own rendered output and essentially nowhere else.
- */
-const SELF_OUTPUT_MARKERS = [
-  "[REDACTED:",
-  "[redacted:",
-  "failproofai audit",
-  "assigned secret]",
-];
-
-/** True when this text is our own report, not a transcript of real work. */
-export function isSelfOutput(text: string): boolean {
-  return SELF_OUTPUT_MARKERS.some((m) => text.includes(m));
-}
-
-/**
- * Literal prefixes that must appear before any vendor regex is worth running.
- *
- * A prefilter, and it is not an optimisation — it is what makes the pattern set
- * shippable at all. The census measured the cost as LINEAR in pattern count:
- * ~0.095s per pattern per 6 MB, so 288 patterns over a 1.6 GB corpus is roughly
- * two hours, while the handful that ever match take five seconds. Scanning is
- * already the audit's slowest phase and the whole point of adding vendors is
- * that most of them never fire on any given machine.
- *
- * So: one cheap pass over the text asking "does any prefix appear at all", and
- * the regexes only run when one does. Derived from `SECRET_PATTERNS` at module
- * load rather than hand-maintained, because a hand-copied list that drifts from
- * its source silently stops matching — the exact failure class this repo already
- * has a tripwire test for elsewhere.
- *
- * Patterns with no literal prefix (a bare JWT, a connection string, the Telegram
- * `\d{8,10}:` shape) yield no gate token and are always run; they are few, and
- * a pattern that cannot be prefiltered is one whose cost has to be paid.
- */
-function literalPrefixOf(source: string): string | null {
-  // Walk the regex source until the first construct that is not a plain
-  // literal. Anything before that is text the input must contain verbatim.
-  let out = "";
-  for (let i = 0; i < source.length; i++) {
-    const c = source[i];
-    if (c === "\\") {
-      // An escape: only a handful are literal characters we can rely on.
-      const next = source[i + 1];
-      if (next === undefined) break;
-      if (/[dwsSDWbB]/.test(next)) break; // a class, not a literal
-      out += next;
-      i++;
-      continue;
-    }
-    if ("[](){}|?*+^$.".includes(c)) break;
-    out += c;
-  }
-  // Two characters is the shortest gate worth having: shorter than that and the
-  // prefilter matches nearly every input and buys nothing.
-  return out.length >= 2 ? out : null;
-}
-
-/** Built once. Each entry is `[pattern, label, gate]`; a null gate always runs. */
-const GATED_PATTERNS: ReadonlyArray<readonly [RegExp, string, string | null]> =
-  SECRET_PATTERNS.map(([re, label]) => [re, label, literalPrefixOf(re.source)] as const);
-
-/** Patterns worth running against this text. */
-function applicablePatterns(text: string): ReadonlyArray<readonly [RegExp, string, string | null]> {
-  return GATED_PATTERNS.filter(([, , gate]) => gate === null || text.includes(gate));
-}
-
-/**
- * Every credential value in one blob of text.
- *
- * Vendor SHAPES first, then secret-NAMED assignments, deduped by value so one
- * key matched by both layers is reported once — under its vendor label, which
- * is the one that can carry a revocation link. The two layers are near-disjoint
- * in practice: 55.9% of real vendor-shaped credentials in the corpus have no
- * secret-ish word within 60 characters, and 230 of 237 secret-named assignments
- * match no vendor pattern at all.
- *
- * Both refuters run before anything is reported: our own replayed output is
- * dropped whole, and every published documentation sample is dropped by hash.
- */
-export function findSecrets(text: string): SecretMatch[] {
-  if (!text) return [];
-  // Our own report replayed into a transcript is not a leak — see isSelfOutput.
-  if (isSelfOutput(text)) return [];
-  const byValue = new Map<string, SecretMatch>();
-
-  for (const [pattern, label] of applicablePatterns(text)) {
-    const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : pattern.flags + "g");
-    for (const m of text.matchAll(re)) {
-      const value = m[0];
-      if (isDocsLiteral(value)) continue;
-      if (!byValue.has(value)) byValue.set(value, { value, name: null, rule: label, shaped: true });
-    }
-  }
-
-  // Names annotate a value the shape layer already proved; they no longer
-  // create findings. Even after increasingly elaborate value heuristics, the
-  // name-only lane still produced hundreds of code literals and test fixtures
-  // on the measured machine. The product now deliberately prefers missing an
-  // unrecognisable first-party password over telling somebody they leaked 500
-  // credentials they do not have.
-  for (const m of text.matchAll(ASSIGNMENT_RE)) {
-    const name = m[2];
-    const value = unquote(m[4]);
-    const existing = byValue.get(value);
-    if (existing) existing.name ??= name;
-  }
-
-  return [...byValue.values()];
-}
-
-/**
- * Serialize a tool input for scanning.
- *
- * Every string leaf, not just `command`: the measured surface includes file
- * bodies on Write, replacement text on Edit, headers on WebFetch and arbitrary
- * MCP payloads. `JSON.stringify` would work but escapes quotes, which breaks
- * the assignment grammar on exactly the JSON-config shape that carries most
- * first-party keys — so the leaves are joined raw.
- */
-export function flattenToolInput(input: unknown, depth = 0): string {
-  if (depth > 6 || input == null) return "";
-  if (typeof input === "string") return input;
-  if (typeof input === "number" || typeof input === "boolean") return String(input);
-  if (Array.isArray(input)) return input.map((v) => flattenToolInput(v, depth + 1)).join("\n");
-  if (typeof input === "object") {
-    return Object.entries(input as Record<string, unknown>)
-      // Keep the key adjacent to the value so `{"api_key": "…"}` still reads as
-      // an assignment to the grammar above.
-      .map(([k, v]) => `${k}=${flattenToolInput(v, depth + 1)}`)
-      .join("\n");
-  }
-  return "";
-}