failproofai 1.0.4-beta.1 → 1.0.4-beta.4

package/src/audit/leak-notice.ts

@@ -0,0 +1,259 @@
+/**
+ * Deciding whether a finding still owes the user a notice — exactly once.
+ *
+ * ## Why this is not a field on the record
+ *
+ * The obvious design is `notifiedAt` on each finding, written back after the
+ * notice goes out. Measured against this repo's real `writeJsonAtomically`
+ * semantics, that fails 100% of the time under the conditions it actually runs
+ * in: two concurrent sessions with the same finding produced two notices, four
+ * produced four, eight produced eight. Worse, two concurrent sessions with
+ * DIFFERENT findings produced two notices and a surviving watermark containing
+ * only one of them — the loser's mark permanently gone, so it re-notified
+ * forever afterwards. `lib/atomic-write.ts` says so in its own header: atomic
+ * per FILE, last-writer-wins per CONTENT, i.e. a lost update on a map.
+ *
+ * Running several agents at once in one project is not an edge case here; it is
+ * how the tool is used.
+ *
+ * So the claim is one marker FILE per finding, created with `wx` — O_EXCL,
+ * which the kernel makes atomic across processes. Eight concurrent claimants
+ * produce exactly one winner, verified 3/3 trials. Concurrent claims on
+ * different findings all succeed and all survive, because they touch different
+ * files.
+ *
+ * ## Claim ordering, and which way to fail
+ *
+ * The marker is written BEFORE the notice reaches the stream, and the caller
+ * only keeps it if the notice was actually shaped. The alternative orderings
+ * were both measured: emit-then-mark nags forever if the process dies between
+ * them (5 sessions → 5 notices), and mark-then-never-emit swallows the alert
+ * permanently with nothing on disk saying so (total delivered: 0).
+ *
+ * Between "told twice" and "never told", a security tool picks told twice.
+ */
+import { existsSync, mkdirSync, readdirSync, statSync, writeFileSync, rmSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { auditDir } from "../hooks/fp-home";
+import { readLeakRecord, activeFindings, readLeakIdentity } from "./leak-store";
+import { isFindingId } from "./leak-fingerprint";
+
+/**
+ * Which way the user was told. Each channel claims independently.
+ *
+ * They are NOT one guarantee. The desktop banner and the in-CLI notice reach a
+ * person at different moments and fail in unrelated ways — and the banner has a
+ * failure it cannot detect at all: on a locked screen the shell accepts the
+ * call and returns an id while nobody sees anything. Letting a successful
+ * `Notify` claim the finding would then silence the in-session notice too, and
+ * the only channel that actually reached a human is the one we suppressed.
+ *
+ * So each channel gets its own marker directory and its own once-per-finding
+ * guarantee. A user at a desktop sees the banner AND the next session's notice;
+ * that is a bounded, deliberate duplicate, and it is the right side of the
+ * trade this file already made once (told twice beats never told).
+ */
+export type NoticeChannel = "cli" | "desktop";
+
+/** One empty file per claimed finding, per channel. Cheap: a few hundred
+ *  zero-byte files at the very worst, and the record itself is capped at 500
+ *  findings. "cli" keeps the original directory name so an existing machine's
+ *  claims survive the upgrade — renaming it would re-notify for every finding
+ *  already delivered. */
+const NOTICE_DIR = (home?: string, channel: NoticeChannel = "cli") =>
+  resolve(auditDir(home), channel === "cli" ? "notified" : `notified-${channel}`);
+
+/** Markers older than this are swept, so the directory cannot grow forever on a
+ *  machine that has been running for years. Comfortably longer than the
+ *  record's own 90-day TTL, so a marker never outlives its finding. */
+const MARKER_TTL_MS = 120 * 86_400_000;
+
+export interface PendingNotice {
+  /** Findings that have never been notified about and are not dismissed. */
+  ids: string[];
+  count: number;
+}
+
+/**
+ * Findings that still owe the user a notice.
+ *
+ * Reads the record and subtracts what has already been claimed. Never throws:
+ * on any failure it returns "nothing pending", because a hook that cannot read
+ * its bookkeeping must stay silent rather than guess.
+ */
+export function pendingLeakNotice(home?: string, channel: NoticeChannel = "cli"): PendingNotice {
+  try {
+    const dir = NOTICE_DIR(home, channel);
+    const claimed = new Set(existsSync(dir) ? readdirSync(dir) : []);
+    const ids = activeFindings(readLeakRecord(home), home)
+      .map((f) => f.id)
+      .filter((id) => !claimed.has(id));
+    return { ids, count: ids.length };
+  } catch {
+    return { ids: [], count: 0 };
+  }
+}
+
+/**
+ * Claim these findings, atomically and per-finding.
+ *
+ * Returns the ids this process actually won. A caller that loses a race simply
+ * gets a shorter list — it never blocks, and it never overwrites a claim made
+ * by a concurrent session.
+ */
+export function markLeakNoticeDelivered(
+  ids: string[],
+  home?: string,
+  channel: NoticeChannel = "cli",
+): string[] {
+  const won: string[] = [];
+  try {
+    const dir = NOTICE_DIR(home, channel);
+    mkdirSync(dir, { recursive: true });
+    for (const id of ids) {
+      // An id is about to become a filename. Anything that is not the exact
+      // shape `fingerprintId` mints is refused rather than resolved — see
+      // `isFindingId`. Refusing also means NOT reporting it as won, so a caller
+      // treats it as unclaimed and never records it as delivered.
+      if (!isFindingId(id)) continue;
+      try {
+        // `wx` is O_EXCL: it throws EEXIST if another process got here first.
+        // That throw IS the lock — there is no read-then-write window for two
+        // processes to both pass through.
+        writeFileSync(resolve(dir, id), "", { flag: "wx", mode: 0o600 });
+        won.push(id);
+      } catch {
+        // Already claimed by a concurrent session. Correct outcome: the user
+        // is told once, by whichever session got there first.
+      }
+    }
+  } catch {
+    // Cannot claim at all — the caller has already decided to emit, and being
+    // told twice beats never being told.
+  }
+  return won;
+}
+
+/** Release claims this process won when delivery failed, so the next scheduled
+ * run retries instead of turning a transient missing desktop into permanent
+ * silence. Only validated ids in the requested channel are touched. */
+export function releaseLeakNoticeClaims(
+  ids: string[],
+  home?: string,
+  channel: NoticeChannel = "cli",
+): void {
+  try {
+    const dir = NOTICE_DIR(home, channel);
+    for (const id of ids) {
+      if (!isFindingId(id)) continue;
+      rmSync(resolve(dir, id), { force: true });
+    }
+  } catch {
+    // Retry bookkeeping must never turn a completed audit into a failure.
+  }
+}
+
+/** Drop markers for findings that no longer exist, and very old ones. */
+export function pruneNoticeMarkers(home?: string, nowMs = Date.now()): void {
+  try {
+    const live = new Set(readLeakRecord(home).findings.map((f) => f.id));
+    for (const channel of ["cli", "desktop"] as const) {
+      const dir = NOTICE_DIR(home, channel);
+      if (!existsSync(dir)) continue;
+      for (const name of readdirSync(dir)) {
+        const path = resolve(dir, name);
+        let stale = !live.has(name);
+        if (!stale) {
+          try {
+            stale = nowMs - statSync(path).mtimeMs > MARKER_TTL_MS;
+          } catch {
+            stale = false;
+          }
+        }
+        if (stale) rmSync(path, { force: true });
+      }
+    }
+  } catch {
+    // Housekeeping only.
+  }
+}
+
+/**
+ * Should this session's agent tell the user about unviewed credentials?
+ *
+ * ## Why this is not the per-finding claim used for the desktop banner
+ *
+ * The per-finding marker answers "have we emitted this once?" — and emitting is
+ * not the same as arriving. Twice in one day it recorded every finding as
+ * delivered while the user saw nothing: once because the notice was attached to
+ * an event whose channel the host ignores (`SessionStart` takes
+ * `additionalContext`, not `systemMessage`), and once because hooks were
+ * disabled in the project under test. Neither is detectable from in here, and
+ * both were permanent — a claimed finding is never retried.
+ *
+ * So the in-CLI notice keys on the USER's action instead of on ours. It shows
+ * while findings exist that are newer than the last time the report was
+ * actually opened, and it stops when they open it. A dropped notice costs one
+ * session's silence rather than the alert; the only thing that silences it for
+ * good is the outcome we wanted anyway.
+ *
+ * Bounded by a per-SESSION marker so a long session gets one line, not one per
+ * turn. Sessions are cheap and few; findings are many and long-lived.
+ */
+export function sessionNoticePending(sessionId: string, home?: string): PendingNotice {
+  try {
+    if (!sessionId || !/^[\w.:-]{1,200}$/.test(sessionId)) return { ids: [], count: 0 };
+    const dir = resolve(auditDir(home), "notified-sessions");
+    if (existsSync(resolve(dir, sessionId))) return { ids: [], count: 0 };
+
+    const findings = activeFindings(readLeakRecord(home), home);
+    if (findings.length === 0) return { ids: [], count: 0 };
+
+    // Only what the user has not already looked at. `firstSeen`, not
+    // `lastSeen`: re-encountering a credential the user has already reviewed is
+    // not news, however recently the scan tripped over it again.
+    const viewedAt = readLeakIdentity(home).reportViewedAt ?? 0;
+    const unviewed = findings.filter((f) => {
+      const seen = Date.parse(f.firstSeen);
+      return !Number.isFinite(seen) || seen > viewedAt;
+    });
+    return { ids: unviewed.map((f) => f.id), count: unviewed.length };
+  } catch {
+    return { ids: [], count: 0 };
+  }
+}
+
+/**
+ * Claim this session, so the notice appears once rather than every turn.
+ *
+ * Returns whether this caller won. O_EXCL again, for the same reason as the
+ * per-finding markers: several agents run in one project at once.
+ */
+export function markSessionNoticed(sessionId: string, home?: string): boolean {
+  try {
+    if (!sessionId || !/^[\w.:-]{1,200}$/.test(sessionId)) return false;
+    const dir = resolve(auditDir(home), "notified-sessions");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(resolve(dir, sessionId), "", { flag: "wx", mode: 0o600 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Session markers outlive nothing useful — a session id never recurs. */
+export function pruneSessionMarkers(home?: string, nowMs = Date.now()): void {
+  try {
+    const dir = resolve(auditDir(home), "notified-sessions");
+    if (!existsSync(dir)) return;
+    for (const name of readdirSync(dir)) {
+      const path = resolve(dir, name);
+      try {
+        if (nowMs - statSync(path).mtimeMs > 30 * 86_400_000) rmSync(path, { force: true });
+      } catch { /* raced */ }
+    }
+  } catch {
+    // Housekeeping only.
+  }
+}

package/src/audit/leak-record.ts

@@ -0,0 +1,232 @@
+/**
+ * The per-credential record — what the audit knows that `AuditCount` cannot say.
+ *
+ * `AuditCount` aggregates BY RULE: it can report that `sanitize-api-keys` fired
+ * 47 times and show three 80-character examples. It cannot enumerate 47 distinct
+ * keys, say which of them is the same key seen twice, or answer "which CLI, in
+ * which project, at what time, and how did it get there". Every surface the
+ * narrowed product needs — the emailed digest, the dashboard's leak view, and
+ * the once-per-finding notification guard — asks exactly those questions, so
+ * none of them can be built on the rule-level record.
+ *
+ * ## Grouped by VALUE, not by occurrence — this is the whole bloat story
+ *
+ * The obvious shape is one row per sighting. Measured against the real corpus
+ * that is 10,301 rows, against roughly 266 distinct values, against about 25
+ * genuinely real credentials. Two orders of magnitude of duplication, because
+ * one pasted key fans out across every session that touched it — the single
+ * confirmed-live password in the corpus appeared in 11 files and 22 occurrences,
+ * partly because Claude's own checkpoint records replay the prompt that carried
+ * it.
+ *
+ * So the unit is the DISTINCT VALUE, with a bounded list of sightings attached
+ * as evidence. A finding seen a thousand times costs the same on disk as one
+ * seen twice: `MAX_SIGHTINGS` entries, plus a count. At the corpus's own numbers
+ * that is roughly 130 KB for the whole file, which is why this is one file and
+ * not a directory — a per-finding file would spend more on inodes and directory
+ * entries than on the findings.
+ *
+ * ## What is deliberately NOT stored
+ *
+ * The credential itself, ever. The record holds a salted id (for dedupe across
+ * scans) and a fingerprint (for a human to recognise the key in a console) and
+ * nothing that could reconstruct the value. `~/.failproofai/` is a directory the
+ * user did not ask for and cannot audit, so a leak scanner that caches the leaks
+ * has made the problem worse rather than better.
+ */
+import type { IntegrationType } from "../hooks/types";
+import type { LeakFingerprint } from "./leak-fingerprint";
+
+/** Sightings kept per distinct value. Enough to answer 5W1H for the first and
+ *  most recent exposure without storing the long tail nobody reads. */
+export const MAX_SIGHTINGS = 5;
+
+/** Findings not seen in this many days are dropped on the next write. A key
+ *  whose transcripts have aged out of the scan window is not evidence of
+ *  anything current, and keeping it forever grows the file without bound. */
+export const FINDING_TTL_DAYS = 90;
+
+/** Cap on distinct findings retained, newest-first. A machine with more than
+ *  this is not going to triage them, and the census measured a name-only rule
+ *  emitting thousands — the cap is what stops a bad matcher filling the disk. */
+export const MAX_FINDINGS = 500;
+
+/**
+ * How a credential came to be where it was found — the "how" of 5W1H.
+ *
+ * Derived from the tool event rather than authored, so it is always specific:
+ * "read from ~/…/.env" beats "detected in a file operation". The path is
+ * home-shortened by the caller before it reaches here.
+ */
+export interface LeakMechanism {
+  /** Short past-tense phrase, e.g. "read from ~/…/.env". */
+  summary: string;
+  /** Canonical tool name (Bash, Read, Write, Edit…). */
+  toolName: string;
+  /** Where the value appeared: the tool's INPUT (the agent sent it) or its
+   *  RESULT (the agent received it). Different exposures with different
+   *  remedies — an input is deniable at PreToolUse, a result is not. */
+  direction: "input" | "result";
+}
+
+/** One exposure of one credential: the where/when/who of 5W1H. */
+export interface LeakSighting {
+  /** WHO — the harness whose transcript carried it. */
+  cli: IntegrationType;
+  /** WHERE — session and project. `cwd` is already home-shortened. */
+  sessionId: string;
+  cwd: string;
+  /** WHEN — ISO-8601, from the transcript line. */
+  at: string;
+  /** HOW. */
+  mechanism: LeakMechanism;
+}
+
+/** One distinct credential, however many times it was seen. */
+export interface LeakFinding {
+  /** Salted, machine-local, stable across scans. Never derived from the
+   *  fingerprint, which is not unique. */
+  id: string;
+  /** WHAT — the masked rendering, its class, and its length. */
+  fingerprint: LeakFingerprint;
+  /** The identifier it was assigned to, when it came from an assignment
+   *  (`COMPOSIO_API_KEY`). For the 230-of-237 first-party class this is the
+   *  only actionable field there is, since no vendor console exists. */
+  name: string | null;
+  /** Which rule found it, for explaining and for suppression. */
+  rule: string;
+  /** How much we trust the attribution — mirrors the pattern census's tiers. */
+  confidence: "doc-verified" | "source-verified" | "partner-corroborated" | "observed-only" | "name-only";
+  firstSeen: string;
+  lastSeen: string;
+  /** Total exposures, which is NOT `sightings.length` — see MAX_SIGHTINGS. */
+  occurrences: number;
+  /** Bounded evidence, oldest first. */
+  sightings: LeakSighting[];
+  /** Set when the user dismisses it. Dismissed findings are retained (so they
+   *  do not re-alert) but never notified about again. */
+  dismissedAt?: string;
+  /** Set when a notice for this finding has been delivered, so the next scan
+   *  stays quiet. The delivery guard itself is a separate O_EXCL marker; this
+   *  is the human-readable record of it. */
+  notifiedAt?: string;
+}
+
+export interface LeakRecord {
+  schemaVersion: number;
+  /** Per-machine HMAC salt for `fingerprintId`. Generated once, never sent. */
+  salt: string;
+  updatedAt: string;
+  findings: LeakFinding[];
+}
+
+export const LEAK_RECORD_SCHEMA_VERSION = 1;
+
+/**
+ * Fold one new sighting into the record.
+ *
+ * Returns whether this is a **newly seen** credential, which is what the
+ * notification guard keys on: a finding whose id is already present must not
+ * alert again, however many new sightings it accumulates. Getting this backwards
+ * is the failure the census measured — keying on the rule-level examples, which
+ * `mergeIncremental` freezes at the first three, silently misses the arrival of
+ * a brand-new key while re-alerting for old ones.
+ */
+export function upsertFinding(
+  record: LeakRecord,
+  incoming: Omit<LeakFinding, "firstSeen" | "lastSeen" | "occurrences" | "sightings"> & {
+    sighting: LeakSighting;
+  },
+): { record: LeakRecord; isNew: boolean } {
+  const { sighting, ...rest } = incoming;
+  const existing = record.findings.find((f) => f.id === rest.id);
+
+  if (!existing) {
+    record.findings.push({
+      ...rest,
+      firstSeen: sighting.at,
+      lastSeen: sighting.at,
+      occurrences: 1,
+      sightings: [sighting],
+    });
+    return { record, isNew: true };
+  }
+
+  existing.occurrences += 1;
+  if (sighting.at > existing.lastSeen) existing.lastSeen = sighting.at;
+  if (sighting.at < existing.firstSeen) existing.firstSeen = sighting.at;
+
+  // Keep the FIRST sightings rather than the most recent. The first exposure is
+  // the one that explains how the credential got into the transcript at all;
+  // later ones are usually the same value being replayed, and a checkpoint
+  // record replaying a prompt is not new information.
+  if (existing.sightings.length < MAX_SIGHTINGS) {
+    const seen = existing.sightings.some(
+      (s) => s.sessionId === sighting.sessionId && s.at === sighting.at,
+    );
+    if (!seen) existing.sightings.push(sighting);
+  }
+  return { record, isNew: false };
+}
+
+/**
+ * Drop what is no longer worth keeping, so the file cannot grow without bound.
+ *
+ * Order matters: age out first, then cap. Capping first would let a burst of
+ * stale findings evict fresh ones.
+ */
+export function pruneRecord(record: LeakRecord, nowMs: number): LeakRecord {
+  const cutoff = new Date(nowMs - FINDING_TTL_DAYS * 86_400_000).toISOString();
+  const live = record.findings.filter((f) => f.lastSeen >= cutoff);
+  live.sort((a, b) => (a.lastSeen < b.lastSeen ? 1 : a.lastSeen > b.lastSeen ? -1 : 0));
+  record.findings = live.slice(0, MAX_FINDINGS);
+  return record;
+}
+
+/** An empty record, for a machine that has never scanned. */
+export function emptyRecord(salt: string, nowIso: string): LeakRecord {
+  return {
+    schemaVersion: LEAK_RECORD_SCHEMA_VERSION,
+    salt,
+    updatedAt: nowIso,
+    findings: [],
+  };
+}
+
+/**
+ * Describe how a credential came to be in a tool event.
+ *
+ * Specific by construction: the path or command is what makes a finding
+ * recognisable to the person who caused it, and a generic phrase ("detected in
+ * tool input") is the kind of alert people learn to ignore.
+ */
+export function describeMechanism(
+  toolName: string,
+  direction: "input" | "result",
+  shortPath: string | null,
+): LeakMechanism {
+  const where = shortPath ? ` ${shortPath}` : "";
+  let summary: string;
+  if (direction === "result") {
+    summary = shortPath ? `appeared in the output of reading${where}` : "appeared in tool output";
+  } else {
+    switch (toolName) {
+      case "Read":
+        summary = `read from${where || " a file"}`;
+        break;
+      case "Write":
+        summary = `written to${where || " a file"}`;
+        break;
+      case "Edit":
+      case "NotebookEdit":
+        summary = `edited into${where || " a file"}`;
+        break;
+      case "Bash":
+        summary = "passed in a shell command";
+        break;
+      default:
+        summary = `sent to the ${toolName} tool`;
+    }
+  }
+  return { summary, toolName, direction };
+}