failproofai 0.0.10 → 0.0.11-beta.10

package/src/audit/scoring.ts

@@ -0,0 +1,174 @@
+/**
+ * Score derivation for the audit dashboard.
+ *
+ * Score is on 0-100, mapped to letter grades that anchor the leaderboard
+ * + tier prose. The thresholds match the reference design (assets/audit):
+ *
+ *     ≥ 90  S    "s tier"
+ *     ≥ 80  A    "a tier"
+ *     ≥ 71  B    "b tier"
+ *     ≥ 55  C    "c tier"
+ *     ≥ 40  D    "d tier"
+ *     <  40 F    "f tier"
+ *
+ * The "projected score" is the hypothetical score after enabling every
+ * recommended unenabled-builtin policy — used by the prescription section
+ * to motivate enabling them.
+ */
+import type { AuditResult } from "./types";
+
+export type Grade = "S" | "A" | "B" | "C" | "D" | "F";
+
+export function gradeFor(score: number): Grade {
+  if (score >= 90) return "S";
+  if (score >= 80) return "A";
+  if (score >= 71) return "B";
+  if (score >= 55) return "C";
+  if (score >= 40) return "D";
+  return "F";
+}
+
+const TIER_NAME: Record<Grade, string> = {
+  S: "s tier", A: "a tier", B: "b tier",
+  C: "c tier", D: "d tier", F: "f tier",
+};
+
+export function tierName(g: Grade): string {
+  return TIER_NAME[g];
+}
+
+/**
+ * Heuristic score on 0-100. Three properties make it *dynamic* — distinct
+ * inputs (almost) never collide on a fixed threshold the way the old
+ * hard-capped scorer did:
+ *
+ *   1. Rate-normalised. Penalty scales with the *rate* of weighted faults per
+ *      tool call (× REF_EVENTS), not the absolute count. A 1000-call session
+ *      with 25 denies scores higher than a 100-call one with 25 denies — and
+ *      heavy users no longer all pile onto the same cap.
+ *
+ *   2. Saturating, not clipped. Each severity bucket asymptotes to a cap via
+ *      `cap·(1 − e^(−p/k))` instead of `min(p, cap)`. One bad area still can't
+ *      tank the score, but the curve is *strictly monotonic*, so every extra
+ *      hit still moves the number — no two counts land on the same value.
+ *
+ *   3. Credited. Up to +5 from the derived strengths, so clean agents spread
+ *      upward instead of all sitting at 100.
+ *
+ * Per-hit weights (pre-normalisation): deny 1.2, warn/instruct 0.7,
+ * sanitize 0.4, audit detector 0.5. Bucket caps 50 / 28 / 16 / 30 (they sum
+ * past 100 so a multi-dimension reckless agent can bottom out at F).
+ *
+ * Sessions with zero scanned transcripts return 0 (no signal, no grade).
+ */
+import { MIN_EVENTS } from "./features";
+import { deriveStrengths } from "./strengths";
+
+/** Reference tool-call volume the rate is normalised against — chosen so a
+ *  moderate footprint lands around C/B and the demo anchor (~58 → C) holds.
+ *  Calibrated via __tests__/audit/scoring.test.ts. */
+const REF_EVENTS = 300;
+
+/** Concave saturator: → `cap` as `p → ∞`, strictly increasing for all p ≥ 0. */
+function saturate(p: number, cap: number, k: number): number {
+  return cap * (1 - Math.exp(-p / k));
+}
+
+/** Total weighted penalty for a set of rows, rate-normalised against `events`.
+ *  Shared by `deriveScore` and `projectedScore` so they can't diverge. */
+function penaltyFor(rows: AuditResult["results"], events: number): number {
+  const norm = REF_EVENTS / Math.max(events, MIN_EVENTS);
+  let deny = 0, instruct = 0, sanitize = 0, detector = 0;
+  for (const row of rows) {
+    if (row.source === "audit-detector") {
+      detector += row.hits * 0.5;
+      continue;
+    }
+    const sev = row.severity;
+    if (sev === "deny") deny += row.hits * 1.2;
+    else if (sev === "instruct" || sev === "warn") instruct += row.hits * 0.7;
+    else sanitize += row.hits * 0.4; // sanitize-* and any other gentle category
+  }
+  // Per-bucket (cap, k). The cap bounds any single failure mode; k sets how
+  // fast it ramps. k is small relative to the cap so penalties bite early and
+  // the score uses the full S→F range rather than clustering near 100. Caps
+  // sum past 100 on purpose, so a multi-dimension reckless agent can reach F.
+  return (
+    saturate(deny * norm, 50, 12) +
+    saturate(instruct * norm, 28, 10) +
+    saturate(sanitize * norm, 16, 8) +
+    saturate(detector * norm, 30, 14)
+  );
+}
+
+/** Positive credit (0-5) from the heuristic strengths. Deliberately small so a
+ *  reckless agent's "absence" strengths (e.g. "no credential leaks") can't
+ *  inflate it back into the top tier. */
+function creditFor(result: AuditResult): number {
+  return Math.min(deriveStrengths(result).length * 1.0, 5);
+}
+
+/** Exact (unrounded) score — used by tests to prove strict monotonicity and
+ *  the absence of threshold collisions. */
+export function deriveScoreExact(result: AuditResult): number {
+  if (result.transcripts.scanned === 0) return 0;
+  const events = result.eventsScanned ?? 0;
+  const score = 100 - penaltyFor(result.results, events) + creditFor(result);
+  return Math.max(0, Math.min(100, score));
+}
+
+export function deriveScore(result: AuditResult): number {
+  return Math.round(deriveScoreExact(result));
+}
+
+/**
+ * Projected score after enabling every unenabled builtin. Doesn't actually
+ * re-run the audit — instead it credits back the hits the user would have
+ * blocked by enabling those policies, applying the same weighted penalty
+ * scheme used by `deriveScore`.
+ *
+ * Caps at 92 so the prescription never promises a guaranteed S — the user
+ * still has to keep the policies on.
+ */
+export function projectedScore(result: AuditResult, currentScore: number): number {
+  const events = result.eventsScanned ?? 0;
+  // Penalty as-is vs. penalty with every unenabled builtin's hits removed
+  // (enabling those policies would have blocked them). The delta is what the
+  // user recovers — computed through the same saturating, rate-normalised
+  // curve as `deriveScore`, so the projection is consistent rather than a
+  // naive per-hit sum.
+  const full = penaltyFor(result.results, events);
+  const fixedRows = result.results.filter(
+    (r) => !(r.source === "builtin" && !r.enabledInConfig),
+  );
+  const fixed = penaltyFor(fixedRows, events);
+  const recoverable = Math.max(0, full - fixed);
+  // Cap at 92 so the prescription never promises a guaranteed S.
+  const proj = Math.min(92, currentScore + Math.round(recoverable));
+  return Math.max(currentScore, proj);
+}
+
+/**
+ * Approximate global rank in the cohort. We don't have a real leaderboard
+ * yet — this is a deterministic synthetic rank derived from the score so
+ * the UI doesn't feel jittery as the user re-runs.
+ *
+ * Distribution roughly matches a bell-shape centered at 60. Cohort size
+ * is fixed at 2316 to match the reference design.
+ */
+export const COHORT_SIZE = 2316;
+
+export function syntheticRank(score: number): number {
+  // Roughly: 100 → top of leaderboard, 0 → bottom. Use a smooth curve so
+  // small score changes feel meaningful but not catastrophic.
+  const percentile = scoreToPercentile(score);
+  return Math.max(1, Math.min(COHORT_SIZE, Math.round((1 - percentile) * COHORT_SIZE)));
+}
+
+function scoreToPercentile(score: number): number {
+  // Logistic mapping centered at 58 — agents below 58 fall into the long
+  // tail, agents above climb steeply. Anchors the default demo (58 → ~p20).
+  const z = (score - 58) / 14;
+  const p = 1 / (1 + Math.exp(-z));
+  return p;
+}

package/src/audit/social-proof.ts

@@ -0,0 +1,34 @@
+/**
+ * Population-level signals shown on the audit poster: how rare your
+ * archetype is and where your score sits in the cohort. Seeded with
+ * snapshot values; swap for live aggregates once that pipeline lands.
+ */
+
+import type { ArchetypeKey } from "./archetypes";
+
+/** Percentage of audited agents that map to a given archetype. */
+const ARCHETYPE_RARITY: Record<ArchetypeKey, number> = {
+  optimist: 18,
+  cowboy: 9,
+  explorer: 14,
+  goldfish: 11,
+  architect: 7,
+  precision: 16,
+  hammer: 13,
+  ghost: 12,
+};
+
+/** Map a 0-100 score to a "top N%" rank band. Bands roughly mirror the
+ *  S/A/B/C/D/F tiers in scoring.ts. */
+export function getScoreRank(score: number): string {
+  if (score >= 90) return "top 5%";
+  if (score >= 80) return "top 15%";
+  if (score >= 71) return "top 35%";
+  if (score >= 55) return "top 60%";
+  if (score >= 40) return "top 85%";
+  return "bottom tier";
+}
+
+export function getArchetypeRarityPct(key: ArchetypeKey): number {
+  return ARCHETYPE_RARITY[key];
+}

package/src/audit/strengths.ts

@@ -0,0 +1,138 @@
+/**
+ * Derive the StrengthsSection rows from a live AuditResult.
+ *
+ * The reference (assets/audit/audit.jsx) ships 5 hand-curated strengths
+ * with placeholder numbers. Here we compute each one off the actual
+ * scanned data. Output shape mirrors the original.
+ *
+ * Most strengths are "absences" — the cleaner the agent, the more
+ * strengths we surface (e.g. "0 credential leaks" only counts as a
+ * strength when no sanitize-* policies fired).
+ */
+import type { AuditResult } from "./types";
+
+export interface Strength {
+  metric: string;
+  unit: string;
+  headline: string;
+  detail: string;
+}
+
+function shortName(name: string): string {
+  const slash = name.indexOf("/");
+  return slash >= 0 ? name.slice(slash + 1) : name;
+}
+
+function hitsForShort(result: AuditResult, names: string[]): number {
+  const set = new Set(names);
+  let total = 0;
+  for (const r of result.results) {
+    if (set.has(shortName(r.name))) total += r.hits;
+  }
+  return total;
+}
+
+/** Pick up to 5 derived strengths. Each strength has a true-or-not test —
+ *  only included when the agent actually demonstrates the behavior. */
+export function deriveStrengths(result: AuditResult): Strength[] {
+  const out: Strength[] = [];
+  const events = result.eventsScanned ?? 0;
+  const totalHits = result.totals.hits;
+  const detectorsTriggered = result.results.filter((r) => r.source === "audit-detector").length;
+  const cleanRate = events > 0 ? Math.max(0, Math.min(100, Math.round(((events - totalHits) / events) * 100))) : 100;
+
+  // 1. Always show the "X tool calls, Y detectors triggered" headline.
+  if (events > 0) {
+    out.push({
+      metric: `${cleanRate}%`,
+      unit: "clean tool calls",
+      headline: `ran ${events.toLocaleString()} tool calls. ${detectorsTriggered} detector${detectorsTriggered === 1 ? "" : "s"} triggered.`,
+      detail: `${cleanRate}% of tool calls came back clean before today's audit.`,
+    });
+  }
+
+  // 2. Zero credential exposure to stdout — only when no sanitize-* and no
+  //    block-env-files / block-secrets-write / block-read-outside-cwd hits.
+  const credentialPolicies = [
+    "sanitize-api-keys", "sanitize-jwt", "sanitize-connection-strings",
+    "sanitize-private-key-content", "sanitize-bearer-tokens",
+    "block-env-files", "block-secrets-write", "block-read-outside-cwd",
+    "protect-env-vars",
+  ];
+  if (hitsForShort(result, credentialPolicies) === 0) {
+    out.push({
+      metric: "0",
+      unit: "credential leaks",
+      headline: "zero credential exposure to stdout.",
+      detail: "no env files, secret writes, or sanitize hits across the audit window.",
+    });
+  }
+
+  // 3. Average sessions task length — `events / sessions`. Faster than
+  //    median (50) is celebrated; slower than it is mentioned in findings.
+  if (result.transcripts.scanned > 0 && events > 0) {
+    const avgTurns = Math.max(1, Math.round(events / result.transcripts.scanned));
+    if (avgTurns < 30) {
+      out.push({
+        metric: String(avgTurns),
+        unit: "avg turns / session",
+        headline: `sessions complete in ${avgTurns} turns on average.`,
+        detail: avgTurns < 15
+          ? "faster than the median agent in this cohort."
+          : "comfortably within the typical session length envelope.",
+      });
+    }
+  }
+
+  // 4. No retry storms — `warn-repeated-tool-calls` + `sleep-polling-loop`
+  //    are both quiet.
+  const retryHits = hitsForShort(result, ["warn-repeated-tool-calls", "sleep-polling-loop"]);
+  if (retryHits === 0) {
+    out.push({
+      metric: "0",
+      unit: "retry storms",
+      headline: "no retry storms or polling loops detected.",
+      detail: "failed calls were diagnosed or moved on from. no six-times-in-a-row spirals.",
+    });
+  }
+
+  // 5. No production-shape git mistakes.
+  const gitHits = hitsForShort(result, [
+    "block-push-master", "block-force-push", "block-work-on-main",
+    "git-commit-no-verify",
+  ]);
+  if (gitHits === 0) {
+    out.push({
+      metric: "0",
+      unit: "push-to-main attempts",
+      headline: "kept changes off main without prompting.",
+      detail: "no direct pushes, force pushes, or hook bypasses across every session.",
+    });
+  }
+
+  // 6. No double-writes / re-reads — agent is efficient with edits.
+  const wastefulEdits = hitsForShort(result, [
+    "reread-after-edit", "prefer-edit-over-sed-awk", "prefer-write-over-heredoc",
+  ]);
+  if (wastefulEdits === 0 && events > 0) {
+    out.push({
+      metric: "0",
+      unit: "double-writes",
+      headline: "no double-writes across production projects.",
+      detail: "the agent never re-read a file it had just edited, or rewrote via shell.",
+    });
+  }
+
+  // Cap to 5. If we somehow have <2 strengths, surface a generic "no
+  // findings in this category" so the section never looks empty.
+  if (out.length < 2) {
+    out.push({
+      metric: "—",
+      unit: "audit window",
+      headline: "audit complete.",
+      detail: `${result.transcripts.scanned} session${result.transcripts.scanned === 1 ? "" : "s"} scanned across ${result.totals.projectsWithHits} project${result.totals.projectsWithHits === 1 ? "" : "s"}.`,
+    });
+  }
+
+  return out.slice(0, 5);
+}

package/src/audit/types.ts

@@ -0,0 +1,216 @@
+/**
+ * Types for the `failproofai audit` command.
+ *
+ * The audit walks past agent-CLI transcripts, replays each tool-use event
+ * through the existing policy engine, and runs each event through a separate
+ * set of audit-only detectors. Counts are aggregated per (policy|detector) and
+ * rendered as a table to stdout plus a markdown report.
+ */
+import type { IntegrationType } from "../hooks/types";
+
+/**
+ * A single tool-use event extracted from a transcript, canonicalized to the
+ * shape failproofai's policy engine + audit detectors expect.
+ */
+export interface NormalizedToolEvent {
+  cli: IntegrationType;
+  sessionId: string;
+  transcriptPath: string;
+  /** Working directory of the session at the time of the event. */
+  cwd: string;
+  /** ISO-8601 timestamp from the transcript line. */
+  timestamp: string;
+  /** Canonical Claude PascalCase tool name (Bash, Read, Edit, …). */
+  toolName: string;
+  /** Canonicalized tool input (snake_case keys for OpenCode/Pi). */
+  toolInput: Record<string, unknown>;
+  /** Pre-canonicalization tool name (e.g. "run_shell_command" for Gemini). */
+  rawToolName: string;
+  /**
+   * Result text from the matching tool_result block, if present in the
+   * transcript. Used to synthesize PostToolUse events for sanitize-* policies.
+   * Truncated to AUDIT_TOOL_RESULT_MAX_BYTES at the adapter to bound memory.
+   */
+  toolResultText?: string;
+}
+
+/** Metadata about a single transcript (one session's JSONL file). */
+export interface TranscriptMetadata {
+  cli: IntegrationType;
+  /** Encoded project folder name (e.g. "-home-user-project"). */
+  projectName: string;
+  sessionId: string;
+  transcriptPath: string;
+  /** mtime of the transcript file. Used for cache invalidation + --since filtering. */
+  mtimeMs: number;
+  /** Byte size of the transcript file. Used for cache invalidation. */
+  sizeBytes: number;
+}
+
+/** Per-session detector state. Detectors mutate this freely. */
+export type DetectorSessionState = Record<string, unknown>;
+
+/** One detected occurrence of a "stupid behavior". */
+export interface DetectorHit {
+  /** A short, one-line summary of what triggered the detector. Used as the
+   *  example column in the table and the example list in the markdown report.
+   *  Truncated to 80 chars upstream. */
+  example: string;
+}
+
+/** A pure function over a normalized event (plus optional per-session state).
+ *  Returns a DetectorHit when the event matches, null otherwise. */
+export type DetectorFn = (
+  event: NormalizedToolEvent,
+  state: DetectorSessionState,
+) => DetectorHit | null;
+
+/** Audit-only detector definition. */
+export interface Detector {
+  name: string;
+  description: string;
+  /** Display group in the markdown report (Wasteful / Risky / …). */
+  category: string;
+  /** Severity tier — drives row color in the table. */
+  severity: "warn" | "info";
+  detect: DetectorFn;
+  /** User-facing past-tense phrase, e.g. "Re-read a file just edited". Falls
+   *  back to `description` when omitted. */
+  displayTitle?: string;
+  /** One short clause describing the consequence. */
+  impact?: string;
+}
+
+/** Aggregated count for one policy or detector. */
+export interface AuditCount {
+  /** Either a policy name ("failproofai/block-force-push") or a detector name
+   *  ("redundant-cd-cwd"). */
+  name: string;
+  /** "builtin" for replayed policies, "audit-detector" for audit-only patterns. */
+  source: "builtin" | "audit-detector";
+  category: string;
+  /** Decision label for builtin replays (deny|instruct|allow|sanitize). For
+   *  audit-detector entries this is the detector's severity. */
+  severity: string;
+  hits: number;
+  /** Number of distinct projects (transcript folders) where this fired. */
+  projects: number;
+  /** ISO-8601 timestamp of the first matching event seen. */
+  firstSeen?: string;
+  /** ISO-8601 timestamp of the last matching event seen. */
+  lastSeen?: string;
+  /** Up to N example commands/snippets that triggered this (80 chars each). */
+  examples: { sessionId: string; cwd: string; timestamp: string; example: string }[];
+  /** User-facing past-tense phrase used in the report. Always populated by the
+   *  orchestrator — falls back to the policy/detector description when no
+   *  `displayTitle` was authored. */
+  displayTitle: string;
+  /** One short clause describing the consequence. May be empty for legacy
+   *  policies without authored impact copy. */
+  impact: string;
+  /** Whether the user currently has this builtin enabled. Drives the
+   *  "already protected" vs "slipping through" split in the report. Always
+   *  `false` for audit-only detectors (they don't have a runtime enforcement
+   *  path today). */
+  enabledInConfig: boolean;
+  /** Pre-built one-line install command (or audit-only notice) the report
+   *  shows next to each row, so users can copy-paste to remediate. */
+  installHint: string;
+}
+
+/** Per-transcript scan result (also the per-transcript cache value). */
+export interface TranscriptAuditResult {
+  /** Cache key: matches TranscriptMetadata.transcriptPath. */
+  transcriptPath: string;
+  cli: IntegrationType;
+  projectName: string;
+  sessionId: string;
+  mtimeMs: number;
+  sizeBytes: number;
+  /** Cwd of the session (taken from the first event with a cwd field).
+   *  Empty string when no events carried cwd. Surfaced up to `AuditResult.
+   *  projectsScanned` so the dashboard's project filter can show every
+   *  scanned project, not just those with examples. */
+  cwd?: string;
+  /** Total normalized tool-use events scanned in this transcript. Surfaced
+   *  via `AuditResult.eventsScanned` so the report can show "X tool calls"
+   *  across the whole audit. */
+  eventsScanned?: number;
+  /** Per-policy/detector hit count for this one transcript. */
+  hitsByName: Record<string, number>;
+  /** Up to 3 example commands per policy/detector (later coalesced upstream). */
+  examplesByName: Record<string, { timestamp: string; cwd: string; example: string }[]>;
+  /** First/last timestamp per name. */
+  rangeByName: Record<string, { first: string; last: string }>;
+}
+
+/** Top-level result of `runAudit()`. */
+export interface AuditResult {
+  /** Schema version of this JSON shape. Increment on incompatible changes.
+   *  v2: added `projectsScanned`. */
+  version: number;
+  scannedAt: string;
+  scope: {
+    cli: IntegrationType[];
+    projects: string[] | "all";
+    since: string | null;
+  };
+  transcripts: {
+    scanned: number;
+    skipped: number;
+    errors: number;
+    durationMs: number;
+  };
+  results: AuditCount[];
+  totals: {
+    hits: number;
+    projectsWithHits: number;
+  };
+  /** Sorted, deduped list of cwds across every transcript that was scanned
+   *  (including those with zero hits). Drives the dashboard's project filter.
+   *  Transcripts without a usable cwd are excluded. */
+  projectsScanned: string[];
+  /** Total normalized tool-use events the audit walked across every
+   *  scanned transcript. The audit dashboard surfaces this as the
+   *  "X tool calls" headline counter. */
+  eventsScanned: number;
+  /** Short names (without `failproofai/` namespace) of every builtin
+   *  policy that was enabled in the user's merged config at scan time.
+   *  Lets the dashboard answer "is this policy already on?" for
+   *  detector-mapped policies that may not have hit during this audit. */
+  enabledBuiltinNames: string[];
+}
+
+/** CLI-supplied options for `runAudit()`. Set by `bin/failproofai.mjs`. */
+export interface RunAuditOptions {
+  /** Restrict to one or more CLIs. Default: all 7. */
+  clis?: IntegrationType[];
+  /** Restrict to sessions whose cwd matches one of these paths. */
+  projects?: string[];
+  /** "7d", "30d", or an ISO date. Filters on transcript mtime. */
+  since?: string;
+  /** Restrict to one or more policy/detector names. */
+  policies?: string[];
+  /** Top-N rows in the table output. */
+  limit?: number;
+  /** Include example column in the table. */
+  showExamples?: boolean;
+  /** Output path for the markdown report. Default: ./failproofai-audit.md */
+  reportPath?: string;
+  /** Skip writing the markdown report. */
+  noReport?: boolean;
+  /** Emit JSON to stdout instead of the table. */
+  json?: boolean;
+  /** Bypass the per-transcript cache. */
+  noCache?: boolean;
+}
+
+/** Truncation budget for `NormalizedToolEvent.toolResultText`. Bounds memory
+ *  for sanitize-* policy replay without losing the typical match window. */
+export const AUDIT_TOOL_RESULT_MAX_BYTES = 64 * 1024;
+
+/** Truncation budget for example strings shown in table + markdown. */
+export const AUDIT_EXAMPLE_MAX_CHARS = 80;
+
+/** Max examples kept per policy/detector in the final report. */
+export const AUDIT_MAX_EXAMPLES_PER_NAME = 3;