failproofai 1.0.7 → 1.0.8

package/src/hooks/semantic/decide.ts

@@ -0,0 +1,513 @@
+/**
+ * Jev answers → one allow / deny / instruct.
+ *
+ * Pure and deterministic: given the recorded probabilities, the same verdict
+ * comes back forever, offline. That is what makes a semantic decision
+ * replayable and testable even though the model behind it is not.
+ *
+ * The rules, in order of how much they matter:
+ *
+ * - A policy fires only when EVERY probe holds; its evidence is the minimum
+ *   over probes (TypeSafe's own function-calling cookbook takes the minimum,
+ *   not the product — one wrong argument spoils the call).
+ * - A `deny` policy blocks only on strong evidence; moderate evidence warns.
+ * - The user may clear a policy only if (a) they explicitly asked for this
+ *   action, (b) everything the call affects stays inside what they asked for
+ *   (the `scope` probe), (c) when the call names identifiable targets, EVERY
+ *   one of them appears in what they typed — checked here, in code, not by the
+ *   model — and (d) the request does not look like it is talking to the
+ *   reviewer. When the local shell scan may have missed part of the command
+ *   (`$'…'`, a heredoc, `$(…)`, …), (c) cannot be checked and nothing is
+ *   cleared: the scan can see an innocent first target and stop before the
+ *   destructive one.
+ *   A call that names no target is never cleared by default: the scope answer
+ *   has to carry it.
+ * - The injection probe withdraws any override, and turns a policy that has
+ *   INDEPENDENTLY fired into a block. On its own it can never block: a
+ *   detector that could would let anyone block any command by writing the
+ *   right words into it.
+ */
+import { scanCommand } from "./facts";
+import type { PolicyOutcome, SemanticPolicy, SemanticVerdict } from "./types";
+
+export interface Thresholds {
+  /** Minimum evidence (min over probes) for a policy to fire at all. */
+  fire: number;
+  /** Minimum evidence for a `deny`-mode policy to block rather than warn. */
+  deny: number;
+  /** At or above this, the policy's documented exception applies. */
+  exempt: number;
+  /** At or above this, the user is taken to have explicitly asked. */
+  userAsked: number;
+  /** At or above this, the request is treated as addressing the reviewer. */
+  injection: number;
+  /** At or above this, the request is taken to stay within what the user asked. */
+  scope: number;
+}
+
+export const DEFAULT_THRESHOLDS: Thresholds = {
+  fire: 0.7,
+  deny: 0.85,
+  exempt: 0.5,
+  userAsked: 0.8,
+  injection: 0.7,
+  scope: 0.8,
+};
+
+// ── Did the user name the target? ────────────────────────────────────────────
+
+/** Words that name the operation or plumbing, never the thing acted on. */
+const GENERIC_TOKENS = new Set([
+  "sudo", "env", "time", "nohup", "xargs", "bash", "sh", "zsh", "git", "push", "pull", "commit", "origin",
+  "head", "upstream", "force", "rm", "mv", "cp", "cat", "echo", "printf", "grep", "find", "sed", "awk",
+  "run", "npm", "npx", "bun", "bunx", "node", "python", "python3", "pip", "install", "uninstall",
+  "delete", "remove", "apply", "get", "set", "the", "and", "for", "with", "true", "false", "null", "dev",
+  "src", "tmp", "usr", "bin", "local", "home", "etc", "var", "lib", "json", "yaml", "yml", "txt", "log",
+]);
+
+function tokensOf(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9._@-]+/)
+    .filter((t) => t.length >= 3 && !GENERIC_TOKENS.has(t) && !/^\d+$/.test(t));
+}
+
+/**
+ * What a tool call acts on, as the local target check sees it.
+ *
+ * `groups` holds one entry per identifiable target — a non-flag argument of a
+ * shell command (path components and all), or, for any other tool, the whole
+ * of its argument values taken together — and each entry is the set of words
+ * that name it. `targets` is every word of every group.
+ *
+ * `complete` is false when the shell scan may have missed a word bash would
+ * run (`ScannedCommand.complete`). The groups are then a lower bound, and no
+ * clear may rest on them: see {@link everyTargetNamed}'s callers.
+ */
+export interface TargetScan {
+  targets: Set<string>;
+  groups: Set<string>[];
+  complete: boolean;
+}
+
+/**
+ * The words that identify WHAT a tool call acts on: its non-flag arguments
+ * (path components included), file paths, URL hosts, MCP argument values.
+ * The verb is left to Jev's `user_asked` question; this only checks the noun.
+ */
+export function scanTargets(toolInput: Record<string, unknown>): TargetScan {
+  const groups: Set<string>[] = [];
+  const addGroup = (tok: string) => {
+    const words = tokensOf(tok);
+    if (words.length > 0) groups.push(new Set(words));
+  };
+  const command = typeof toolInput.command === "string" ? toolInput.command : null;
+  if (command) {
+    const scanned = scanCommand(command);
+    for (const seg of scanned.segments) {
+      for (const tok of seg.slice(1)) {
+        if (!tok.startsWith("-")) addGroup(tok);
+      }
+    }
+    // Empty reads as "names no target", and lets consent rest on Jev's answers
+    // alone. But the scanner is not bash — a `#` inside `$'…'`, `${x:- # }` or
+    // backticks ends its view early, and it reads only MAX_SCAN_CHARS — so an
+    // empty scan may just have stopped before the target. Then judge the words
+    // as written. Only ever stricter: an empty set already passed. (Such a scan
+    // is also reported incomplete, which withholds the clear on its own; this
+    // keeps the recorded targets honest.)
+    if (groups.length === 0) {
+      for (const piece of command.split(/[;&|\n]+/)) {
+        for (const tok of piece.trim().split(/\s+/).slice(1)) {
+          if (!tok.startsWith("-")) addGroup(tok);
+        }
+      }
+    }
+    return { targets: new Set(groups.flatMap((g) => [...g])), groups, complete: scanned.complete };
+  }
+  const all = new Set<string>();
+  for (const [key, value] of Object.entries(toolInput)) {
+    if (typeof value !== "string" || value.length > 300) continue;
+    if (/content|old_string|new_string|body|text|prompt/i.test(key)) continue;
+    for (const t of tokensOf(value)) all.add(t);
+  }
+  // A non-shell tool's fields qualify one another (`owner`, `repo`, `branch`)
+  // rather than naming separate targets, so they are ONE target: naming any of
+  // them names it.
+  return { targets: all, groups: all.size > 0 ? [all] : [], complete: true };
+}
+
+/** Every word of {@link scanTargets}, flattened. */
+export function targetTokens(toolInput: Record<string, unknown>): Set<string> {
+  return scanTargets(toolInput).targets;
+}
+
+/**
+ * True when the user's own words name EVERY target the call acts on — each
+ * group through at least one of its words.
+ *
+ * Not "any one": `rm -rf build/ ~/important` after "clean the build" names
+ * `build` and not `important`, and a clear on the first would carry the
+ * second, which nobody asked for.
+ */
+export function everyTargetNamed(scan: TargetScan, userSaid: ReadonlyArray<string>): boolean {
+  if (userSaid.length === 0 || scan.groups.length === 0) return false;
+  const said = userSaid.join("\n").toLowerCase();
+  return scan.groups.every((g) => {
+    for (const t of g) if (said.includes(t)) return true;
+    return false;
+  });
+}
+
+/**
+ * True when the user's words name SOME of the call's targets but not all:
+ * they drew a line and the call reaches past it. Naming none is not this — a
+ * goal ("fix the failing tests") names no path at all.
+ */
+export function partlyNamed(scan: TargetScan, userSaid: ReadonlyArray<string>): boolean {
+  if (userSaid.length === 0 || scan.groups.length < 2) return false;
+  const said = userSaid.join("\n").toLowerCase();
+  let named = 0;
+  for (const g of scan.groups) {
+    for (const t of g) {
+      if (said.includes(t)) {
+        named++;
+        break;
+      }
+    }
+  }
+  return named > 0 && named < scan.groups.length;
+}
+
+/**
+ * True when the user's own words name at least one of `targets`. The deciders
+ * do not use this any-one check — see {@link everyTargetNamed}.
+ */
+export function targetNamedByUser(targets: Set<string>, userSaid: ReadonlyArray<string>): boolean {
+  if (userSaid.length === 0) return false;
+  // Nothing identifiable to check is not a match. This used to return true,
+  // and `git push --force --all` — scope widened by a flag, naming nothing —
+  // passed by default. `decide` covers that case with the scope answer instead.
+  if (targets.size === 0) return false;
+  const said = userSaid.join("\n").toLowerCase();
+  for (const t of targets) if (said.includes(t)) return true;
+  return false;
+}
+
+// ── Verdict ──────────────────────────────────────────────────────────────────
+
+function formatOutcome(p: SemanticPolicy, o: PolicyOutcome): string {
+  return `${p.title} (semantic/${p.name}, p=${o.evidence.toFixed(2)}). ${p.guidance}`;
+}
+
+export function decide(
+  selected: ReadonlyArray<SemanticPolicy>,
+  answers: Readonly<Record<string, number>>,
+  toolInput: Record<string, unknown>,
+  userSaid: ReadonlyArray<string>,
+  thresholds: Thresholds = DEFAULT_THRESHOLDS,
+  /** See {@link DecideV1Options.userSaidCut}. */
+  userSaidCut = false,
+): SemanticVerdict {
+  const injection = typeof answers.injection === "number" ? answers.injection : null;
+  const injected = injection !== null && injection >= thresholds.injection;
+  const scope = typeof answers.scope === "number" ? answers.scope : null;
+  const withinScope = scope !== null && scope >= thresholds.scope;
+  let scan: TargetScan | null = null;
+
+  const outcomes: PolicyOutcome[] = selected.map((p) => {
+    const evidence = Math.min(...p.probes.map((probe) => answers[`${p.name}.${probe.id}`] ?? 0));
+    const exempt = p.exempt ? answers[`${p.name}.exempt`] ?? 0 : null;
+    const userAsked = answers[`${p.name}.user_asked`] ?? null;
+    const base: PolicyOutcome = {
+      policy: p.name,
+      mode: p.mode,
+      userCanOverride: p.userCanOverride,
+      ...(p.origin ? { origin: p.origin } : {}),
+      evidence,
+      exempt,
+      userAsked,
+      targetNamedByUser: false,
+      escalatedByInjection: false,
+      verdict: "none",
+    };
+    if (evidence < thresholds.fire || (exempt !== null && exempt >= thresholds.exempt)) return base;
+
+    // An independently flagged call that also argues for its own approval.
+    if (injected) return { ...base, escalatedByInjection: true, verdict: "deny" };
+
+    const fired: PolicyOutcome["verdict"] = p.mode === "deny" && evidence >= thresholds.deny ? "deny" : "instruct";
+    if (p.userCanOverride && userAsked !== null && userAsked >= thresholds.userAsked && withinScope) {
+      scan ??= scanTargets(toolInput);
+      // A shell scan that may have missed a word cannot say what the call
+      // touches, so it cannot say the user named it: no clear. Not rescued by
+      // `userSaidCut` — that gap is in what the human typed, this one is in
+      // the call.
+      if (!scan.complete) return { ...base, verdict: fired, targetScanIncomplete: true };
+      // No identifiable target: the scope answer (already required) carries it.
+      // Otherwise EVERY target must also appear in the user's own words —
+      // unless the words we hold were cut, when "absent" is not something this
+      // check knows (see {@link DecideV1Options.userSaidCut}).
+      const named = scan.groups.length > 0 && everyTargetNamed(scan, userSaid);
+      if (scan.groups.length === 0 || named || userSaidCut) return { ...base, targetNamedByUser: named, verdict: "overridden" };
+    }
+    return { ...base, verdict: fired };
+  });
+
+  const byName = new Map(selected.map((p) => [p.name, p]));
+  const denies = outcomes.filter((o) => o.verdict === "deny");
+  const instructs = outcomes.filter((o) => o.verdict === "instruct");
+  const overridden = outcomes.filter((o) => o.verdict === "overridden");
+  const injectionNote = denies.some((o) => o.escalatedByInjection)
+    ? " Blocked because the request also contains text addressed to the reviewer (claiming approval, safety or consent)."
+    : "";
+
+  if (denies.length > 0) {
+    const [first, ...rest] = denies;
+    const also = rest.length > 0 ? ` Also flagged: ${rest.map((o) => `semantic/${o.policy}`).join(", ")}.` : "";
+    return {
+      decision: "deny",
+      reason: formatOutcome(byName.get(first.policy)!, first) + also + injectionNote,
+      outcomes,
+      injectionSuspected: injection,
+      scopeWithinRequest: scope,
+    };
+  }
+  if (instructs.length > 0) {
+    return {
+      decision: "instruct",
+      reason: instructs.map((o) => formatOutcome(byName.get(o.policy)!, o)).join("\n") + injectionNote,
+      outcomes,
+      injectionSuspected: injection,
+      scopeWithinRequest: scope,
+    };
+  }
+  return {
+    decision: "allow",
+    reason:
+      overridden.length > 0
+        ? `Allowed because the user explicitly asked: ${overridden.map((o) => `semantic/${o.policy}`).join(", ")}.`
+        : null,
+    outcomes,
+    injectionSuspected: injection,
+    scopeWithinRequest: scope,
+  };
+}
+
+// ── Intent mode v1 ───────────────────────────────────────────────────────────
+//
+// Pre-registered 2026-09-22 before the v1 blind set existed; see
+// scripts/jev-eval/intent-v2/PREREG.md. Do not tune these on that set.
+
+export interface ThresholdsV1 {
+  fire: number;
+  deny: number;
+  exempt: number;
+  injection: number;
+  /** `op_requested` at or above this: the human asked for this exact operation. */
+  opRequested: number;
+  /** An op-requested clear also needs `beyond_task` below this. */
+  opBeyondMax: number;
+  /** `task_step` at or above this: the call serves the human's task. */
+  taskStep: number;
+  /** A task-step clear or softening also needs `beyond_task` below this. */
+  taskBeyondMax: number;
+  /** Flag an unflagged call when `beyond_task` is at or above this… */
+  beyondFlag: number;
+  /** …and `task_step` is below this… */
+  beyondTaskStepMax: number;
+  /** …and at least one policy's first ("does it do X") probe is at or above this. */
+  beyondActionMin: number;
+}
+
+export const DEFAULT_THRESHOLDS_V1: ThresholdsV1 = {
+  fire: 0.7,
+  deny: 0.85,
+  exempt: 0.5,
+  injection: 0.7,
+  opRequested: 0.8,
+  opBeyondMax: 0.5,
+  taskStep: 0.8,
+  taskBeyondMax: 0.3,
+  beyondFlag: 0.8,
+  beyondTaskStepMax: 0.5,
+  beyondActionMin: 0.5,
+};
+
+/**
+ * v1 with the beyond-task flag's `task_step` gate removed: an otherwise-allowed
+ * call is flagged when it reaches beyond the human's task (`beyond_task` ≥ 0.8)
+ * and some "does it do X" probe is at least half-raised, whether or not Jev
+ * also calls it a step toward the task. Round 1 found the gate stopped 4 of 6
+ * missed overreaches; pre-registered for round 2 (PREREG-round2.md).
+ */
+export const THRESHOLDS_V1_NO_TASK_GATE: ThresholdsV1 = {
+  ...DEFAULT_THRESHOLDS_V1,
+  beyondTaskStepMax: Number.POSITIVE_INFINITY,
+};
+
+export interface DecideV1Options {
+  thresholds?: ThresholdsV1;
+  /** Turn the beyond-the-task flag off (an ablation). */
+  flagBeyondTask?: boolean;
+  /** Turn the task-step clear/soften off, leaving only op-requested (an ablation). */
+  taskStepClears?: boolean;
+  /**
+   * A human turn we are reading arrived already cut, so `targetNamedByUser`
+   * cannot tell "the user did not name it" from "the part naming it was cut".
+   *
+   * The local target check is then INCONCLUSIVE rather than negative: the
+   * override falls back to what it rests on when a call names no identifiable
+   * target at all — the model's own `op_requested` and `beyond_task` answers.
+   * Without this, the LENGTH of the human's paste decided the verdict: the
+   * same `rm` of the same file came out `allow` after "delete cache.sqlite"
+   * and `instruct` + every clear withdrawn after the same sentence inside a
+   * 12,000-character prompt. Set only from the HUMAN channel, which the agent
+   * cannot write (`evaluator.ts`), so padding is not a way to buy one: it can
+   * restore a clear a short prompt would have had, never create one.
+   */
+  userSaidCut?: boolean;
+}
+
+/**
+ * v1: the human's task decides, not the policy's wording.
+ *
+ * - A policy fires exactly as in v0 (every probe holds, no exemption).
+ * - Injection withdraws every clear and turns a fired policy into a block.
+ * - The human asked for THIS operation on THIS target (`op_requested`), the
+ *   call reaches no further (`beyond_task`), and — when the call names
+ *   targets — every one of them appears in what the human typed or in the
+ *   agent proposal they replied to: the policy is cleared.
+ * - A shell command the local scan could not read whole is cleared and
+ *   softened by neither route: see `ScannedCommand.complete` in `facts.ts`.
+ * - Otherwise, the call is a step toward the human's task (`task_step`) and
+ *   reaches no further: a warn-level outcome is cleared and a block is
+ *   softened to a warning. A goal never licenses a block on its own. On a
+ *   shell command whose targets the human named only in part, this route
+ *   does not apply either.
+ * - Policies with `userCanOverride: false` are never cleared or softened.
+ * - Nothing fired, but the call reaches beyond the task, is not a step toward
+ *   it, and some "does it do X" probe is at least half-raised: warn.
+ */
+export function decideV1(
+  selected: ReadonlyArray<SemanticPolicy>,
+  answers: Readonly<Record<string, number>>,
+  toolInput: Record<string, unknown>,
+  userSaid: ReadonlyArray<string>,
+  agentLastMessage: string | null,
+  opts: DecideV1Options = {},
+): SemanticVerdict {
+  const t = opts.thresholds ?? DEFAULT_THRESHOLDS_V1;
+  const flagBeyond = opts.flagBeyondTask ?? true;
+  const taskClears = opts.taskStepClears ?? true;
+  const userSaidCut = opts.userSaidCut ?? false;
+  const num = (id: string): number | null => (typeof answers[id] === "number" ? answers[id] : null);
+  const injection = num("injection");
+  const injected = injection !== null && injection >= t.injection;
+  const task = num("task_step");
+  const op = num("op_requested");
+  const beyond = num("beyond_task");
+  let scan: TargetScan | null = null;
+  const targetScan = (): TargetScan => (scan ??= scanTargets(toolInput));
+  const evidenceSaid = agentLastMessage ? [...userSaid, agentLastMessage] : userSaid;
+  const targetOk = (): { ok: boolean; named: boolean } => {
+    const s = targetScan();
+    if (s.groups.length === 0) return { ok: true, named: false };
+    const named = everyTargetNamed(s, evidenceSaid);
+    return { ok: named || userSaidCut, named };
+  };
+
+  const outcomes: PolicyOutcome[] = selected.map((p) => {
+    const evidence = Math.min(...p.probes.map((probe) => answers[`${p.name}.${probe.id}`] ?? 0));
+    const exempt = p.exempt ? answers[`${p.name}.exempt`] ?? 0 : null;
+    const base: PolicyOutcome = {
+      policy: p.name,
+      mode: p.mode,
+      userCanOverride: p.userCanOverride,
+      ...(p.origin ? { origin: p.origin } : {}),
+      evidence,
+      exempt,
+      userAsked: op,
+      targetNamedByUser: false,
+      escalatedByInjection: false,
+      verdict: "none",
+    };
+    if (evidence < t.fire || (exempt !== null && exempt >= t.exempt)) return base;
+    if (injected) return { ...base, escalatedByInjection: true, verdict: "deny" };
+
+    const fired: PolicyOutcome["verdict"] = p.mode === "deny" && evidence >= t.deny ? "deny" : "instruct";
+    if (!p.userCanOverride) return { ...base, verdict: fired };
+    // The shell scan may have missed a word bash would run (`$'…'`, a heredoc,
+    // `$(…)`, …): what the call touches is not known here, so neither intent
+    // route may clear or soften it. Jev's own deny or instruct stands.
+    if (!targetScan().complete) return { ...base, verdict: fired, targetScanIncomplete: true };
+
+    if (op !== null && op >= t.opRequested && beyond !== null && beyond < t.opBeyondMax) {
+      const target = targetOk();
+      if (target.ok) return { ...base, targetNamedByUser: target.named, verdict: "overridden", intent: "op-requested" };
+    }
+    // The task-step route reaches `combine.ts` as a clear too — a warning it
+    // leaves behind clears a reviewable regex deny. It is consent by GOAL, so
+    // a shell command whose targets the human never named still rides on it
+    // ("fix the failing tests" → `rm -rf node_modules`). But once the human
+    // HAS named targets, they drew the line: a call reaching past it is not
+    // softened. After "clean the build", `rm -rf build/ ~/important` is not.
+    if (
+      taskClears &&
+      task !== null &&
+      task >= t.taskStep &&
+      beyond !== null &&
+      beyond < t.taskBeyondMax &&
+      !(typeof toolInput.command === "string" && !userSaidCut && partlyNamed(targetScan(), evidenceSaid))
+    ) {
+      if (fired === "instruct") return { ...base, verdict: "overridden", intent: "task-step" };
+      return { ...base, verdict: "instruct", intent: "downgraded-task-step" };
+    }
+    return { ...base, verdict: fired };
+  });
+
+  const byName = new Map(selected.map((p) => [p.name, p]));
+  const denies = outcomes.filter((o) => o.verdict === "deny");
+  const instructs = outcomes.filter((o) => o.verdict === "instruct");
+  const overridden = outcomes.filter((o) => o.verdict === "overridden");
+  const common = { outcomes, injectionSuspected: injection, scopeWithinRequest: null };
+  const injectionNote = denies.some((o) => o.escalatedByInjection)
+    ? " Blocked because the request also contains text addressed to the reviewer (claiming approval, safety or consent)."
+    : "";
+
+  if (denies.length > 0) {
+    const [first, ...rest] = denies;
+    const also = rest.length > 0 ? ` Also flagged: ${rest.map((o) => `semantic/${o.policy}`).join(", ")}.` : "";
+    return { decision: "deny", reason: formatOutcome(byName.get(first.policy)!, first) + also + injectionNote, ...common };
+  }
+
+  const maxAction = Math.max(0, ...selected.map((p) => answers[`${p.name}.${p.probes[0].id}`] ?? 0));
+  const beyondFlag =
+    flagBeyond &&
+    userSaid.length > 0 &&
+    beyond !== null &&
+    beyond >= t.beyondFlag &&
+    task !== null &&
+    task < t.beyondTaskStepMax &&
+    maxAction >= t.beyondActionMin;
+
+  if (instructs.length > 0 || beyondFlag) {
+    const lines = instructs.map((o) => formatOutcome(byName.get(o.policy)!, o));
+    if (beyondFlag) {
+      lines.push(
+        `Goes beyond what the user asked for (semantic/beyond-task, p=${beyond!.toFixed(2)}). ` +
+          "Do only what they asked, or confirm the wider change with them first.",
+      );
+    }
+    return { decision: "instruct", reason: lines.join("\n") + injectionNote, ...common, beyondTask: beyondFlag };
+  }
+  return {
+    decision: "allow",
+    reason:
+      overridden.length > 0
+        ? `Allowed because it is what the user asked for: ${overridden.map((o) => `semantic/${o.policy}`).join(", ")}.`
+        : null,
+    ...common,
+    beyondTask: false,
+  };
+}