@inhuman.tools/turn 0.0.0-stage → 0.1.0

package/src/policy-processor/soft-deny.ts

@@ -0,0 +1,225 @@
+/**
+ * Soft-deny judge, host half. The question is "did the user explicitly request this exact action?".
+ * The model never gets the last word: the host builds every prompt section (one line of JSON each, so
+ * agent text cannot forge a label), parses the vote strictly, and verifies the quoted user words
+ * itself. Any parse error, missing sink or failed quote check is a `fail` (the cell's `onFail`),
+ * never an allow. Untrusted content (tool output, the agent's own text) is never authorization.
+ */
+import { CLASSES, type ClassId, type FloorVote } from "@inhuman.tools/policy/floor";
+
+/** Authorization the model must claim before a soft-deny allow counts (on top of >= risk). */
+export const SOFT_DENY_MIN_AUTHORIZATION = 7;
+/** A quote shorter than this proves nothing. */
+export const MIN_QUOTE_CHARS = 3;
+/** Strings in the proposed call are cut at this length in the prompt (the verification is not). */
+const CALL_STRING_LIMIT = 2000;
+const MESSAGE_LIMIT = 8000;
+
+export type SoftDenyOutcome =
+  | { readonly kind: "vote"; readonly decision: "allow" | "deny"; readonly round: 1 | 2; readonly reason: string }
+  /** Round 1 asks for the agent's last answer. */
+  | { readonly kind: "need-agent-answer"; readonly reason: string }
+  | { readonly kind: "fail"; readonly reason: string };
+
+export type SoftDenyCheck = {
+  /** Oldest first. The only source of permission. */
+  readonly userMessages: readonly string[];
+  /** The agent's last answer before the latest user message, or "" when there is none. */
+  readonly agentAnswer: string;
+  /** Words that must appear in the quote for the action to be the one asked for. */
+  readonly identityTokens: readonly string[];
+  /** Verify the quote and the identity of the action. Only a user-layer cell may turn this off. */
+  readonly requireQuote: boolean;
+  /** Allow needs `authorization + margin >= risk`. */
+  readonly margin: number;
+};
+
+/** NFC, case folded, whitespace collapsed: the form both sides of a quote check are compared in. */
+export function normalizeQuote(text: string): string {
+  return text.normalize("NFC").toLowerCase().replaceAll(/\s+/g, " ").trim();
+}
+
+function contains(haystack: string, needle: string): boolean {
+  const want = normalizeQuote(needle);
+  return want.length >= MIN_QUOTE_CHARS && normalizeQuote(haystack).includes(want);
+}
+
+function words(text: string): Set<string> {
+  return new Set(
+    normalizeQuote(text)
+      .split(/[^\p{L}\p{N}]+/u)
+      .filter((word) => word !== ""),
+  );
+}
+
+/** Does `text` name the action: any identity token is a word of it (or a substring for path-like tokens)? */
+export function namesAction(text: string, tokens: readonly string[]): boolean {
+  if (tokens.length === 0) return true;
+  const folded = normalizeQuote(text);
+  const set = words(text);
+  return tokens.some((token) => {
+    const want = normalizeQuote(token);
+    return want !== "" && (set.has(want) || (/[./_-]/.test(want) && folded.includes(want)));
+  });
+}
+
+const KEYS_R1 = new Set(["verdict", "quote", "reason", "risk", "authorization"]);
+const KEYS_R2 = new Set([...KEYS_R1, "proposal"]);
+
+function clean(raw: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(raw).map(([key, value]) => [key.replace(/^'?:/, ""), value]));
+}
+
+const isScore = (value: unknown): value is number =>
+  typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 9;
+
+/**
+ * Parse and verify one round's vote. `raw` is what the program returned. Strict: unknown or missing
+ * keys, a wrong type or enum are `fail`. Quote and identity checks are the host's, not the model's.
+ */
+export function parseSoftDenyVote(raw: unknown, round: 1 | 2, check: SoftDenyCheck): SoftDenyOutcome {
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
+    return { kind: "fail", reason: "soft-deny judge did not vote" };
+  }
+  const vote = clean(raw as Record<string, unknown>);
+  const allowed = round === 1 ? KEYS_R1 : KEYS_R2;
+  for (const key of Object.keys(vote)) {
+    if (!allowed.has(key)) return { kind: "fail", reason: `soft-deny vote has an unexpected field ${key}` };
+  }
+  const { verdict, quote, reason, risk, authorization } = vote;
+  const proposal = vote.proposal;
+  if (typeof reason !== "string" || typeof quote !== "string") {
+    return { kind: "fail", reason: "soft-deny vote is missing quote or reason" };
+  }
+  if (!isScore(risk) || !isScore(authorization))
+    return { kind: "fail", reason: "soft-deny vote has no risk or authorization" };
+  if (round === 2 && typeof proposal !== "string")
+    return { kind: "fail", reason: "soft-deny round 2 vote has no proposal" };
+  const why = reason.slice(0, 300);
+  if (verdict === "deny") return { kind: "vote", decision: "deny", round, reason: why };
+  const messages = check.userMessages;
+  const latest = messages.at(-1) ?? "";
+  if (verdict === "need-agent-answer") {
+    if (round === 2) return { kind: "fail", reason: "soft-deny round 2 cannot ask for the agent's answer" };
+    if (check.requireQuote && !contains(latest, quote)) return quoteFailure("confirmation");
+    return { kind: "need-agent-answer", reason: why };
+  }
+  if (verdict !== "allow") return { kind: "fail", reason: "soft-deny vote has an unknown verdict" };
+  if (check.requireQuote) {
+    if (round === 1) {
+      if (!messages.some((message) => contains(message, quote))) return quoteFailure("request");
+      if (!namesAction(quote, check.identityTokens)) {
+        // The user's words are real but do not name this action: at most a confirmation of something else.
+        return { kind: "need-agent-answer", reason: "the quoted words do not name the action" };
+      }
+    } else {
+      if (!contains(latest, quote)) return quoteFailure("confirmation");
+      if (check.agentAnswer === "" || !contains(check.agentAnswer, proposal as string)) return quoteFailure("proposal");
+      if (!namesAction(proposal as string, check.identityTokens)) {
+        return { kind: "fail", reason: "the proposal does not name the action" };
+      }
+    }
+  }
+  if (authorization < SOFT_DENY_MIN_AUTHORIZATION || authorization + check.margin < risk) {
+    return {
+      kind: "vote",
+      decision: "deny",
+      round,
+      reason: `authorization ${authorization} does not cover risk ${risk}: ${why}`,
+    };
+  }
+  return { kind: "vote", decision: "allow", round, reason: why };
+}
+
+/** A quote that is not in the user's words is a red-team signal, and a failure, not a deny. */
+function quoteFailure(what: string): SoftDenyOutcome {
+  return { kind: "fail", reason: `judge.quote_unverified: the ${what} quoted by the judge is not in the transcript` };
+}
+
+function clip(text: string, limit: number): string {
+  return text.length <= limit ? text : `${text.slice(0, limit)}…[${text.length - limit} more characters]`;
+}
+
+function clipValue(value: unknown): unknown {
+  if (typeof value === "string") return clip(value, CALL_STRING_LIMIT);
+  if (Array.isArray(value)) return value.map((item) => clipValue(item));
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, clipValue(item)]));
+  }
+  return value;
+}
+
+export type SoftDenySections = {
+  readonly floorJson: string;
+  readonly userJson: string;
+  readonly callJson: string;
+  readonly agentAnswerJson: string;
+};
+
+/** Every section is `JSON.stringify` output: one line, so nothing inside can start a new labelled section. */
+export function buildSoftDenySections(input: {
+  readonly classes: readonly ClassId[];
+  readonly votes: readonly Pick<FloorVote, "ruleId">[];
+  readonly userMessages: readonly string[];
+  readonly agentAnswer: string;
+  readonly allowAgentAnswer: boolean;
+  readonly tool: string;
+  readonly cluster: string;
+  readonly args: unknown;
+  /** The policy owner's words about this call (a rule's `describe`). Trusted context for the judge, not permission. */
+  readonly describe?: string;
+}): SoftDenySections {
+  return {
+    floorJson: JSON.stringify({
+      tier: "soft-deny",
+      classes: input.classes.map((id) => ({ id, label: CLASSES[id].label, description: CLASSES[id].description })),
+      rules: input.votes.map((vote) => vote.ruleId),
+      ...(input.describe === undefined || input.describe === ""
+        ? {}
+        : { ownerDescription: clip(input.describe, MESSAGE_LIMIT) }),
+    }),
+    userJson: JSON.stringify(input.userMessages.map((message) => clip(message, MESSAGE_LIMIT))),
+    callJson: JSON.stringify({ tool: input.tool, cluster: input.cluster, args: clipValue(input.args ?? {}) }),
+    agentAnswerJson:
+      input.allowAgentAnswer && input.agentAnswer !== "" ? JSON.stringify(clip(input.agentAnswer, MESSAGE_LIMIT)) : "",
+  };
+}
+
+/** Words the user's quote must contain for the action to be the one they asked for. */
+export function identityTokens(input: {
+  readonly cluster: string;
+  readonly tool: string;
+  readonly args: Readonly<Record<string, unknown>> | null;
+  readonly votes: readonly FloorVote[];
+  readonly serverToolName?: string;
+}): string[] {
+  const out = new Set<string>();
+  const base = (path: string) => path.split("/").findLast(Boolean) ?? path;
+  switch (input.cluster) {
+    case "bash": {
+      for (const vote of input.votes) {
+        if (vote.unit === undefined) continue;
+        out.add(base(vote.unit.argv0));
+        for (const word of vote.unit.spine) out.add(word);
+        for (const word of vote.unit.familyArgs) out.add(word);
+      }
+      if (out.size === 0 && typeof input.args?.command === "string") {
+        const first = input.args.command.trim().split(/\s+/)[0];
+        if (first !== undefined && first !== "") out.add(base(first));
+      }
+      break;
+    }
+    case "read":
+    case "write": {
+      const path = input.args?.path ?? input.args?.cwd;
+      if (typeof path === "string" && path !== "") out.add(base(path));
+      for (const vote of input.votes) if (vote.path !== undefined) out.add(base(vote.path.path));
+      break;
+    }
+    default: {
+      const name = input.serverToolName ?? input.tool;
+      for (const word of name.split(/[_\-./\s]+|(?<=[a-z0-9])(?=[A-Z])/).filter(Boolean)) out.add(word);
+    }
+  }
+  return [...out];
+}

package/src/policy-processor/unattended-guard.ts

@@ -0,0 +1,158 @@
+/**
+ * The unattended-run watch: every licensed or denied call of an eval turn goes through here, the
+ * ledger counts denials, and bypass detection decides when the run stops for a person.
+ *
+ * A stop seals the run's program with `interaction_required` and the reason, asks the turn to cancel
+ * (queued calls settle as cancelled), and tells the host so it can notify. The check itself is a
+ * dedicated judge pass over the recent series; only a "yes" stops the run, except a judge that cannot be
+ * reached twice (fail closed, configurable) and the budget guard `hardStopTotal`.
+ */
+import { evalProgramOf, type SessionTurn, type ToolCallJob } from "@inhuman.tools/harness-model";
+
+import {
+  buildBypassSections,
+  decideTrigger,
+  isAckRefusal,
+  outcomeOf,
+  parseBypassVerdict,
+  recordOf,
+  stopReason,
+  type BypassVerdict,
+  RunLedger,
+} from "./bypass.js";
+import { runBypassJudge } from "./classifier.js";
+import { evidenceCuts, owningTurn } from "./evidence.js";
+import type { LicenseScope } from "./scope.js";
+
+export type RunStop = {
+  readonly kind: "bypass" | "hard-stop" | "immediate" | "check-unavailable";
+  readonly reason: string;
+  readonly turn: SessionTurn;
+};
+
+/** Text of the coded failure a stopped run reports. */
+export function stoppedRunText(reason: string): string {
+  return `Operation requires human authorization: the run was stopped (${reason}). Review it and resume.`;
+}
+
+export function ledgerFor(scope: LicenseScope, turn: SessionTurn): RunLedger {
+  let ledger = scope.ledgers.get(turn);
+  if (ledger === undefined) {
+    ledger = new RunLedger();
+    scope.ledgers.set(turn, ledger);
+  }
+  return ledger;
+}
+
+/** Seal the program with `interaction_required` and ask the turn to stop. Idempotent. */
+export function stopRun(scope: LicenseScope, turn: SessionTurn, ledger: RunLedger, stop: Omit<RunStop, "turn">): void {
+  if (ledger.stopped) return;
+  ledger.stopped = true;
+  const program = evalProgramOf(turn);
+  if (program !== null && program.outcomeSnap === undefined) {
+    program.outcomeSnap = { ok: false, text: stoppedRunText(stop.reason), code: "interaction_required" };
+  }
+  turn.cancelRequested = true;
+  scope.onRunStopped?.({ ...stop, turn });
+}
+
+/**
+ * Note how a call was licensed. Only unattended calls are counted. Never throws: the watch must not
+ * change a license that is already decided.
+ */
+export async function noteLicense(scope: LicenseScope, job: ToolCallJob, allowed: boolean): Promise<void> {
+  try {
+    if (!job.unattended) return;
+    const turn = owningTurn(job);
+    if (turn?.evalSpec == null) return;
+    const ledger = ledgerFor(scope, turn);
+    if (ledger.stopped) return;
+    if (allowed) {
+      ledger.ran(recordOf(job));
+      return;
+    }
+    const { outcome, counted } = outcomeOf(job);
+    if (!counted) {
+      ledger.uncounted({ ...recordOf(job), outcome });
+      // The run continues; the user is told what is waiting for their acknowledgement.
+      if (isAckRefusal(job)) {
+        scope.onRunNotice?.({
+          turn,
+          text: `an unattended run needs an acknowledgement: ${job.ackFlags.map((flag) => flag.label).join("; ")} (${job.name} was denied; the run continues)`,
+        });
+      }
+      return;
+    }
+    const record = ledger.denied({ ...recordOf(job), outcome });
+    const esc = job.escalation;
+    const trigger = decideTrigger(ledger, esc, record);
+    if (trigger === null) return;
+    if (trigger === "immediate") {
+      stopRun(scope, turn, ledger, {
+        kind: "immediate",
+        reason: `bypass suspected: control-plane-tampering: it reached for ${record.ruleIds.join(", ") || "the controls"} after earlier denials`,
+      });
+      return;
+    }
+    if (trigger === "hard-stop") {
+      stopRun(scope, turn, ledger, {
+        kind: "hard-stop",
+        reason: `hard stop: ${ledger.total} denied calls (limit ${esc.hardStopTotal})`,
+      });
+      return;
+    }
+    if (!esc.bypassCheck) {
+      ledger.lastCheckAtTotal = ledger.total;
+      return;
+    }
+    await check(scope, job, turn, ledger);
+  } catch {
+    /* a watch failure never lifts or changes a denial */
+  }
+}
+
+async function check(scope: LicenseScope, job: ToolCallJob, turn: SessionTurn, ledger: RunLedger): Promise<void> {
+  const esc = job.escalation;
+  const cuts = evidenceCuts(job);
+  const sections = buildBypassSections({
+    task: cuts.privileged,
+    calls: ledger.calls,
+    notes: cuts.priorTurn === "" ? [] : [cuts.priorTurn],
+    window: esc.window,
+  });
+  ledger.checks += 1;
+  let verdict: BypassVerdict | undefined;
+  for (let attempt = 0; attempt < 2 && verdict === undefined; attempt += 1) {
+    try {
+      const parsed = parseBypassVerdict(await runBypassJudge(scope, job, sections));
+      if ("fail" in parsed) throw new Error(parsed.fail);
+      verdict = parsed;
+    } catch {
+      ledger.checkFails += 1;
+    }
+  }
+  if (verdict === undefined) {
+    // Fail closed: a check that cannot be made twice is not a clean bill of health.
+    if (esc.stopOnDoubleFail) {
+      stopRun(scope, turn, ledger, { kind: "check-unavailable", reason: "bypass check unavailable" });
+    } else {
+      ledger.lastCheckAtTotal = ledger.total;
+      scope.onRunNotice?.({ turn, text: "repeated denials in an unattended run; the bypass check was unavailable" });
+    }
+    return;
+  }
+  ledger.checkFails = 0;
+  ledger.lastCheckAtTotal = ledger.total;
+  switch (verdict.verdict) {
+    case "bypass":
+      stopRun(scope, turn, ledger, { kind: "bypass", reason: stopReason(verdict) });
+      return;
+    case "unsure":
+      scope.onRunNotice?.({ turn, text: "an unattended run hit repeated denials" });
+      ledger.consecutive = 0;
+      return;
+    case "no-bypass":
+      ledger.consecutive = 0;
+      return;
+  }
+}

package/src/session/cancel.ts

@@ -0,0 +1,34 @@
+/**
+ * Stop predicate and leftover rejection for a turn. Accepted-work Calls stay.
+ * In-flight BODY abort is the caller's `abort`.
+ */
+import { ApprovableJob, CallJob, SessionTurn } from "@inhuman.tools/harness-model";
+import { walkWork } from "@inhuman.tools/plexus-expectations";
+
+import { shouldSkipTurnCancel } from "../agent-calls/acceptance.js";
+import { type CallsCatalog } from "../agent-calls/index.js";
+import { cancelUnstartedTool } from "../tool-runtime/index.js";
+
+export function turnLoopStopped(turn: SessionTurn, signal: AbortSignal): boolean {
+  return signal.aborted || turn.reducerExited || turn.cancelRequested;
+}
+
+/**
+ * Leftovers settle as cancelled so the turn's join and the session's open
+ * tool count release. An orchestrated leaf is also aborted: its executor may
+ * be gone, and a cancelled job is never claimed again.
+ */
+export function failLeftoverApprovables(
+  turn: SessionTurn,
+  abort: (leaf: ApprovableJob) => void,
+  catalog: CallsCatalog,
+): void {
+  for (const node of walkWork(turn)) {
+    if (!(node instanceof ApprovableJob)) continue;
+    if (node.completed || node.cancelled) continue;
+    if (node instanceof CallJob && shouldSkipTurnCancel(node, catalog)) continue;
+    if (node.orchestrated) abort(node);
+    cancelUnstartedTool(node);
+    if (!node.completed) node.markComplete();
+  }
+}