@zanii/blackbox 0.22.0 → 0.23.0

package/dist/analysis/index.js

@@ -8,6 +8,7 @@ import { healthFindings } from "../health/index.js";
 import { policyFindings } from "../policy/index.js";
 import { precogFindings } from "../precog/index.js";
 import { preflightFindings } from "../preflight/index.js";
+import { envelopesOf, foldState, trackRecord } from "../reconcile/record.js";
 import { canonical } from "../reconcile/shared.js";
 import { undoFindings } from "../undo/index.js";
 import { wallFindings } from "../walls/index.js";
@@ -26,27 +27,68 @@ export { checkFlightPlan, landing, } from "./landing.js";
 export { checkAgainstPacks, loadReferencePacks, packsFor, referenceContext, } from "./reference.js";
 export { OUTCOMES, rollup, waste, } from "./waste.js";
 const decoder = new TextDecoder();
-/** Parses the lines once, attaching SDK event bodies. */
+/** Parses the lines once, attaching SDK event bodies. On a tracked record (`Analyzer`), only the
+ * lines added since the last call are read. */
 export function eventsOf(lines, bodies) {
-    // N4 (idea R6): a re-sent SDK event (meta.duplicate_of) stays on the record but counts once.
-    return lines.flatMap((line) => {
-        const e = JSON.parse(line);
-        if (e.kind === "sdk.event" && e.meta.duplicate_of !== undefined)
-            return [];
-        if (e.kind === "sdk.event") {
-            const b = bodies(e.body_hash);
-            try {
-                const parsed = b === undefined ? {} : JSON.parse(decoder.decode(b));
-                e.data =
-                    typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : {};
-            }
-            catch {
-                e.data = {};
-            }
+    const fold = foldState(lines, "eventsOf", () => ({ done: 0, out: [], bodies }));
+    if (fold && fold.bodies === bodies) {
+        const all = envelopesOf(lines);
+        for (; fold.done < all.length; fold.done++) {
+            const e = withData(all[fold.done], bodies);
+            if (e)
+                fold.out.push(e);
         }
-        return [e];
+        return [...fold.out];
+    }
+    return envelopesOf(lines).flatMap((e) => {
+        const out = withData(e, bodies);
+        return out ? [out] : [];
     });
 }
+function withData(e, bodies) {
+    // N4 (idea R6): a re-sent SDK event (meta.duplicate_of) stays on the record but counts once.
+    if (e.kind === "sdk.event" && e.meta.duplicate_of !== undefined)
+        return null;
+    if (e.kind !== "sdk.event")
+        return e;
+    const b = bodies(e.body_hash);
+    let data = {};
+    try {
+        const parsed = b === undefined ? {} : JSON.parse(decoder.decode(b));
+        data = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : {};
+    }
+    catch { }
+    return Object.isFrozen(e) ? Object.freeze({ ...e, data: deepFreezeJson(data) }) : { ...e, data };
+}
+function deepFreezeJson(value) {
+    if (typeof value === "object" && value !== null && !Object.isFrozen(value)) {
+        Object.freeze(value);
+        for (const v of Object.values(value))
+            deepFreezeJson(v);
+    }
+    return value;
+}
+/**
+ * spec/findings.md §14 (G33): the analyses over one session's growing record. Each `findings` call
+ * reads only the lines added since the last one (each line is parsed once, the calls are rebuilt
+ * from the new events only) and gives exactly what `analyze` gives over the whole record.
+ */
+export class Analyzer {
+    lines = [];
+    bodies;
+    constructor(bodies) {
+        this.bodies = bodies;
+        trackRecord(this.lines);
+    }
+    /** Appends the record's new lines (in seq order; a record only grows). */
+    push(lines) {
+        for (const l of lines)
+            this.lines.push(l);
+    }
+    findings(options) {
+        return analyze(this.lines, { ...options, bodies: this.bodies });
+    }
+}
 /** Every analysis, in spec order. */
 export function analyze(lines, options) {
     const events = eventsOf(lines, options.bodies);
@@ -60,6 +102,7 @@ export function analyze(lines, options) {
         ...toolChanged(events),
         ...toolDrifted(events),
         ...secretSent(events),
+        ...elicitationSecret(events),
         ...userDownvotes(events),
         ...selfChanges(events),
         ...promptChanged(events),
@@ -473,16 +516,55 @@ export function secretSent(events) {
             n = count(e.meta.redacted);
             to = "tool";
         }
+        else if ((e.kind === "tool.elicitation" || e.kind === "tool.sampling") &&
+            e.meta.phase === "reply" &&
+            typeof e.meta.method === "string") {
+            // spec/mcp.md §6: the client's answer to a server's request went to that MCP server
+            n = count(e.meta.redacted);
+            to = "server";
+        }
         if (!to || n <= 0)
             continue;
         out.push({
             code: "SECRET_SENT",
             source: "detectors",
             severity: "warning",
-            ref: { seq: e.seq, to, count: n, ...(to === "tool" ? { tool: e.meta.tool } : {}) },
+            ref: {
+                seq: e.seq,
+                to: to === "server" ? "tool" : to,
+                count: n,
+                ...(to === "model"
+                    ? {}
+                    : { tool: (to === "tool" ? e.meta.tool : e.meta.method) }),
+            },
             detail: to === "model"
                 ? "A credential (a key, a token, a password) went to the model provider in the request."
-                : "A credential (a key, a token, a password) went to a tool in its arguments; the gateway's credential broker keeps them out of the agent's hands.",
+                : to === "tool"
+                    ? "A credential (a key, a token, a password) went to a tool in its arguments; the gateway's credential broker keeps them out of the agent's hands."
+                    : "A credential (a key, a token, a password) went to an MCP server in the client's answer to its request.",
+        });
+    }
+    return out;
+}
+/** spec/mcp.md §6: an MCP server asked the user for a credential through `elicitation/create`. */
+export function elicitationSecret(events) {
+    const out = [];
+    for (const e of events) {
+        const fields = e.meta.secret_fields;
+        if (e.kind !== "tool.elicitation" || e.meta.phase !== "request")
+            continue;
+        if (!Array.isArray(fields) || fields.length === 0)
+            continue;
+        out.push({
+            code: "ELICITATION_SECRET",
+            source: "detectors",
+            severity: "warning",
+            ref: {
+                seq: e.seq,
+                ...(typeof e.meta.server === "string" ? { server: e.meta.server } : {}),
+                fields: fields.map(String).join(","),
+            },
+            detail: "An MCP server asked the user for a credential (a password, a token, a key) through elicitation; a server should never collect one this way.",
         });
     }
     return out;

package/dist/analysis/injection.d.ts

@@ -7,7 +7,8 @@ export declare const INJECTION_RULES: ReadonlyArray<{
 export declare function hiddenText(text: string): boolean;
 /** The first rule a text breaks, or null. */
 export declare function injectionRule(text: string): string | null;
-/** spec/findings.md §1b: PROMPT_INJECTION findings over a record. */
+/** spec/findings.md §1b: PROMPT_INJECTION findings over a record. Each event is judged on its own
+ * (on a tracked record, once: G33). */
 export declare function injectionFindings(lines: readonly string[], bodies: (hash: string) => Uint8Array | undefined): {
     code: string;
     source: string;

package/dist/analysis/injection.js

@@ -4,6 +4,7 @@
 // role hijacks, exfiltration asks and hidden text, in English and Arabic. One finding per event and
 // source; the finding names the rule, never the text. Mirrors analysis/injection.py; pinned by
 // spec/vectors/injection.json.
+import { perEvent } from "../reconcile/record.js";
 const decoder = new TextDecoder();
 const MAX_TEXT = 200_000; // characters scanned per event: an injection is near the start or not worth the cost
 /** The rules, in the order they're tried. Patterns run on ASCII-lowercased, space-folded text. */
@@ -120,10 +121,10 @@ function newestTurn(body) {
     }
     return out;
 }
-/** spec/findings.md §1b: PROMPT_INJECTION findings over a record. */
+/** spec/findings.md §1b: PROMPT_INJECTION findings over a record. Each event is judged on its own
+ * (on a tracked record, once: G33). */
 export function injectionFindings(lines, bodies) {
-    const out = [];
-    const add = (seq, source, rule, tool) => out.push({
+    const found = (seq, source, rule, tool) => ({
         code: "PROMPT_INJECTION",
         source: "detectors",
         severity: "warning",
@@ -132,32 +133,31 @@ export function injectionFindings(lines, bodies) {
             ? "Text the reader can't see (hidden characters) reached the model: a classic way to smuggle instructions."
             : "Text that reached the model tries to give it new instructions: an override, a role change, or a request to send out secrets.",
     });
-    for (const l of lines) {
-        const e = JSON.parse(l);
+    return perEvent(lines, "injection", bodies, (envelope) => {
+        const e = envelope;
         if (e.kind !== "tool.result" && e.kind !== "llm.request")
-            continue;
+            return [];
         const raw = bodies(e.body_hash);
         if (!raw || raw.length === 0)
-            continue;
+            return [];
         let doc;
         try {
             doc = JSON.parse(decoder.decode(raw));
         }
         catch {
-            continue;
+            return [];
         }
         if (e.kind === "tool.result") {
             const result = doc?.result;
             if (result === undefined)
-                continue;
+                return [];
             const rule = injectionRule(textOf(result));
             const tool = typeof e.meta.tool === "string" ? e.meta.tool : undefined;
-            if (rule)
-                add(e.seq, "tool_result", rule, tool);
-            continue;
+            return rule ? [found(e.seq, "tool_result", rule, tool)] : [];
         }
         if (!doc || typeof doc !== "object" || Array.isArray(doc))
-            continue;
+            return [];
+        const out = [];
         const seen = new Set();
         for (const part of newestTurn(doc)) {
             if (seen.has(part.source))
@@ -165,9 +165,9 @@ export function injectionFindings(lines, bodies) {
             const rule = injectionRule(part.text);
             if (rule) {
                 seen.add(part.source);
-                add(e.seq, part.source, rule);
+                out.push(found(e.seq, part.source, rule));
             }
         }
-    }
-    return out;
+        return out;
+    });
 }

package/dist/analysis/landing.js

@@ -4,6 +4,8 @@ import { checkDuty } from "../duty/index.js";
 import { callsOf } from "../reconcile/record.js";
 import { isObj } from "../reconcile/shared.js";
 import { toolCallsOf, toolResultsOf } from "./detectors.js";
+import { answerText } from "./hallucination.js";
+import { checkPredicate, judgeAnswer, judgeArgs } from "./plan-checks.js";
 /** Every tool call in the record, with what its result said (provider tool_use ids; MCP call_seq). */
 function usesOf(events, lines, bodies) {
     const calls = callsOf(lines, bodies);
@@ -13,16 +15,26 @@ function usesOf(events, lines, bodies) {
             results.set(r.id, r);
     const uses = toolCallsOf(calls).map((t) => {
         const r = t.id === undefined ? undefined : results.get(t.id);
-        return r
-            ? { seq: t.seq, tool: t.tool, ok: !r.error, resultSeq: r.seq }
-            : { seq: t.seq, tool: t.tool };
+        const base = { seq: t.seq, tool: t.tool, args: t.args ?? null, hasArgs: true };
+        return r ? { ...base, ok: !r.error, resultSeq: r.seq } : base;
     });
     const mcp = new Map();
     for (const e of events) {
         if (e.kind === "tool.call" &&
             e.meta.method === "tools/call" &&
             typeof e.meta.tool === "string") {
-            const u = { seq: e.seq, tool: e.meta.tool };
+            const u = { seq: e.seq, tool: e.meta.tool, hasArgs: false };
+            // the arguments are the request body's params.arguments (gone when the body was erased)
+            try {
+                const body = bodies(e.body_hash);
+                if (body !== undefined) {
+                    const rpc = JSON.parse(new TextDecoder().decode(body));
+                    const params = isObj(rpc) ? rpc.params : undefined;
+                    u.args = isObj(params) && "arguments" in params ? params.arguments : {};
+                    u.hasArgs = true;
+                }
+            }
+            catch { }
             mcp.set(e.seq, u);
             uses.push(u);
         }
@@ -62,13 +74,33 @@ function plansOf(events) {
     return { plan: null, invalid };
 }
 const listOf = (v) => (Array.isArray(v) ? v : []);
-/** One predicate over the tool calls (and results) recorded before `before`. */
-function judge(p, uses, before) {
+/** The final answers on the record (a complete model response with text and no tool call). */
+function answersOf(lines, bodies) {
+    const calls = callsOf(lines, bodies);
+    const ends = new Set(toolCallsOf(calls).map((t) => t.seq));
+    return [...calls]
+        .sort((a, b) => a.endSeq - b.endSeq)
+        .filter((c) => c.complete && c.request && !ends.has(c.endSeq))
+        .map((c) => ({ seq: c.endSeq, text: answerText(c) }))
+        .filter((a) => a.text.length > 0);
+}
+/** One predicate over the tool calls (and results, and answers) recorded before `before`. */
+function judge(p, uses, before, answers = []) {
     const seen = uses.filter((u) => u.seq < before);
     const one = isObj(p) && Object.keys(p).length === 1 ? Object.entries(p)[0] : null;
     const [key, arg] = one ?? ["", null];
     const named = typeof arg === "string" ? seen.filter((u) => u.tool === arg) : [];
     const out = (result, gap) => ({ predicate: p, result, ...(gap ? { gap } : {}) });
+    if (key === "tool_args" || key === "answer") {
+        // spec/findings.md §5a: a claim's own asserts aren't checked when filed, so here
+        const bad = checkPredicate(p);
+        if (bad)
+            return out("cannot_verify", `invalid predicate: ${bad}`);
+        const r = key === "tool_args"
+            ? judgeArgs(arg, uses, before)
+            : judgeAnswer(arg, answers, before);
+        return out(r.result, r.gap);
+    }
     if (key === "tool_called" && typeof arg === "string")
         return named.length ? out("pass") : out("fail", `${arg} was never called`);
     if (key === "tool_not_called" && typeof arg === "string")
@@ -94,6 +126,7 @@ function judge(p, uses, before) {
 export function landing(events, lines, bodies) {
     const plan = planOf(events);
     const uses = usesOf(events, lines, bodies);
+    const answers = answersOf(lines, bodies);
     const claims = [];
     for (const e of events) {
         if (e.kind !== "sdk.event" || e.meta.type !== "claim")
@@ -102,11 +135,11 @@ export function landing(events, lines, bodies) {
         const criteria = [
             ...listOf(plan?.criteria).map((p, i) => ({
                 criterion: `plan:${i}`,
-                ...judge(p, uses, e.seq),
+                ...judge(p, uses, e.seq, answers),
             })),
             ...listOf(data.asserts).map((p, i) => ({
                 criterion: `assert:${i}`,
-                ...judge(p, uses, e.seq),
+                ...judge(p, uses, e.seq, answers),
             })),
         ];
         const verdict = criteria.some((c) => c.result === "fail")
@@ -145,7 +178,9 @@ export function flightPlan(events, lines, bodies) {
                 }
         }
         listOf(plan.criteria).forEach((p, i) => {
-            if (!isObj(p) || !("tool_not_called" in p || "max_tool_calls" in p))
+            // broken as soon as it is: a limit, or a rule on every call's arguments (§5a.2)
+            const every = isObj(p) && isObj(p.tool_args) && p.tool_args.for === "every";
+            if (!isObj(p) || !("tool_not_called" in p || "max_tool_calls" in p || every))
                 return;
             const r = judge(p, uses, Number.POSITIVE_INFINITY);
             if (r.result === "fail")
@@ -196,11 +231,22 @@ export function checkFlightPlan(plan) {
             tools.every((x) => typeof x === "string" && x.length > 0 && x.length <= 128)))
         return "flight_plan.expected_tools must be at most 100 tool names";
     const value = (v) => typeof v === "string" ? v.length <= 128 : Number.isSafeInteger(v) && v >= 0;
-    if (criteria !== undefined &&
-        !(Array.isArray(criteria) &&
-            criteria.length <= 50 &&
-            criteria.every((c) => isObj(c) && Object.keys(c).length === 1 && value(Object.values(c)[0]))))
-        return 'flight_plan.criteria must be at most 50 predicates like {"tool_called": "run_tests"}';
+    const legacy = 'flight_plan.criteria must be at most 50 predicates like {"tool_called": "run_tests"}';
+    if (criteria !== undefined) {
+        if (!Array.isArray(criteria) || criteria.length > 50)
+            return legacy;
+        for (const [i, c] of criteria.entries()) {
+            if (!isObj(c) || Object.keys(c).length !== 1)
+                return legacy;
+            if ("tool_args" in c || "answer" in c) {
+                const bad = checkPredicate(c); // spec/findings.md §5a
+                if (bad)
+                    return `flight_plan.criteria[${i}]: ${bad}`;
+            }
+            else if (!value(Object.values(c)[0]))
+                return legacy;
+        }
+    }
     const name = (x) => typeof x === "string" && x.length > 0 && x.length <= 128;
     const { sterile } = plan;
     if (sterile !== undefined &&

package/dist/analysis/plan-checks.d.ts

@@ -0,0 +1,55 @@
+type Test = (c: string) => boolean;
+type Inst = {
+    op: "char";
+    test: Test;
+} | {
+    op: "split";
+    x: number;
+    y: number;
+} | {
+    op: "jmp";
+    x: number;
+} | {
+    op: "assert";
+    kind: "bol" | "eol" | "word" | "nonword";
+} | {
+    op: "match";
+};
+export interface Pattern {
+    prog: Inst[];
+}
+/** Text longer than this (in code points) isn't matched (cannot_verify). */
+export declare const MAX_MATCH_TEXT = 100000;
+/** Parses and compiles a pattern, or says why it isn't one (spec/findings.md §5a.1). */
+export declare function compilePattern(source: unknown): Pattern | string;
+/** Whether the pattern matches anywhere in the text (a search, not a full match). */
+export declare function patternMatches(p: Pattern, text: string): boolean;
+/** A value in integer milli-units (spec/findings.md §5a.2), or null. */
+export declare function milliOf(v: unknown): number | null;
+/** Why a schema is outside the subset, or undefined. */
+export declare function checkSchema(schema: unknown): string | undefined;
+/** The first place a value breaks a (checked) schema, or undefined (spec/findings.md §5a.3). */
+export declare function schemaError(value: unknown, schema: unknown, path?: string): string | undefined;
+/** Why a `tool_args` or `answer` predicate is invalid, or undefined. */
+export declare function checkPredicate(p: unknown): string | undefined;
+/** A tool call with its arguments, when they're on the record (`args` absent: unknown). */
+export interface ArgUse {
+    seq: number;
+    tool: string;
+    args?: unknown;
+    hasArgs: boolean;
+}
+type Out = {
+    result: "pass" | "fail" | "cannot_verify";
+    gap?: string;
+};
+/** A `tool_args` predicate over the tool calls before `before` (spec/findings.md §5a.2). */
+export declare function judgeArgs(arg: Record<string, unknown>, uses: readonly ArgUse[], before: number): Out;
+/** The JSON an answer holds: the whole text, or one ``` fence around it; ABSENT when it isn't JSON. */
+export declare function answerJson(text: string): unknown;
+/** An `answer` predicate over the last final answer before `before` (spec/findings.md §5a.4). */
+export declare function judgeAnswer(arg: Record<string, unknown>, answers: ReadonlyArray<{
+    seq: number;
+    text: string;
+}>, before: number): Out;
+export {};