@zanii/blackbox 0.21.2 → 0.23.0

package/dist/analysis/landing.js

@@ -4,6 +4,8 @@ import { checkDuty } from "../duty/index.js";
 import { callsOf } from "../reconcile/record.js";
 import { isObj } from "../reconcile/shared.js";
 import { toolCallsOf, toolResultsOf } from "./detectors.js";
+import { answerText } from "./hallucination.js";
+import { checkPredicate, judgeAnswer, judgeArgs } from "./plan-checks.js";
 /** Every tool call in the record, with what its result said (provider tool_use ids; MCP call_seq). */
 function usesOf(events, lines, bodies) {
     const calls = callsOf(lines, bodies);
@@ -13,16 +15,26 @@ function usesOf(events, lines, bodies) {
             results.set(r.id, r);
     const uses = toolCallsOf(calls).map((t) => {
         const r = t.id === undefined ? undefined : results.get(t.id);
-        return r
-            ? { seq: t.seq, tool: t.tool, ok: !r.error, resultSeq: r.seq }
-            : { seq: t.seq, tool: t.tool };
+        const base = { seq: t.seq, tool: t.tool, args: t.args ?? null, hasArgs: true };
+        return r ? { ...base, ok: !r.error, resultSeq: r.seq } : base;
     });
     const mcp = new Map();
     for (const e of events) {
         if (e.kind === "tool.call" &&
             e.meta.method === "tools/call" &&
             typeof e.meta.tool === "string") {
-            const u = { seq: e.seq, tool: e.meta.tool };
+            const u = { seq: e.seq, tool: e.meta.tool, hasArgs: false };
+            // the arguments are the request body's params.arguments (gone when the body was erased)
+            try {
+                const body = bodies(e.body_hash);
+                if (body !== undefined) {
+                    const rpc = JSON.parse(new TextDecoder().decode(body));
+                    const params = isObj(rpc) ? rpc.params : undefined;
+                    u.args = isObj(params) && "arguments" in params ? params.arguments : {};
+                    u.hasArgs = true;
+                }
+            }
+            catch { }
             mcp.set(e.seq, u);
             uses.push(u);
         }
@@ -62,13 +74,33 @@ function plansOf(events) {
     return { plan: null, invalid };
 }
 const listOf = (v) => (Array.isArray(v) ? v : []);
-/** One predicate over the tool calls (and results) recorded before `before`. */
-function judge(p, uses, before) {
+/** The final answers on the record (a complete model response with text and no tool call). */
+function answersOf(lines, bodies) {
+    const calls = callsOf(lines, bodies);
+    const ends = new Set(toolCallsOf(calls).map((t) => t.seq));
+    return [...calls]
+        .sort((a, b) => a.endSeq - b.endSeq)
+        .filter((c) => c.complete && c.request && !ends.has(c.endSeq))
+        .map((c) => ({ seq: c.endSeq, text: answerText(c) }))
+        .filter((a) => a.text.length > 0);
+}
+/** One predicate over the tool calls (and results, and answers) recorded before `before`. */
+function judge(p, uses, before, answers = []) {
     const seen = uses.filter((u) => u.seq < before);
     const one = isObj(p) && Object.keys(p).length === 1 ? Object.entries(p)[0] : null;
     const [key, arg] = one ?? ["", null];
     const named = typeof arg === "string" ? seen.filter((u) => u.tool === arg) : [];
     const out = (result, gap) => ({ predicate: p, result, ...(gap ? { gap } : {}) });
+    if (key === "tool_args" || key === "answer") {
+        // spec/findings.md §5a: a claim's own asserts aren't checked when filed, so here
+        const bad = checkPredicate(p);
+        if (bad)
+            return out("cannot_verify", `invalid predicate: ${bad}`);
+        const r = key === "tool_args"
+            ? judgeArgs(arg, uses, before)
+            : judgeAnswer(arg, answers, before);
+        return out(r.result, r.gap);
+    }
     if (key === "tool_called" && typeof arg === "string")
         return named.length ? out("pass") : out("fail", `${arg} was never called`);
     if (key === "tool_not_called" && typeof arg === "string")
@@ -94,6 +126,7 @@ function judge(p, uses, before) {
 export function landing(events, lines, bodies) {
     const plan = planOf(events);
     const uses = usesOf(events, lines, bodies);
+    const answers = answersOf(lines, bodies);
     const claims = [];
     for (const e of events) {
         if (e.kind !== "sdk.event" || e.meta.type !== "claim")
@@ -102,11 +135,11 @@ export function landing(events, lines, bodies) {
         const criteria = [
             ...listOf(plan?.criteria).map((p, i) => ({
                 criterion: `plan:${i}`,
-                ...judge(p, uses, e.seq),
+                ...judge(p, uses, e.seq, answers),
             })),
             ...listOf(data.asserts).map((p, i) => ({
                 criterion: `assert:${i}`,
-                ...judge(p, uses, e.seq),
+                ...judge(p, uses, e.seq, answers),
             })),
         ];
         const verdict = criteria.some((c) => c.result === "fail")
@@ -145,7 +178,9 @@ export function flightPlan(events, lines, bodies) {
                 }
         }
         listOf(plan.criteria).forEach((p, i) => {
-            if (!isObj(p) || !("tool_not_called" in p || "max_tool_calls" in p))
+            // broken as soon as it is: a limit, or a rule on every call's arguments (§5a.2)
+            const every = isObj(p) && isObj(p.tool_args) && p.tool_args.for === "every";
+            if (!isObj(p) || !("tool_not_called" in p || "max_tool_calls" in p || every))
                 return;
             const r = judge(p, uses, Number.POSITIVE_INFINITY);
             if (r.result === "fail")
@@ -196,11 +231,22 @@ export function checkFlightPlan(plan) {
             tools.every((x) => typeof x === "string" && x.length > 0 && x.length <= 128)))
         return "flight_plan.expected_tools must be at most 100 tool names";
     const value = (v) => typeof v === "string" ? v.length <= 128 : Number.isSafeInteger(v) && v >= 0;
-    if (criteria !== undefined &&
-        !(Array.isArray(criteria) &&
-            criteria.length <= 50 &&
-            criteria.every((c) => isObj(c) && Object.keys(c).length === 1 && value(Object.values(c)[0]))))
-        return 'flight_plan.criteria must be at most 50 predicates like {"tool_called": "run_tests"}';
+    const legacy = 'flight_plan.criteria must be at most 50 predicates like {"tool_called": "run_tests"}';
+    if (criteria !== undefined) {
+        if (!Array.isArray(criteria) || criteria.length > 50)
+            return legacy;
+        for (const [i, c] of criteria.entries()) {
+            if (!isObj(c) || Object.keys(c).length !== 1)
+                return legacy;
+            if ("tool_args" in c || "answer" in c) {
+                const bad = checkPredicate(c); // spec/findings.md §5a
+                if (bad)
+                    return `flight_plan.criteria[${i}]: ${bad}`;
+            }
+            else if (!value(Object.values(c)[0]))
+                return legacy;
+        }
+    }
     const name = (x) => typeof x === "string" && x.length > 0 && x.length <= 128;
     const { sterile } = plan;
     if (sterile !== undefined &&

package/dist/analysis/plan-checks.d.ts

@@ -0,0 +1,55 @@
+type Test = (c: string) => boolean;
+type Inst = {
+    op: "char";
+    test: Test;
+} | {
+    op: "split";
+    x: number;
+    y: number;
+} | {
+    op: "jmp";
+    x: number;
+} | {
+    op: "assert";
+    kind: "bol" | "eol" | "word" | "nonword";
+} | {
+    op: "match";
+};
+export interface Pattern {
+    prog: Inst[];
+}
+/** Text longer than this (in code points) isn't matched (cannot_verify). */
+export declare const MAX_MATCH_TEXT = 100000;
+/** Parses and compiles a pattern, or says why it isn't one (spec/findings.md §5a.1). */
+export declare function compilePattern(source: unknown): Pattern | string;
+/** Whether the pattern matches anywhere in the text (a search, not a full match). */
+export declare function patternMatches(p: Pattern, text: string): boolean;
+/** A value in integer milli-units (spec/findings.md §5a.2), or null. */
+export declare function milliOf(v: unknown): number | null;
+/** Why a schema is outside the subset, or undefined. */
+export declare function checkSchema(schema: unknown): string | undefined;
+/** The first place a value breaks a (checked) schema, or undefined (spec/findings.md §5a.3). */
+export declare function schemaError(value: unknown, schema: unknown, path?: string): string | undefined;
+/** Why a `tool_args` or `answer` predicate is invalid, or undefined. */
+export declare function checkPredicate(p: unknown): string | undefined;
+/** A tool call with its arguments, when they're on the record (`args` absent: unknown). */
+export interface ArgUse {
+    seq: number;
+    tool: string;
+    args?: unknown;
+    hasArgs: boolean;
+}
+type Out = {
+    result: "pass" | "fail" | "cannot_verify";
+    gap?: string;
+};
+/** A `tool_args` predicate over the tool calls before `before` (spec/findings.md §5a.2). */
+export declare function judgeArgs(arg: Record<string, unknown>, uses: readonly ArgUse[], before: number): Out;
+/** The JSON an answer holds: the whole text, or one ``` fence around it; ABSENT when it isn't JSON. */
+export declare function answerJson(text: string): unknown;
+/** An `answer` predicate over the last final answer before `before` (spec/findings.md §5a.4). */
+export declare function judgeAnswer(arg: Record<string, unknown>, answers: ReadonlyArray<{
+    seq: number;
+    text: string;
+}>, before: number): Out;
+export {};