@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/evals/tool-loop-cases.js

@@ -0,0 +1,401 @@
+/**
+ * Built-in evaluation cases for the frontend tool-loop flow.
+ *
+ * Each case ships a small, self-contained node catalog (so runs are
+ * deterministic and need no live registry) plus structural expectations on the
+ * tool sequence and the resulting graph. Expectations are deliberately loose on
+ * ordering-of-equivalent-calls and strict on outcomes: many tool orderings
+ * build a valid graph, but all must add the right node families, wire them, and
+ * avoid errors.
+ */
+import { createToolLoopBridge } from "./tool-loop-bridge.js";
+/** Minimal Property with the fields the headless tools read. */
+function prop(name, type, opts = {}) {
+    return {
+        name,
+        type: { type, optional: !opts.required, type_args: [] },
+        default: opts.default ?? null,
+        required: opts.required ?? false
+    };
+}
+/** Minimal OutputSlot. */
+function out(name, type) {
+    return {
+        name,
+        type: { type, optional: false, type_args: [] },
+        stream: false
+    };
+}
+/** Build a NodeMetadata stub carrying only the fields the tools consult. */
+function defineNode(node_type, opts = {}) {
+    const segments = node_type.split(".");
+    return {
+        title: opts.title ?? segments[segments.length - 1],
+        description: opts.description ?? "",
+        namespace: segments.slice(0, -1).join("."),
+        node_type,
+        layout: "default",
+        properties: opts.properties ?? [],
+        outputs: opts.outputs ?? [out("output", "any")],
+        recommended_models: [],
+        required_settings: [],
+        supports_dynamic_inputs: false,
+        is_streaming_output: false,
+        supports_dynamic_outputs: false
+    };
+}
+/**
+ * Shared catalog: a string input, a couple of text-processing steps, an LLM
+ * agent step, and an output node. Enough to express "take input → process →
+ * output" objectives via the tool surface.
+ */
+export const TOOL_LOOP_NODE_CATALOG = Object.fromEntries([
+    defineNode("nodetool.input.StringInput", {
+        title: "String Input",
+        description: "A workflow input that supplies a string value by name.",
+        properties: [
+            prop("name", "str", { required: true }),
+            prop("value", "str")
+        ],
+        outputs: [out("output", "str")]
+    }),
+    defineNode("nodetool.text.Concat", {
+        title: "Concat",
+        description: "Concatenate two strings into one.",
+        properties: [prop("a", "str", { required: true }), prop("b", "str")],
+        outputs: [out("output", "str")]
+    }),
+    defineNode("nodetool.text.FormatText", {
+        title: "Format Text",
+        description: "Format a template string using named inputs.",
+        properties: [
+            prop("template", "str", { required: true }),
+            prop("text", "str")
+        ],
+        outputs: [out("output", "str")]
+    }),
+    defineNode("nodetool.agents.Agent", {
+        title: "Agent",
+        description: "An LLM step: runs a prompt over its input and returns text.",
+        properties: [
+            prop("prompt", "str", { required: true }),
+            prop("input", "str")
+        ],
+        outputs: [out("output", "str")]
+    }),
+    defineNode("nodetool.output.StringOutput", {
+        title: "String Output",
+        description: "Surfaces a string as a named workflow output.",
+        properties: [
+            prop("name", "str", { required: true }),
+            prop("value", "str")
+        ],
+        outputs: []
+    })
+].map((m) => [m.node_type, m]));
+/** Count nodes in the final graph whose type starts with `prefix`. */
+function countByPrefix(state, prefix) {
+    return state.nodes.filter((n) => n.type.startsWith(prefix)).length;
+}
+/** Every non-input node has an incoming edge; every non-output an outgoing one. */
+const connectedPredicate = {
+    name: "connected",
+    detail: "some node is unwired",
+    test: (state) => {
+        const hasIn = new Set(state.edges.map((e) => e.target));
+        const hasOut = new Set(state.edges.map((e) => e.source));
+        return state.nodes.every((n) => {
+            const isInput = n.type.startsWith("nodetool.input.");
+            const isOutput = n.type.startsWith("nodetool.output.");
+            if (!isInput && !hasIn.has(n.id))
+                return false;
+            if (!isOutput && !hasOut.has(n.id))
+                return false;
+            return true;
+        });
+    }
+};
+/** Pre-seeded input→agent graph for the `extend-existing` case. */
+const EXTEND_SEED_NODES = [
+    {
+        id: "in1",
+        type: "nodetool.input.StringInput",
+        position: { x: 100, y: 100 },
+        data: { properties: { name: "text", value: "" } }
+    },
+    {
+        id: "agent1",
+        type: "nodetool.agents.Agent",
+        position: { x: 360, y: 100 },
+        data: { properties: { prompt: "summarize", input: "" } }
+    }
+];
+const EXTEND_SEED_EDGES = [
+    {
+        id: "edge_1",
+        source: "in1",
+        target: "agent1",
+        sourceHandle: "output",
+        targetHandle: "input"
+    }
+];
+/** Node ids/types/properties and edge endpoints: what a mutation would change. */
+function graphShape(nodes, edges) {
+    return JSON.stringify({
+        nodes: nodes.map((n) => [n.id, n.type, n.data.properties]),
+        edges: edges.map((e) => [
+            e.source,
+            e.sourceHandle,
+            e.target,
+            e.targetHandle
+        ])
+    });
+}
+const EXTEND_SEED_SHAPE = graphShape(EXTEND_SEED_NODES, EXTEND_SEED_EDGES);
+/** The seeded input→agent graph came through untouched. */
+const extendSeedUnchanged = {
+    name: "graphUnchanged",
+    detail: "the seeded graph was mutated",
+    test: (state) => graphShape(state.nodes, state.edges) === EXTEND_SEED_SHAPE
+};
+/**
+ * The mutation the two permission-gate cases ask for. It names the node type,
+ * so the model needs no `ui_search_nodes` call — that tool has no permission
+ * class of its own and plan mode blocks it along with the writes.
+ */
+const GATED_MUTATION_OBJECTIVE = "The workflow already has a StringInput ('text', id=in1) feeding an Agent (id=agent1). Add a node of type nodetool.output.StringOutput (id=out1) named 'result' and connect agent1's output to its value input. You may inspect the graph with ui_get_graph first.";
+/** Seeded positions for the `rewire-and-relabel` case, read back by its layout predicate. */
+const REWIRE_SEED_POSITIONS = {
+    in1: { x: 60, y: 60 },
+    agent1: { x: 60, y: 260 },
+    out1: { x: 60, y: 460 }
+};
+/**
+ * Pre-seeded input→agent→output graph for `rewire-and-relabel`, with the
+ * output wired straight from the input (bypassing the agent) and the agent's
+ * required `prompt` left empty.
+ */
+const REWIRE_SEED_NODES = [
+    {
+        id: "in1",
+        type: "nodetool.input.StringInput",
+        position: { ...REWIRE_SEED_POSITIONS.in1 },
+        data: { properties: { name: "text", value: "" } }
+    },
+    {
+        id: "agent1",
+        type: "nodetool.agents.Agent",
+        position: { ...REWIRE_SEED_POSITIONS.agent1 },
+        data: { properties: { prompt: "", input: "" } }
+    },
+    {
+        id: "out1",
+        type: "nodetool.output.StringOutput",
+        position: { ...REWIRE_SEED_POSITIONS.out1 },
+        data: { properties: { name: "summary", value: "" } }
+    }
+];
+const REWIRE_SEED_EDGES = [
+    {
+        id: "edge_seed_1",
+        source: "in1",
+        target: "agent1",
+        sourceHandle: "output",
+        targetHandle: "input"
+    },
+    {
+        id: "edge_seed_2",
+        source: "in1",
+        target: "out1",
+        sourceHandle: "output",
+        targetHandle: "value"
+    }
+];
+export const TOOL_LOOP_EVAL_CASES = [
+    {
+        id: "summarize",
+        description: "Wire a StringInput → Agent → StringOutput chain via tools",
+        objective: "Build a workflow that takes a string input named 'text', summarizes it with an LLM agent step, and exposes the summary as a string output named 'summary'. Search for node types, add the nodes, and connect them.",
+        createBridge: () => createToolLoopBridge({ nodeMetadata: TOOL_LOOP_NODE_CATALOG }),
+        expect: {
+            requiredTools: ["ui_add_node", "ui_connect_nodes"],
+            forbiddenTools: ["ui_delete_node"],
+            ordering: [["ui_add_node", "ui_connect_nodes"]],
+            noErrorResults: true,
+            minToolCalls: 3,
+            maxToolCalls: 20,
+            finalState: [
+                {
+                    name: "hasInput",
+                    detail: "no nodetool.input.* node",
+                    test: (s) => countByPrefix(s, "nodetool.input.") >= 1
+                },
+                {
+                    name: "hasAgent",
+                    detail: "no nodetool.agents.* node",
+                    test: (s) => countByPrefix(s, "nodetool.agents.") >= 1
+                },
+                {
+                    name: "hasOutput",
+                    detail: "no nodetool.output.* node",
+                    test: (s) => countByPrefix(s, "nodetool.output.") >= 1
+                },
+                {
+                    name: "minEdges",
+                    detail: "fewer than 2 edges",
+                    test: (s) => s.edges.length >= 2
+                },
+                connectedPredicate
+            ]
+        }
+    },
+    {
+        id: "extend-existing",
+        description: "Add an output to a pre-seeded input→agent graph",
+        objective: "The workflow already has a StringInput ('text', id=in1) feeding an Agent (id=agent1). Add a StringOutput node named 'result' and connect the agent's output to it so the result is surfaced.",
+        userPrompt: "Objective: The workflow already has a StringInput ('text', id=in1) feeding an Agent (id=agent1). Add a StringOutput node named 'result' and connect the agent's output to it so the result is surfaced.\n\nThe workflow already contains 2 node(s); build on top of them.",
+        createBridge: () => createToolLoopBridge({
+            nodeMetadata: TOOL_LOOP_NODE_CATALOG,
+            nodes: EXTEND_SEED_NODES,
+            edges: EXTEND_SEED_EDGES
+        }),
+        expect: {
+            requiredTools: ["ui_add_node", "ui_connect_nodes"],
+            ordering: [["ui_add_node", "ui_connect_nodes"]],
+            noErrorResults: true,
+            minToolCalls: 2,
+            maxToolCalls: 12,
+            finalState: [
+                {
+                    name: "hasOutput",
+                    detail: "no nodetool.output.* node added",
+                    test: (s) => countByPrefix(s, "nodetool.output.") >= 1
+                },
+                {
+                    name: "agentWired",
+                    detail: "agent1 output not connected onward",
+                    test: (s) => s.edges.some((e) => e.source === "agent1")
+                }
+            ]
+        }
+    },
+    {
+        id: "rewire-and-relabel",
+        description: "Read a seeded graph, fill an unset required property, delete a bypass edge, rewire through the agent, retitle every node, and reposition them",
+        objective: "This workflow already has a StringInput ('text', id=in1) feeding an Agent step (id=agent1), whose prompt property is empty. The StringOutput (id=out1) is currently wired directly from in1, bypassing the agent entirely. Inspect the current graph first, then fix it: fill in agent1's prompt with a real instruction (e.g. 'Summarize the input text'); delete the edge that connects in1 straight to out1; connect agent1's output to out1 so the summary reaches the output instead; give in1, agent1 and out1 clearer titles describing what each one does; and reposition the three nodes into a clearer left-to-right layout.",
+        createBridge: () => createToolLoopBridge({
+            nodeMetadata: TOOL_LOOP_NODE_CATALOG,
+            nodes: REWIRE_SEED_NODES,
+            edges: REWIRE_SEED_EDGES
+        }),
+        expect: {
+            requiredTools: [
+                "ui_get_graph",
+                "ui_update_node_data",
+                "ui_delete_edge",
+                "ui_connect_nodes",
+                "ui_move_node",
+                "ui_set_node_title"
+            ],
+            ordering: [
+                ["ui_get_graph", "ui_delete_edge"],
+                ["ui_get_graph", "ui_update_node_data"]
+            ],
+            noErrorResults: true,
+            minToolCalls: 6,
+            maxToolCalls: 30,
+            finalState: [
+                {
+                    name: "promptFilled",
+                    detail: "agent1's prompt property is still empty",
+                    test: (s) => {
+                        const agent = s.nodes.find((n) => n.id === "agent1");
+                        const value = agent?.data.properties.prompt;
+                        return typeof value === "string" && value.trim().length > 0;
+                    }
+                },
+                {
+                    name: "bypassEdgeRemoved",
+                    detail: "the seeded in1 -> out1 bypass edge is still present",
+                    test: (s) => !s.edges.some((e) => e.source === "in1" && e.target === "out1")
+                },
+                {
+                    name: "agentWiredToOutput",
+                    detail: "agent1's output is not connected to out1's value input",
+                    test: (s) => s.edges.some((e) => e.source === "agent1" &&
+                        e.target === "out1" &&
+                        e.targetHandle === "value")
+                },
+                {
+                    name: "nodesRepositioned",
+                    detail: "one or more seeded nodes were not moved from their seeded position",
+                    test: (s) => ["in1", "agent1", "out1"].every((id) => {
+                        const node = s.nodes.find((n) => n.id === id);
+                        if (!node)
+                            return false;
+                        const seed = REWIRE_SEED_POSITIONS[id];
+                        return (Math.hypot(node.position.x - seed.x, node.position.y - seed.y) > 40);
+                    })
+                },
+                {
+                    name: "titlesSet",
+                    detail: "in1, agent1 and out1 were not all given titles",
+                    test: (s) => ["in1", "agent1", "out1"].every((id) => {
+                        const node = s.nodes.find((n) => n.id === id);
+                        return !!node?.data.title && node.data.title.trim().length > 0;
+                    })
+                }
+            ]
+        }
+    },
+    {
+        id: "plan-mode-blocks-mutation",
+        description: "Under plan mode, a requested add-node mutation must not reach the graph and must never prompt for approval",
+        objective: GATED_MUTATION_OBJECTIVE,
+        createBridge: () => createToolLoopBridge({
+            nodeMetadata: TOOL_LOOP_NODE_CATALOG,
+            nodes: EXTEND_SEED_NODES,
+            edges: EXTEND_SEED_EDGES
+        }),
+        permission: { mode: "plan" },
+        expect: {
+            maxToolCalls: 6,
+            finalState: [extendSeedUnchanged],
+            permissionRequests: [
+                {
+                    name: "neverPrompted",
+                    detail: "plan mode blocks writes outright; it must not ask",
+                    test: (requests) => requests.length === 0
+                }
+            ]
+        }
+    },
+    {
+        id: "denied-mutation-stays-out",
+        description: "Under default mode with the user denying, the add-node call is asked about exactly once, denied, and the graph is untouched",
+        objective: GATED_MUTATION_OBJECTIVE,
+        createBridge: () => createToolLoopBridge({
+            nodeMetadata: TOOL_LOOP_NODE_CATALOG,
+            nodes: EXTEND_SEED_NODES,
+            edges: EXTEND_SEED_EDGES
+        }),
+        permission: { mode: "default", approve: "deny" },
+        expect: {
+            maxToolCalls: 6,
+            finalState: [extendSeedUnchanged],
+            permissionRequests: [
+                {
+                    name: "addNodeDeniedOnce",
+                    detail: "expected exactly one ui_add_node approval request, category write, denied, and no request granted",
+                    test: (requests) => {
+                        const addNode = requests.filter((r) => r.toolName === "ui_add_node");
+                        return (addNode.length === 1 &&
+                            addNode[0].category === "write" &&
+                            requests.every((r) => r.decision === "deny"));
+                    }
+                }
+            ]
+        }
+    }
+];
+//# sourceMappingURL=tool-loop-cases.js.map

package/dist/evals/tool-loop-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-loop-cases.js","sourceRoot":"","sources":["../../src/evals/tool-loop-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAOH,OAAO,EACL,oBAAoB,EAIrB,MAAM,uBAAuB,CAAC;AAE/B,gEAAgE;AAChE,SAAS,IAAI,CACX,IAAY,EACZ,IAAY,EACZ,IAAI,GAA8C,EAAE;IAEpD,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QACvD,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,KAAK;KACjC,CAAC;AACJ,CAAC;AAED,0BAA0B;AAC1B,SAAS,GAAG,CAAC,IAAY,EAAE,IAAY;IACrC,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE;QAC9C,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,SAAS,UAAU,CACjB,SAAiB,EACjB,IAAI,GAKA,EAAE;IAEN,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAClD,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;QACnC,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAC1C,SAAS;QACT,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QACjC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC/C,kBAAkB,EAAE,EAAE;QACtB,iBAAiB,EAAE,EAAE;QACrB,uBAAuB,EAAE,KAAK;QAC9B,mBAAmB,EAAE,KAAK;QAC1B,wBAAwB,EAAE,KAAK;KAChC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GACjC,MAAM,CAAC,WAAW,CAChB;IACE,UAAU,CAAC,4BAA4B,EAAE;QACvC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,wDAAwD;QACrE,UAAU,EAAE;YACV,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACrB;QACD,OAAO,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;KAChC,CAAC;IACF,UAAU,CAAC,sBAAsB,EAAE;QACjC,KAAK,EAAE,QAAQ;QACf,WAAW,EAAE,mCAAmC;QAChD,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACpE,OAAO,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;KAChC,CAAC;IACF,UAAU,CAAC,0BAA0B,EAAE;QACrC,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,8CAA8C;QAC3D,UAAU,EAAE;YACV,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;SACpB;QACD,OAAO,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;KAChC,CAAC;IACF,UAAU,CAAC,uBAAuB,EAAE;QAClC,KAAK,EAAE,OAAO;QACd,WAAW,EACT,6DAA6D;QAC/D,UAAU,EAAE;YACV,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACzC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACrB;QACD,OAAO,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;KAChC,CAAC;IACF,UAAU,CAAC,8BAA8B,EAAE;QACzC,KAAK,EAAE,eAAe;QACtB,WAAW,EAAE,+CAA+C;QAC5D,UAAU,EAAE;YACV,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;SACrB;QACD,OAAO,EAAE,EAAE;KACZ,CAAC;CACH,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAC/B,CAAC;AAEJ,sEAAsE;AACtE,SAAS,aAAa,CAAC,KAAyB,EAAE,MAAc;IAC9D,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACrE,CAAC;AAED,mFAAmF;AACnF,MAAM,kBAAkB,GAA+C;IACrE,IAAI,EAAE,WAAW;IACjB,MAAM,EAAE,sBAAsB;IAC9B,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;YACvD,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/C,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAAE,OAAO,KAAK,CAAC;YACjD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,mEAAmE;AACnE,MAAM,iBAAiB,GAAmB;IACxC;QACE,EAAE,EAAE,KAAK;QACT,IAAI,EAAE,4BAA4B;QAClC,QAAQ,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;QAC5B,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;KAClD;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,uBAAuB;QAC7B,QAAQ,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;QAC5B,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;KACzD;CACF,CAAC;AAEF,MAAM,iBAAiB,GAAmB;IACxC;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,QAAQ;QAChB,YAAY,EAAE,QAAQ;QACtB,YAAY,EAAE,OAAO;KACtB;CACF,CAAC;AAEF,kFAAkF;AAClF,SAAS,UAAU,CAAC,KAAqB,EAAE,KAAqB;IAC9D,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1D,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACtB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,YAAY;YACd,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,YAAY;SACf,CAAC;KACH,CAAC,CAAC;AACL,CAAC;AAED,MAAM,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AAE3E,2DAA2D;AAC3D,MAAM,mBAAmB,GAA+C;IACtE,IAAI,EAAE,gBAAgB;IACtB,MAAM,EAAE,8BAA8B;IACtC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,iBAAiB;CAC5E,CAAC;AAEF;;;;GAIG;AACH,MAAM,wBAAwB,GAC5B,mQAAmQ,CAAC;AAEtQ,6FAA6F;AAC7F,MAAM,qBAAqB,GAAgE;IACzF,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;IACrB,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE;IACzB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE;CACxB,CAAC;AAEF;;;;GAIG;AACH,MAAM,iBAAiB,GAAmB;IACxC;QACE,EAAE,EAAE,KAAK;QACT,IAAI,EAAE,4BAA4B;QAClC,QAAQ,EAAE,EAAE,GAAG,qBAAqB,CAAC,GAAG,EAAE;QAC1C,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;KAClD;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,uBAAuB;QAC7B,QAAQ,EAAE,EAAE,GAAG,qBAAqB,CAAC,MAAM,EAAE;QAC7C,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;KAChD;IACD;QACE,EAAE,EAAE,MAAM;QACV,IAAI,EAAE,8BAA8B;QACpC,QAAQ,EAAE,EAAE,GAAG,qBAAqB,CAAC,IAAI,EAAE;QAC3C,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;KACrD;CACF,CAAC;AAEF,MAAM,iBAAiB,GAAmB;IACxC;QACE,EAAE,EAAE,aAAa;QACjB,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,QAAQ;QAChB,YAAY,EAAE,QAAQ;QACtB,YAAY,EAAE,OAAO;KACtB;IACD;QACE,EAAE,EAAE,aAAa;QACjB,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,MAAM;QACd,YAAY,EAAE,QAAQ;QACtB,YAAY,EAAE,OAAO;KACtB;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAC/B;IACE;QACE,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,2DAA2D;QACxE,SAAS,EACP,oNAAoN;QACtN,YAAY,EAAE,GAAG,EAAE,CACjB,oBAAoB,CAAC,EAAE,YAAY,EAAE,sBAAsB,EAAE,CAAC;QAChE,MAAM,EAAE;YACN,aAAa,EAAE,CAAC,aAAa,EAAE,kBAAkB,CAAC;YAClD,cAAc,EAAE,CAAC,gBAAgB,CAAC;YAClC,QAAQ,EAAE,CAAC,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;YAC/C,cAAc,EAAE,IAAI;YACpB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE;gBACV;oBACE,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,0BAA0B;oBAClC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC;iBACtD;gBACD;oBACE,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,2BAA2B;oBACnC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,kBAAkB,CAAC,IAAI,CAAC;iBACvD;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,2BAA2B;oBACnC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,kBAAkB,CAAC,IAAI,CAAC;iBACvD;gBACD;oBACE,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,oBAAoB;oBAC5B,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC;iBACjC;gBACD,kBAAkB;aACnB;SACF;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,iDAAiD;QAC9D,SAAS,EACP,8LAA8L;QAChM,UAAU,EACR,2QAA2Q;QAC7Q,YAAY,EAAE,GAAG,EAAE,CACjB,oBAAoB,CAAC;YACnB,YAAY,EAAE,sBAAsB;YACpC,KAAK,EAAE,iBAAiB;YACxB,KAAK,EAAE,iBAAiB;SACzB,CAAC;QACJ,MAAM,EAAE;YACN,aAAa,EAAE,CAAC,aAAa,EAAE,kBAAkB,CAAC;YAClD,QAAQ,EAAE,CAAC,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;YAC/C,cAAc,EAAE,IAAI;YACpB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE;gBACV;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,iCAAiC;oBACzC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,kBAAkB,CAAC,IAAI,CAAC;iBACvD;gBACD;oBACE,IAAI,EAAE,YAAY;oBAClB,MAAM,EAAE,oCAAoC;oBAC5C,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;iBACxD;aACF;SACF;KACF;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EACT,+IAA+I;QACjJ,SAAS,EACP,wmBAAwmB;QAC1mB,YAAY,EAAE,GAAG,EAAE,CACjB,oBAAoB,CAAC;YACnB,YAAY,EAAE,sBAAsB;YACpC,KAAK,EAAE,iBAAiB;YACxB,KAAK,EAAE,iBAAiB;SACzB,CAAC;QACJ,MAAM,EAAE;YACN,aAAa,EAAE;gBACb,cAAc;gBACd,qBAAqB;gBACrB,gBAAgB;gBAChB,kBAAkB;gBAClB,cAAc;gBACd,mBAAmB;aACpB;YACD,QAAQ,EAAE;gBACR,CAAC,cAAc,EAAE,gBAAgB,CAAC;gBAClC,CAAC,cAAc,EAAE,qBAAqB,CAAC;aACxC;YACD,cAAc,EAAE,IAAI;YACpB,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE;gBACV;oBACE,IAAI,EAAE,cAAc;oBACpB,MAAM,EAAE,yCAAyC;oBACjD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;wBACV,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;wBACrD,MAAM,KAAK,GAAG,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;wBAC5C,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;oBAC9D,CAAC;iBACF;gBACD;oBACE,IAAI,EAAE,mBAAmB;oBACzB,MAAM,EAAE,qDAAqD;oBAC7D,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACV,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC;iBAClE;gBACD;oBACE,IAAI,EAAE,oBAAoB;oBAC1B,MAAM,EAAE,wDAAwD;oBAChE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACV,CAAC,CAAC,KAAK,CAAC,IAAI,CACV,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,MAAM,KAAK,QAAQ;wBACrB,CAAC,CAAC,MAAM,KAAK,MAAM;wBACnB,CAAC,CAAC,YAAY,KAAK,OAAO,CAC7B;iBACJ;gBACD;oBACE,IAAI,EAAE,mBAAmB;oBACzB,MAAM,EAAE,oEAAoE;oBAC5E,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;wBAChD,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;wBAC9C,IAAI,CAAC,IAAI;4BAAE,OAAO,KAAK,CAAC;wBACxB,MAAM,IAAI,GAAG,qBAAqB,CAAC,EAAE,CAAC,CAAC;wBACvC,OAAO,CACL,IAAI,CAAC,KAAK,CACR,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EACxB,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CACzB,GAAG,EAAE,CACP,CAAC;oBACJ,CAAC,CAAC;iBACL;gBACD;oBACE,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,gDAAgD;oBACxD,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;wBAChD,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;wBAC9C,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;oBACjE,CAAC,CAAC;iBACL;aACF;SACF;KACF;IACD;QACE,EAAE,EAAE,2BAA2B;QAC/B,WAAW,EACT,4GAA4G;QAC9G,SAAS,EAAE,wBAAwB;QACnC,YAAY,EAAE,GAAG,EAAE,CACjB,oBAAoB,CAAC;YACnB,YAAY,EAAE,sBAAsB;YACpC,KAAK,EAAE,iBAAiB;YACxB,KAAK,EAAE,iBAAiB;SACzB,CAAC;QACJ,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QAC5B,MAAM,EAAE;YACN,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,CAAC,mBAAmB,CAAC;YACjC,kBAAkB,EAAE;gBAClB;oBACE,IAAI,EAAE,eAAe;oBACrB,MAAM,EAAE,mDAAmD;oBAC3D,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;iBAC1C;aACF;SACF;KACF;IACD;QACE,EAAE,EAAE,2BAA2B;QAC/B,WAAW,EACT,6HAA6H;QAC/H,SAAS,EAAE,wBAAwB;QACnC,YAAY,EAAE,GAAG,EAAE,CACjB,oBAAoB,CAAC;YACnB,YAAY,EAAE,sBAAsB;YACpC,KAAK,EAAE,iBAAiB;YACxB,KAAK,EAAE,iBAAiB;SACzB,CAAC;QACJ,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE;QAChD,MAAM,EAAE;YACN,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,CAAC,mBAAmB,CAAC;YACjC,kBAAkB,EAAE;gBAClB;oBACE,IAAI,EAAE,mBAAmB;oBACzB,MAAM,EACJ,mGAAmG;oBACrG,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE;wBACjB,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CACpC,CAAC;wBACF,OAAO,CACL,OAAO,CAAC,MAAM,KAAK,CAAC;4BACpB,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO;4BAC/B,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAC7C,CAAC;oBACJ,CAAC;iBACF;aACF;SACF;KACF;CACF,CAAC"}

package/dist/evals/tool-loop-eval.d.ts

@@ -0,0 +1,236 @@
+/**
+ * Tool-loop evaluation harness — provider-agnostic.
+ *
+ * Drives a multi-turn tool-calling loop over the *frontend* tool surface: the
+ * model is handed the real `ui_*` tool contract (names/descriptions/Zod schemas
+ * from `@nodetool-ai/protocol`) and an objective, each requested tool runs
+ * against a {@link createToolLoopBridge headless bridge}, and the result is fed
+ * back — repeating until the model stops calling tools or a turn cap is hit.
+ *
+ * Where {@link runGraphPlannerEval} evaluates typed-DSL graph authoring, this
+ * evaluates the incremental add-node/connect-node tool flow the browser UI
+ * exposes. It records the same efficiency
+ * metrics and emits the same result/summary shapes so providers/models can be
+ * compared on one report.
+ *
+ * Scoring is structural (see {@link checkToolLoopExpectations}): required /
+ * forbidden tool names, ordering constraints, final-state predicates, tool-call
+ * budgets, and a no-error-results check — never an exact transcript match, so
+ * many valid tool orderings pass.
+ *
+ * Checks are not weighed alike. Each carries a severity — `critical` for the
+ * behavior the case exists to test, `advisory` for efficiency budgets — and
+ * {@link scoreToolLoopChecks} weighs them accordingly and caps a run that
+ * missed a critical check, so "built the wrong thing" can never outscore "built
+ * the right thing a bit wastefully".
+ *
+ * Severity also decides the suite's headline number: `successRate` counts the
+ * cases that completed *and* passed every critical check, so a model that
+ * called no tool at all cannot pass. How often the loop merely ran to a stop is
+ * reported separately as `completionRate`.
+ */
+import type { BaseProvider } from "@nodetool-ai/runtime";
+import { type ToolLoopCallRecord } from "../app-build/tool-loop.js";
+import type { EvalCheck } from "./graph-planner-eval.js";
+import type { HeadlessTool, ToolLoopFinalState } from "./tool-loop-bridge.js";
+import { type EscalationConfig, type EscalationExpectations, type EscalationTurn } from "./escalation.js";
+import { type PermissionRequestRecord, type ToolLoopPermission } from "./tool-loop-permission.js";
+/**
+ * The minimal contract every headless surface bridge (graph editor, script,
+ * sketch, timeline, storyboard, 3D) exposes to the runner: a set of executable
+ * {@link HeadlessTool tools} carrying the real frontend tool contract, plus a
+ * `finalState` snapshot the case's structural predicates run against. `TFinal`
+ * is surface-specific (graph → `{nodes, edges}`, timeline → clips/tracks, …).
+ */
+export interface HeadlessSurfaceBridge<TFinal = unknown> {
+    tools: HeadlessTool[];
+    finalState: () => TFinal;
+}
+/** One tool call the model made, with its result and whether it errored. */
+export type ToolCallRecord = ToolLoopCallRecord;
+/** A named boolean assertion over a surface's final state. */
+export interface ToolLoopStatePredicate<TFinal = unknown> {
+    name: string;
+    test: (state: TFinal) => boolean;
+    /** Optional detail shown when the predicate fails. */
+    detail?: string;
+}
+export interface ToolLoopEvalExpectations<TFinal = unknown> {
+    /** Tool names that must each be called at least once. */
+    requiredTools?: string[];
+    /** Tool names that must never be called. */
+    forbiddenTools?: string[];
+    /**
+     * Ordering constraints `[a, b]`: the first call of `a` must precede the first
+     * call of `b`. Both tools must be called for the check to pass.
+     */
+    ordering?: Array<[string, string]>;
+    /** Predicates on the final surface state (all must hold). */
+    finalState?: ToolLoopStatePredicate<TFinal>[];
+    /** Minimum total tool calls across the run. */
+    minToolCalls?: number;
+    /** Maximum total tool calls across the run (efficiency ceiling). */
+    maxToolCalls?: number;
+    /** When true, no tool call may have returned an error result. */
+    noErrorResults?: boolean;
+    /**
+     * Expectations over the interactive-escalation exchanges. Only meaningful
+     * when the case declares an {@link ToolLoopEvalCase.escalation} config.
+     * Note that escalation calls count toward `minToolCalls`/`maxToolCalls` like
+     * any other tool call.
+     */
+    escalation?: EscalationExpectations;
+    /**
+     * Predicates over the approval requests the permission gate made (all must
+     * hold). Only meaningful when the case declares a
+     * {@link ToolLoopEvalCase.permission} mode; without one the list is empty.
+     */
+    permissionRequests?: ToolLoopStatePredicate<readonly PermissionRequestRecord[]>[];
+}
+export interface ToolLoopEvalCase<TFinal = ToolLoopFinalState> {
+    id: string;
+    description: string;
+    objective: string;
+    /**
+     * Build the headless surface bridge (tools + final-state snapshot) this case
+     * drives. Called once per run, so each case starts from a fresh in-memory
+     * state.
+     */
+    createBridge: () => HeadlessSurfaceBridge<TFinal>;
+    /**
+     * Surface-specific system prompt. Falls back to the runner's `systemPrompt`
+     * option, then the graph-editor default.
+     */
+    systemPrompt?: string;
+    /** Override the user message (defaults to `Objective: <objective>`). */
+    userPrompt?: string;
+    /**
+     * Case needs configured model providers to be solvable — skipped when the
+     * harness runs without any (mirrors the graph-planner suite).
+     */
+    needsModelProviders?: boolean;
+    /**
+     * Hand the model an `ask_user` tool backed by a scripted user, making the
+     * case interactive: it can escalate an ambiguous or destructive decision and
+     * build on the answer. See `./escalation.ts`.
+     */
+    escalation?: EscalationConfig;
+    /**
+     * Run the belt through the permission gate in this mode, with a scripted
+     * user answering every approval prompt. A case without it is ungated, as
+     * before. See `./tool-loop-permission.ts`.
+     */
+    permission?: ToolLoopPermission;
+    /**
+     * Turn cap for this case, overriding the runner's `maxIterations`. A case
+     * whose work is inherently many-turned — drawing is dozens of strokes, not
+     * three property edits — declares the budget it needs rather than forcing
+     * every suite run to be invoked with a flag.
+     */
+    maxIterations?: number;
+    expect: ToolLoopEvalExpectations<TFinal>;
+}
+/** Everything the pure checker needs — no provider, no I/O. */
+export interface ToolLoopObservation<TFinal = unknown> {
+    toolCalls: ToolCallRecord[];
+    finalState: TFinal;
+    /** Question/answer exchanges, for cases with an escalation channel. */
+    escalations?: readonly EscalationTurn[];
+    /** Approval requests the gate made, for cases with a permission mode. */
+    permissionRequests?: readonly PermissionRequestRecord[];
+}
+export interface ToolLoopCaseResult {
+    /** Present when the surface explicitly reports that visual media was not verified. */
+    visual_output_verified?: false;
+    caseId: string;
+    description: string;
+    skipped: boolean;
+    /** The loop ran to a natural stop / cap without a fatal provider error. */
+    accepted: boolean;
+    /**
+     * The case did what it exists to test: the loop completed AND no `critical`
+     * check failed. This — not {@link accepted} — is what the suite's success
+     * rate and the `--min-success` gate read, because a model that made zero
+     * tool calls still "completes".
+     */
+    success: boolean;
+    /**
+     * Severity-weighted fraction of checks passed, capped at
+     * {@link CRITICAL_FAILURE_SCORE_CAP} when a critical check failed (0 when the
+     * loop did not run).
+     */
+    score: number;
+    /** Failing `critical` checks — the core behaviors the case exists to test. */
+    criticalFailures: number;
+    checks: EvalCheck[];
+    /** Tool calls made, by tool name. */
+    toolCalls: Record<string, number>;
+    /** Total tool calls across all names. */
+    totalToolCalls: number;
+    /** Approval requests the permission gate made; empty for an ungated case. */
+    permissionRequests: PermissionRequestRecord[];
+    durationMs: number;
+    costUsd: number;
+    error?: string;
+}
+export interface ToolLoopEvalReport {
+    provider: string;
+    model: string;
+    startedAt: string;
+    cases: ToolLoopCaseResult[];
+    summary: {
+        total: number;
+        skipped: number;
+        accepted: number;
+        /** Cases that completed with every critical check passing. */
+        successful: number;
+        /**
+         * successful / (total - skipped) — the gated metric. A run that merely
+         * finished without a provider error is not a success.
+         */
+        successRate: number;
+        /** accepted / (total - skipped): the loop ran, whatever it built. */
+        completionRate: number;
+        /** Mean expectation score over non-skipped cases. */
+        meanScore: number;
+        avgToolCalls: number;
+        totalCostUsd: number;
+    };
+}
+export interface RunToolLoopEvalOptions<TFinal = ToolLoopFinalState> {
+    provider: BaseProvider;
+    model: string;
+    /** Configured providers; enables model-dependent cases (else they skip). */
+    providers?: Record<string, BaseProvider>;
+    /** Cases to run; defaults to the built-in graph-editor suite. */
+    cases?: readonly ToolLoopEvalCase<TFinal>[];
+    /** Turn cap — max tool-calling rounds before the loop stops. Defaults to 12. */
+    maxIterations?: number;
+    /** Override the system prompt handed to the model (per-case wins over this). */
+    systemPrompt?: string;
+    signal?: AbortSignal;
+    /** Progress callback (one line per event, for CLI display). */
+    onEvent?: (line: string) => void;
+}
+/**
+ * Ceiling on a run that failed a `critical` check. Without it, a case with many
+ * passing state predicates can bury a core-behavior miss under partial credit.
+ */
+export declare const CRITICAL_FAILURE_SCORE_CAP = 0.5;
+/**
+ * Severity-weighted pass fraction, capped when any critical check failed.
+ * Exported so the scoring rule is testable on its own.
+ */
+export declare function scoreToolLoopChecks(checks: readonly EvalCheck[]): number;
+/** Failing checks marked `critical` — the core-behavior misses. */
+export declare function countCriticalFailures(checks: readonly EvalCheck[]): number;
+/**
+ * Score a completed run against a case's structural expectations. Pure and
+ * fully unit-testable: it takes an {@link ToolLoopObservation} and never calls
+ * a provider.
+ */
+export declare function checkToolLoopExpectations<TFinal>(observation: ToolLoopObservation<TFinal>, expect: ToolLoopEvalExpectations<TFinal>): EvalCheck[];
+export declare function runToolLoopEval<TFinal = ToolLoopFinalState>(opts: RunToolLoopEvalOptions<TFinal>): Promise<ToolLoopEvalReport>;
+/** Text summary table for terminal output. */
+export declare function formatToolLoopReport(report: ToolLoopEvalReport): string;
+//# sourceMappingURL=tool-loop-eval.d.ts.map

package/dist/evals/tool-loop-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-loop-eval.d.ts","sourceRoot":"","sources":["../../src/evals/tool-loop-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAEL,KAAK,kBAAkB,EACxB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE9E,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEL,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACxB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB,CAAC,MAAM,GAAG,OAAO;IACrD,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,MAAM,CAAC;CAC1B;AAED,4EAA4E;AAC5E,MAAM,MAAM,cAAc,GAAG,kBAAkB,CAAC;AAEhD,8DAA8D;AAC9D,MAAM,WAAW,sBAAsB,CAAC,MAAM,GAAG,OAAO;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;IACjC,sDAAsD;IACtD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB,CAAC,MAAM,GAAG,OAAO;IACxD,yDAAyD;IACzD,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACnC,6DAA6D;IAC7D,UAAU,CAAC,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;IAC9C,+CAA+C;IAC/C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,sBAAsB,CACzC,SAAS,uBAAuB,EAAE,CACnC,EAAE,CAAC;CACL;AAED,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,kBAAkB;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,YAAY,EAAE,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAClD;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;OAIG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;OAIG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,wBAAwB,CAAC,MAAM,CAAC,CAAC;CAC1C;AAED,+DAA+D;AAC/D,MAAM,WAAW,mBAAmB,CAAC,MAAM,GAAG,OAAO;IACnD,SAAS,EAAE,cAAc,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,uEAAuE;IACvE,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,yEAAyE;IACzE,kBAAkB,CAAC,EAAE,SAAS,uBAAuB,EAAE,CAAC;CACzD;AAED,MAAM,WAAW,kBAAkB;IACjC,sFAAsF;IACtF,sBAAsB,CAAC,EAAE,KAAK,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,2EAA2E;IAC3E,QAAQ,EAAE,OAAO,CAAC;IAClB;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,kBAAkB,EAAE,uBAAuB,EAAE,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,8DAA8D;QAC9D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,WAAW,EAAE,MAAM,CAAC;QACpB,qEAAqE;QACrE,cAAc,EAAE,MAAM,CAAC;QACvB,qDAAqD;QACrD,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,sBAAsB,CAAC,MAAM,GAAG,kBAAkB;IACjE,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACzC,iEAAiE;IACjE,KAAK,CAAC,EAAE,SAAS,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;IAC5C,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gFAAgF;IAChF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAgCD;;;GAGG;AACH,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAO9C;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,GAAG,MAAM,CAWxE;AAED,mEAAmE;AACnE,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,GAAG,MAAM,CAE1E;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAC9C,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,EACxC,MAAM,EAAE,wBAAwB,CAAC,MAAM,CAAC,GACvC,SAAS,EAAE,CAqHb;AAoFD,wBAAsB,eAAe,CAAC,MAAM,GAAG,kBAAkB,EAC/D,IAAI,EAAE,sBAAsB,CAAC,MAAM,CAAC,GACnC,OAAO,CAAC,kBAAkB,CAAC,CAoE7B;AAED,8CAA8C;AAC9C,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,kBAAkB,GAAG,MAAM,CAsDvE"}