@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/evals/subtask-cases.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Cases + instrumented tool library for the sub-agent execution eval.
+ *
+ * Where the tool-loop suites score a model driving a single flat tool surface,
+ * this suite scores {@link RunSubtaskTool} — the primitive that lets an agent
+ * decompose work by spawning a fresh child agent that inherits the parent's
+ * toolset. Each case hands the parent an objective it is expected to delegate,
+ * and the scoring checks that the *child* agent actually ran the inherited
+ * tools (not the parent), plus depth, subtask count, and error propagation.
+ *
+ * The tools are {@link InstrumentedTool}s: real {@link Tool} subclasses that
+ * record every invocation together with the sub-agent depth
+ * ({@link SUBTASK_DEPTH_KEY}) at which it ran — 0 for a parent-level call, >= 1
+ * for a call made inside a subtask. That single field is what lets the scorer
+ * distinguish "the parent did it itself" from "the parent delegated and the
+ * child did it".
+ */
+import { Tool } from "../tools/base-tool.js";
+/** One recorded tool invocation, tagged with the depth it ran at. */
+export interface ToolInvocation {
+    name: string;
+    args: Record<string, unknown>;
+    /** `SUBTASK_DEPTH_KEY` at call time: 0 = parent, >= 1 = inside a subtask. */
+    depth: number;
+    result: unknown;
+    isError: boolean;
+}
+/** Shared, per-run recorder handed to every instrumented tool. */
+export interface ToolRecorder {
+    invocations: ToolInvocation[];
+    /** Key/value scratch store the write/read tools mutate. */
+    store: Map<string, string>;
+}
+export declare function createToolRecorder(): ToolRecorder;
+/**
+ * The six instrumented tools every case shares. One per interaction shape a
+ * sub-agent has to handle: compute, side-effecting write, dependent read,
+ * canned retrieval, pure transform, and a tool that always errors (for the
+ * error-propagation case).
+ */
+export declare function createInstrumentedTools(recorder: ToolRecorder): Tool[];
+/** The tool names the instrumented library exposes. */
+export declare const INSTRUMENTED_TOOL_NAMES: readonly ["calculate", "kv_write", "kv_read", "lookup_fact", "slugify", "flaky_fail"];
+/** Structural expectations for a sub-agent execution case. */
+export interface SubtaskEvalExpectations {
+    /** Tool names that must be called at least once at the parent level (depth 0). */
+    requiredParentTools?: string[];
+    /**
+     * Tool names that must be called at least once *inside a subtask* (depth >=
+     * 1) — the crux of the eval: proof the delegated child, not the parent, did
+     * the work.
+     */
+    requiredChildTools?: string[];
+    /** Tool names that must never be called at any depth. */
+    forbiddenTools?: string[];
+    /** Minimum number of subtasks the parent must spawn (run_subtask calls). */
+    minSubtasks?: number;
+    /** Maximum number of subtasks (efficiency ceiling). */
+    maxSubtasks?: number;
+    /** Minimum sub-agent depth that must be reached (>= 1 means it delegated). */
+    minDepth?: number;
+    /** When true, no spawned subtask may have failed. */
+    noSubtaskErrors?: boolean;
+    /** Keys the shared store must contain when the run ends. */
+    requiredStoreKeys?: string[];
+    /**
+     * Substrings the parent's final answer must contain (case-insensitive) —
+     * proof the delegated result flowed back into the parent's response.
+     */
+    answerContains?: string[];
+}
+export interface SubtaskEvalCase {
+    id: string;
+    description: string;
+    /** The parent agent's objective (its step instructions). */
+    objective: string;
+    /** Optional system-prompt preamble for the parent step. */
+    systemPrompt?: string;
+    /** Case needs a real model provider to be solvable (all of them do). */
+    needsModelProviders?: boolean;
+    expect: SubtaskEvalExpectations;
+}
+/**
+ * The sub-agent execution suite. Every case forces delegation and then checks
+ * that the child agent exercised the inherited tools.
+ */
+export declare const SUBTASK_EVAL_CASES: readonly SubtaskEvalCase[];
+//# sourceMappingURL=subtask-cases.d.ts.map

package/dist/evals/subtask-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"subtask-cases.d.ts","sourceRoot":"","sources":["../../src/evals/subtask-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAQ7C,qEAAqE;AACrE,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,kEAAkE;AAClE,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,2DAA2D;IAC3D,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC5B;AAED,wBAAgB,kBAAkB,IAAI,YAAY,CAEjD;AAiED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,EAAE,CA4ItE;AAED,uDAAuD;AACvD,eAAO,MAAM,uBAAuB,YAClC,WAAW,EACX,UAAU,EACV,SAAS,EACT,aAAa,EACb,SAAS,EACT,YAAY,CACJ,CAAC;AAEX,8DAA8D;AAC9D,MAAM,WAAW,uBAAuB;IACtC,kFAAkF;IAClF,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4DAA4D;IAC5D,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wEAAwE;IACxE,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,EAAE,uBAAuB,CAAC;CACjC;AAWD;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,SAAS,eAAe,EA2HxD,CAAC"}

package/dist/evals/subtask-cases.js

@@ -0,0 +1,322 @@
+/**
+ * Cases + instrumented tool library for the sub-agent execution eval.
+ *
+ * Where the tool-loop suites score a model driving a single flat tool surface,
+ * this suite scores {@link RunSubtaskTool} — the primitive that lets an agent
+ * decompose work by spawning a fresh child agent that inherits the parent's
+ * toolset. Each case hands the parent an objective it is expected to delegate,
+ * and the scoring checks that the *child* agent actually ran the inherited
+ * tools (not the parent), plus depth, subtask count, and error propagation.
+ *
+ * The tools are {@link InstrumentedTool}s: real {@link Tool} subclasses that
+ * record every invocation together with the sub-agent depth
+ * ({@link SUBTASK_DEPTH_KEY}) at which it ran — 0 for a parent-level call, >= 1
+ * for a call made inside a subtask. That single field is what lets the scorer
+ * distinguish "the parent did it itself" from "the parent delegated and the
+ * child did it".
+ */
+import { Tool } from "../tools/base-tool.js";
+import { SUBTASK_DEPTH_KEY } from "../tools/subtask-fields.js";
+import { isNumber, isObjectLike, isString } from "../utils/type-guards.js";
+export function createToolRecorder() {
+    return { invocations: [], store: new Map() };
+}
+/**
+ * A real {@link Tool} that records each call (with its subtask depth) into a
+ * shared {@link ToolRecorder}. The same instance is shared between the parent
+ * toolset and the inherited child toolset, so the depth read from `context`
+ * tells us which agent made the call.
+ */
+class InstrumentedTool extends Tool {
+    name;
+    description;
+    jsonSchema;
+    recorder;
+    handler;
+    constructor(name, description, schema, recorder, handler) {
+        super();
+        this.name = name;
+        this.description = description;
+        this.jsonSchema = schema;
+        this.recorder = recorder;
+        this.handler = handler;
+    }
+    async process(context, params) {
+        const depth = context.get(SUBTASK_DEPTH_KEY) ?? 0;
+        let result;
+        let isError = false;
+        try {
+            result = this.handler(params, { store: this.recorder.store, depth });
+        }
+        catch (e) {
+            isError = true;
+            result = { error: e instanceof Error ? e.message : String(e) };
+        }
+        if (isObjectLike(result) &&
+            "error" in result) {
+            isError = true;
+        }
+        this.recorder.invocations.push({
+            name: this.name,
+            args: params,
+            depth,
+            result,
+            isError
+        });
+        return result;
+    }
+}
+/**
+ * The six instrumented tools every case shares. One per interaction shape a
+ * sub-agent has to handle: compute, side-effecting write, dependent read,
+ * canned retrieval, pure transform, and a tool that always errors (for the
+ * error-propagation case).
+ */
+export function createInstrumentedTools(recorder) {
+    const num = (v) => {
+        const n = isNumber(v) ? v : Number(v);
+        if (!Number.isFinite(n))
+            throw new Error(`not a number: ${String(v)}`);
+        return n;
+    };
+    const str = (v) => (isString(v) ? v : String(v));
+    const FACTS = {
+        "capital of france": "Paris",
+        "speed of light": "299792458 m/s",
+        "largest planet": "Jupiter"
+    };
+    return [
+        new InstrumentedTool("calculate", "Evaluate a single arithmetic operation on two numbers.", {
+            type: "object",
+            properties: {
+                a: { type: "number", description: "Left operand." },
+                op: {
+                    type: "string",
+                    enum: ["add", "subtract", "multiply", "divide"],
+                    description: "Operation to apply."
+                },
+                b: { type: "number", description: "Right operand." }
+            },
+            required: ["a", "op", "b"],
+            additionalProperties: false
+        }, recorder, (args) => {
+            const a = num(args.a);
+            const b = num(args.b);
+            switch (str(args.op)) {
+                case "add":
+                    return { result: a + b };
+                case "subtract":
+                    return { result: a - b };
+                case "multiply":
+                    return { result: a * b };
+                case "divide":
+                    if (b === 0)
+                        return { error: "division by zero" };
+                    return { result: a / b };
+                default:
+                    return { error: `unknown op: ${str(args.op)}` };
+            }
+        }),
+        new InstrumentedTool("kv_write", "Store a string value under a key in the shared scratch store.", {
+            type: "object",
+            properties: {
+                key: { type: "string", description: "Key to write." },
+                value: { type: "string", description: "Value to store." }
+            },
+            required: ["key", "value"],
+            additionalProperties: false
+        }, recorder, (args, { store }) => {
+            store.set(str(args.key), str(args.value));
+            return { ok: true, key: str(args.key) };
+        }),
+        new InstrumentedTool("kv_read", "Read the value previously stored under a key.", {
+            type: "object",
+            properties: {
+                key: { type: "string", description: "Key to read." }
+            },
+            required: ["key"],
+            additionalProperties: false
+        }, recorder, (args, { store }) => {
+            const key = str(args.key);
+            if (!store.has(key))
+                return { error: `no value for key: ${key}` };
+            return { key, value: store.get(key) };
+        }),
+        new InstrumentedTool("lookup_fact", "Look up a canned fact by query. Known queries: capital of france, speed of light, largest planet.", {
+            type: "object",
+            properties: {
+                query: { type: "string", description: "The fact to look up." }
+            },
+            required: ["query"],
+            additionalProperties: false
+        }, recorder, (args) => {
+            const q = str(args.query).toLowerCase().trim();
+            if (q in FACTS)
+                return { query: q, fact: FACTS[q] };
+            return { error: `unknown fact: ${q}` };
+        }),
+        new InstrumentedTool("slugify", "Transform text into a lowercase, dash-separated slug.", {
+            type: "object",
+            properties: {
+                text: { type: "string", description: "Text to slugify." }
+            },
+            required: ["text"],
+            additionalProperties: false
+        }, recorder, (args) => {
+            const slug = str(args.text)
+                .toLowerCase()
+                .replace(/[^a-z0-9]+/g, "-")
+                .replace(/^-+|-+$/g, "");
+            return { slug };
+        }),
+        new InstrumentedTool("flaky_fail", "A tool that always fails. Call it when asked to test error handling.", {
+            type: "object",
+            properties: {
+                reason: { type: "string", description: "Why you are calling it." }
+            },
+            required: [],
+            additionalProperties: false
+        }, recorder, () => ({ error: "flaky_fail always fails by design" }))
+    ];
+}
+/** The tool names the instrumented library exposes. */
+export const INSTRUMENTED_TOOL_NAMES = [
+    "calculate",
+    "kv_write",
+    "kv_read",
+    "lookup_fact",
+    "slugify",
+    "flaky_fail"
+];
+const DELEGATION_PREAMBLE = [
+    "You are an orchestrator. You do NOT do work yourself — you delegate every",
+    "concrete task to a focused sub-agent by calling the `run_subtask` tool with",
+    "a self-contained `prompt`. The sub-agent inherits your tools and will run",
+    "them. When a subtask returns, use its result to answer. Do not call the",
+    "worker tools (calculate, kv_write, kv_read, lookup_fact, slugify) directly —",
+    "always go through `run_subtask`."
+].join(" ");
+/**
+ * The sub-agent execution suite. Every case forces delegation and then checks
+ * that the child agent exercised the inherited tools.
+ */
+export const SUBTASK_EVAL_CASES = [
+    {
+        id: "delegate-compute",
+        description: "Delegate an arithmetic task; child must use `calculate`",
+        objective: "Compute 47 multiplied by 89 and report the exact numeric result.",
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["calculate"],
+            minSubtasks: 1,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            answerContains: ["4183"]
+        }
+    },
+    {
+        id: "delegate-read-write",
+        description: "Delegate a write-then-read task; child must use kv_write + kv_read",
+        objective: "Store the value \"emerald\" under the key \"gem\", then read it back and report the value you read.",
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["kv_write", "kv_read"],
+            minSubtasks: 1,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            requiredStoreKeys: ["gem"],
+            answerContains: ["emerald"]
+        }
+    },
+    {
+        id: "delegate-lookup",
+        description: "Delegate a retrieval task; child must use `lookup_fact`",
+        objective: "Find out the capital of France using the fact lookup tool and report it.",
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["lookup_fact"],
+            minSubtasks: 1,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            answerContains: ["paris"]
+        }
+    },
+    {
+        id: "delegate-transform",
+        description: "Delegate a text transform; child must use `slugify`",
+        objective: "Turn the title \"Hello World Example\" into a URL slug and report the slug.",
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["slugify"],
+            minSubtasks: 1,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            answerContains: ["hello-world-example"]
+        }
+    },
+    {
+        id: "parallel-subtasks",
+        description: "Two independent tasks; parent should spawn multiple subtasks",
+        objective: [
+            "Do two independent things and report both answers:",
+            "(1) compute 12 plus 30, and",
+            "(2) look up the largest planet.",
+            "Delegate each as its own separate subtask."
+        ].join(" "),
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["calculate", "lookup_fact"],
+            minSubtasks: 2,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            answerContains: ["42", "jupiter"]
+        }
+    },
+    {
+        id: "error-propagation",
+        description: "Child hits a failing tool; the error must surface, not be hidden",
+        objective: "Call the flaky_fail tool once inside a subtask to test error handling, then report in your answer whether the tool succeeded or failed.",
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: ["flaky_fail"],
+            minSubtasks: 1,
+            minDepth: 1,
+            answerContains: ["fail"]
+        }
+    },
+    {
+        id: "all-tools",
+        description: "One objective that exercises every inherited tool via subtasks",
+        objective: [
+            "Complete all of the following, delegating the work to subtasks:",
+            "1. compute 8 multiplied by 8,",
+            "2. store the result under the key \"square\" and read it back,",
+            "3. look up the speed of light,",
+            "4. slugify the phrase \"Final Report\".",
+            "Report every result."
+        ].join(" "),
+        systemPrompt: DELEGATION_PREAMBLE,
+        expect: {
+            requiredParentTools: ["run_subtask"],
+            requiredChildTools: [
+                "calculate",
+                "kv_write",
+                "kv_read",
+                "lookup_fact",
+                "slugify"
+            ],
+            minSubtasks: 1,
+            minDepth: 1,
+            noSubtaskErrors: true,
+            requiredStoreKeys: ["square"],
+            answerContains: ["64", "final-report"]
+        }
+    }
+];
+//# sourceMappingURL=subtask-cases.js.map

package/dist/evals/subtask-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"subtask-cases.js","sourceRoot":"","sources":["../../src/evals/subtask-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,QAAQ,EACT,MAAM,yBAAyB,CAAC;AAmBjC,MAAM,UAAU,kBAAkB;IAChC,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AAC/C,CAAC;AAOD;;;;;GAKG;AACH,MAAM,gBAA0B,SAAQ,IAAI;IACjC,IAAI,CAAS;IACb,WAAW,CAAS;IACpB,UAAU,CAAa;IACf,QAAQ,CAAe;IACvB,OAAO,CAAuB;IAE/C,YACE,IAAY,EACZ,WAAmB,EACnB,MAAkB,EAClB,QAAsB,EACtB,OAA6B;QAE7B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,OAA0B,EAC1B,MAA+B;QAE/B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAS,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,MAAmC,CAAC;QACxC,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,GAAG,EAAE,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,CAAC;QACD,IACE,YAAY,CAAC,MAAM,CAAC;YACpB,OAAO,IAAK,MAAkC,EAC9C,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC;YAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,MAAM;YACZ,KAAK;YACL,MAAM;YACN,OAAO;SACR,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,QAAsB;IAC5D,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE;QACjC,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,MAAM,KAAK,GAA2B;QACpC,mBAAmB,EAAE,OAAO;QAC5B,gBAAgB,EAAE,eAAe;QACjC,gBAAgB,EAAE,SAAS;KAC5B,CAAC;IAEF,OAAO;QACL,IAAI,gBAAgB,CAClB,WAAW,EACX,wDAAwD,EACxD;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE;gBACnD,EAAE,EAAE;oBACF,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,CAAC;oBAC/C,WAAW,EAAE,qBAAqB;iBACnC;gBACD,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE;aACrD;YACD,QAAQ,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC;YAC1B,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,CAAC,IAAI,EAAE,EAAE;YACP,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,QAAQ,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrB,KAAK,KAAK;oBACR,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,UAAU;oBACb,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,UAAU;oBACb,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,KAAK,QAAQ;oBACX,IAAI,CAAC,KAAK,CAAC;wBAAE,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC;oBAClD,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B;oBACE,OAAO,EAAE,KAAK,EAAE,eAAe,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,CAAC;QACH,CAAC,CACF;QACD,IAAI,gBAAgB,CAClB,UAAU,EACV,+DAA+D,EAC/D;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE;gBACrD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE;aAC1D;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC;YAC1B,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;YAClB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1C,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,CAAC,CACF;QACD,IAAI,gBAAgB,CAClB,SAAS,EACT,+CAA+C,EAC/C;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;aACrD;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;YACjB,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;YAClB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,qBAAqB,GAAG,EAAE,EAAE,CAAC;YAClE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACxC,CAAC,CACF;QACD,IAAI,gBAAgB,CAClB,aAAa,EACb,mGAAmG,EACnG;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE;aAC/D;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;YACnB,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,CAAC,IAAI,EAAE,EAAE;YACP,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/C,IAAI,CAAC,IAAI,KAAK;gBAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,iBAAiB,CAAC,EAAE,EAAE,CAAC;QACzC,CAAC,CACF;QACD,IAAI,gBAAgB,CAClB,SAAS,EACT,uDAAuD,EACvD;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE;aAC1D;YACD,QAAQ,EAAE,CAAC,MAAM,CAAC;YAClB,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,CAAC,IAAI,EAAE,EAAE;YACP,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;iBACxB,WAAW,EAAE;iBACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;iBAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,CAAC;QAClB,CAAC,CACF;QACD,IAAI,gBAAgB,CAClB,YAAY,EACZ,sEAAsE,EACtE;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,yBAAyB,EAAE;aACnE;YACD,QAAQ,EAAE,EAAE;YACZ,oBAAoB,EAAE,KAAK;SAC5B,EACD,QAAQ,EACR,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CACvD;KACF,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,WAAW;IACX,UAAU;IACV,SAAS;IACT,aAAa;IACb,SAAS;IACT,YAAY;CACJ,CAAC;AA2CX,MAAM,mBAAmB,GAAG;IAC1B,2EAA2E;IAC3E,6EAA6E;IAC7E,2EAA2E;IAC3E,yEAAyE;IACzE,8EAA8E;IAC9E,kCAAkC;CACnC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAA+B;IAC5D;QACE,EAAE,EAAE,kBAAkB;QACtB,WAAW,EAAE,yDAAyD;QACtE,SAAS,EACP,kEAAkE;QACpE,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,WAAW,CAAC;YACjC,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,MAAM,CAAC;SACzB;KACF;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,WAAW,EAAE,oEAAoE;QACjF,SAAS,EACP,qGAAqG;QACvG,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;YAC3C,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,iBAAiB,EAAE,CAAC,KAAK,CAAC;YAC1B,cAAc,EAAE,CAAC,SAAS,CAAC;SAC5B;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,yDAAyD;QACtE,SAAS,EACP,0EAA0E;QAC5E,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,aAAa,CAAC;YACnC,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,OAAO,CAAC;SAC1B;KACF;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,qDAAqD;QAClE,SAAS,EACP,6EAA6E;QAC/E,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,SAAS,CAAC;YAC/B,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,qBAAqB,CAAC;SACxC;KACF;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,8DAA8D;QAC3E,SAAS,EAAE;YACT,oDAAoD;YACpD,6BAA6B;YAC7B,iCAAiC;YACjC,4CAA4C;SAC7C,CAAC,IAAI,CAAC,GAAG,CAAC;QACX,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,WAAW,EAAE,aAAa,CAAC;YAChD,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,cAAc,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC;SAClC;KACF;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,kEAAkE;QAC/E,SAAS,EACP,yIAAyI;QAC3I,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE,CAAC,YAAY,CAAC;YAClC,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,cAAc,EAAE,CAAC,MAAM,CAAC;SACzB;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,gEAAgE;QAC7E,SAAS,EAAE;YACT,iEAAiE;YACjE,+BAA+B;YAC/B,gEAAgE;YAChE,gCAAgC;YAChC,yCAAyC;YACzC,sBAAsB;SACvB,CAAC,IAAI,CAAC,GAAG,CAAC;QACX,YAAY,EAAE,mBAAmB;QACjC,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,aAAa,CAAC;YACpC,kBAAkB,EAAE;gBAClB,WAAW;gBACX,UAAU;gBACV,SAAS;gBACT,aAAa;gBACb,SAAS;aACV;YACD,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,CAAC;YACX,eAAe,EAAE,IAAI;YACrB,iBAAiB,EAAE,CAAC,QAAQ,CAAC;YAC7B,cAAc,EAAE,CAAC,IAAI,EAAE,cAAc,CAAC;SACvC;KACF;CACF,CAAC"}

package/dist/evals/subtask-eval.d.ts

@@ -0,0 +1,129 @@
+/**
+ * Sub-agent execution evaluation harness — provider-agnostic.
+ *
+ * Drives a real parent agent ({@link CodeActExecutor}) equipped with the
+ * {@link RunSubtaskTool} plus a library of {@link createInstrumentedTools
+ * instrumented tools}. Each case's objective is written so the parent must
+ * delegate to a child sub-agent; the harness then scores whether the *child*
+ * (depth >= 1), not the parent, actually ran the inherited tools, plus subtask
+ * count, recursion depth, error propagation, and whether the delegated result
+ * flowed back into the parent's final answer.
+ *
+ * Unlike the tool-loop suites (headless bridges over a flat `ui_*` surface),
+ * this exercises the genuine recursion machinery: `RunSubtaskTool` spins up a
+ * fresh executor with a copied, depth-bumped context and the parent's
+ * toolset. The instrumented tools are the same instances at both levels, so the
+ * `SUBTASK_DEPTH_KEY` each call reads from its context is the ground truth for
+ * "who ran this".
+ *
+ * Scoring is structural (see {@link checkSubtaskExpectations}) — required
+ * parent/child tools, forbidden tools, subtask-count and depth bounds, no
+ * failed subtasks, required store keys, and answer substrings — never an exact
+ * transcript, so many valid delegations pass.
+ */
+import type { BaseProvider } from "@nodetool-ai/runtime";
+import type { EvalCheck } from "./graph-planner-eval.js";
+import { type SubtaskEvalCase, type SubtaskEvalExpectations } from "./subtask-cases.js";
+/** One spawned subtask and how it resolved. */
+export interface SubtaskSpawnRecord {
+    description: string;
+    /** True when `run_subtask` returned an error envelope. */
+    failed: boolean;
+    errorCode: string | null;
+    /** The subtask's returned value coerced to text (the result the parent saw). */
+    resultText: string;
+}
+/** Everything the pure checker needs — no provider, no I/O. */
+export interface SubtaskObservation {
+    /** Tool names the parent (depth 0) called directly, plus `run_subtask`. */
+    parentTools: Set<string>;
+    /** Tool names called inside a subtask (depth >= 1). */
+    childTools: Set<string>;
+    /** Every tool name called at any depth (incl. `run_subtask`). */
+    allTools: Set<string>;
+    /** Subtasks the parent spawned (excludes depth-refused / bad-arg calls). */
+    spawns: SubtaskSpawnRecord[];
+    /** Deepest sub-agent level a tool actually ran at. */
+    maxDepth: number;
+    /** Keys present in the shared store at the end of the run. */
+    storeKeys: Set<string>;
+    /** The parent agent's final answer text. */
+    answer: string;
+    /** Text each spawned subtask returned to the parent. */
+    subtaskResults: string[];
+}
+export interface SubtaskCaseResult {
+    caseId: string;
+    description: string;
+    skipped: boolean;
+    /** The parent loop ran to a natural stop without a fatal provider error. */
+    accepted: boolean;
+    /**
+     * Accepted AND every expectation check passed. This — not {@link accepted} —
+     * is what the suite's success rate and the `--min-success` gate read. A
+     * parent that answered correctly by doing the work itself is live but has
+     * failed the only thing a delegation case measures, and reporting that as a
+     * success made the suite a scoreboard that could not go red.
+     */
+    success: boolean;
+    /** Expectation checks that failed. Zero is required for {@link success}. */
+    criticalFailures: number;
+    score: number;
+    checks: EvalCheck[];
+    /** Number of subtasks spawned. */
+    subtasks: number;
+    /** Deepest sub-agent level reached. */
+    maxDepth: number;
+    /** Tool calls made at any depth (incl. run_subtask). */
+    totalToolCalls: number;
+    durationMs: number;
+    costUsd: number;
+    error?: string;
+}
+export interface SubtaskEvalReport {
+    provider: string;
+    model: string;
+    startedAt: string;
+    cases: SubtaskCaseResult[];
+    summary: {
+        total: number;
+        skipped: number;
+        accepted: number;
+        /** Cases that were accepted AND passed every expectation check. */
+        successful: number;
+        /** success / (total - skipped) — the gated metric. */
+        successRate: number;
+        /**
+         * accepted / (total - skipped). A liveness signal, not a result: a parent
+         * that called no tools at all still scores 100% on it.
+         */
+        completionRate: number;
+        /** Mean expectation score over non-skipped cases. */
+        meanScore: number;
+        avgSubtasks: number;
+        totalCostUsd: number;
+    };
+}
+export interface RunSubtaskEvalOptions {
+    provider: BaseProvider;
+    model: string;
+    /** Configured providers; enables model-dependent cases (else they skip). */
+    providers?: Record<string, BaseProvider>;
+    /** Cases to run; defaults to the built-in suite. */
+    cases?: readonly SubtaskEvalCase[];
+    /** Turn cap for the parent step and each subtask. Defaults to 16. */
+    maxIterations?: number;
+    signal?: AbortSignal;
+    /** Progress callback (one line per event, for CLI display). */
+    onEvent?: (line: string) => void;
+}
+/**
+ * Score a completed run against a case's structural expectations. Pure and
+ * fully unit-testable: it takes a {@link SubtaskObservation} and never calls a
+ * provider.
+ */
+export declare function checkSubtaskExpectations(obs: SubtaskObservation, expect: SubtaskEvalExpectations): EvalCheck[];
+export declare function runSubtaskEval(opts: RunSubtaskEvalOptions): Promise<SubtaskEvalReport>;
+/** Text summary table for terminal output. */
+export declare function formatSubtaskReport(report: SubtaskEvalReport): string;
+//# sourceMappingURL=subtask-eval.d.ts.map

package/dist/evals/subtask-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"subtask-eval.d.ts","sourceRoot":"","sources":["../../src/evals/subtask-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAKzD,OAAO,EAIL,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAE7B,MAAM,oBAAoB,CAAC;AAK5B,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,+DAA+D;AAC/D,MAAM,WAAW,kBAAkB;IACjC,2EAA2E;IAC3E,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,uDAAuD;IACvD,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,iEAAiE;IACjE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,4EAA4E;IAC5E,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvB,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,4EAA4E;IAC5E,QAAQ,EAAE,OAAO,CAAC;IAClB;;;;;;OAMG;IACH,OAAO,EAAE,OAAO,CAAC;IACjB,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,mEAAmE;QACnE,UAAU,EAAE,MAAM,CAAC;QACnB,sDAAsD;QACtD,WAAW,EAAE,MAAM,CAAC;QACpB;;;WAGG;QACH,cAAc,EAAE,MAAM,CAAC;QACvB,qDAAqD;QACrD,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACzC,oDAAoD;IACpD,KAAK,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACnC,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AA6CD;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,kBAAkB,EACvB,MAAM,EAAE,uBAAuB,GAC9B,SAAS,EAAE,CA0Fb;AA6JD,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAoE5B;AAED,8CAA8C;AAC9C,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CA4CrE"}