@nodetool-ai/agents 0.7.0-rc.23 → 0.8.1

package/dist/evals/codeact-eval.js

@@ -0,0 +1,250 @@
+/**
+ * CodeAct evaluation harness — provider-agnostic.
+ *
+ * Drives a real {@link CodeActExecutor} over the instrumented offline
+ * toolbelt in `codeact-cases.ts` and scores each run structurally: required
+ * tools invoked through the bridge, action-round and tool-call bounds, and a
+ * predicate over the final result.
+ */
+import { randomUUID } from "node:crypto";
+import { ProcessingContext } from "@nodetool-ai/runtime";
+import { CodeActExecutor, EXECUTE_CODE_TOOL_NAME } from "../codeact/codeact-executor.js";
+import { CODEACT_EVAL_CASES, createCodeActRecorder, createCodeActTools } from "./codeact-cases.js";
+import { shippedPackCatalog } from "./codeact-sandbox-pack-cases.js";
+const DEFAULT_MAX_ITERATIONS = 8;
+/** Pure scorer over one run's observation. */
+export function checkCodeActExpectations(obs, expect) {
+    const checks = [];
+    for (const name of expect.requiredTools ?? []) {
+        const pass = obs.toolsInvoked.has(name);
+        checks.push({
+            name: `tool:${name}`,
+            pass,
+            detail: pass ? undefined : `never invoked ${name}`
+        });
+    }
+    for (const name of expect.forbiddenTools ?? []) {
+        const hit = obs.toolsInvoked.has(name);
+        checks.push({
+            name: `not-tool:${name}`,
+            pass: !hit,
+            detail: hit ? `invoked forbidden tool ${name}` : undefined
+        });
+    }
+    for (const name of expect.requiredSessionTools ?? []) {
+        const pass = obs.sessionToolsInvoked.has(name);
+        checks.push({
+            name: `session-tool:${name}`,
+            pass,
+            detail: pass ? undefined : `never invoked ${name}`
+        });
+    }
+    if (expect.maxActions !== undefined) {
+        checks.push({
+            name: `actions<=${expect.maxActions}`,
+            pass: obs.actions <= expect.maxActions,
+            detail: `took ${obs.actions}`
+        });
+    }
+    if (expect.minToolCalls !== undefined) {
+        checks.push({
+            name: `toolCalls>=${expect.minToolCalls}`,
+            pass: obs.toolCalls >= expect.minToolCalls,
+            detail: `made ${obs.toolCalls}`
+        });
+    }
+    if (expect.maxToolCalls !== undefined) {
+        checks.push({
+            name: `toolCalls<=${expect.maxToolCalls}`,
+            pass: obs.toolCalls <= expect.maxToolCalls,
+            detail: `made ${obs.toolCalls}`
+        });
+    }
+    if (expect.resultCheck) {
+        const pass = expect.resultCheck(obs.result);
+        checks.push({
+            name: `result:${expect.resultCheckLabel ?? "predicate"}`,
+            pass,
+            detail: pass
+                ? undefined
+                : `got ${JSON.stringify(obs.result)?.slice(0, 200)}`
+        });
+    }
+    return checks;
+}
+async function runCase(evalCase, opts) {
+    const recorder = createCodeActRecorder();
+    const tools = (evalCase.createTools ?? createCodeActTools)(recorder);
+    const maxIterations = evalCase.maxIterations ?? opts.maxIterations ?? DEFAULT_MAX_ITERATIONS;
+    // A case that allows packages needs a catalog to resolve them; one that does
+    // not stays on the process default, which is what every other case runs with.
+    const sandboxPackages = evalCase.sandboxPackages ?? [];
+    const contextInit = {
+        jobId: `codeact-eval-${randomUUID()}`,
+        userId: "eval-user"
+    };
+    if (sandboxPackages.length > 0) {
+        contextInit.sandboxModuleCatalog = shippedPackCatalog();
+    }
+    const context = new ProcessingContext(contextInit);
+    const step = {
+        id: randomUUID(),
+        instructions: evalCase.objective,
+        completed: false,
+        dependsOn: [],
+        logs: [],
+        outputSchema: evalCase.outputSchema
+            ? JSON.stringify(evalCase.outputSchema)
+            : undefined
+    };
+    const task = {
+        id: randomUUID(),
+        title: evalCase.description,
+        steps: [step]
+    };
+    const executor = new CodeActExecutor({
+        task,
+        step,
+        context,
+        provider: opts.provider,
+        model: opts.model,
+        tools,
+        maxIterations,
+        signal: opts.signal,
+        sandboxPackages
+    });
+    const costBefore = opts.provider.getTotalCost();
+    const startedAt = Date.now();
+    let result = null;
+    let actions = 0;
+    let error;
+    const sessionToolsInvoked = new Set();
+    try {
+        for await (const item of executor.execute()) {
+            if (opts.signal?.aborted)
+                break;
+            if (item.type === "tool_call_update") {
+                const tc = item;
+                if (tc.name)
+                    sessionToolsInvoked.add(tc.name);
+                // A "round" is one execute_code turn.
+                if (tc.name === EXECUTE_CODE_TOOL_NAME)
+                    actions++;
+            }
+            if (item.type === "step_result") {
+                const sr = item;
+                if (sr.error)
+                    error = sr.error;
+                else
+                    result = sr.result;
+            }
+        }
+    }
+    catch (e) {
+        error = e instanceof Error ? e.message : String(e);
+    }
+    const durationMs = Date.now() - startedAt;
+    const costUsd = opts.provider.getTotalCost() - costBefore;
+    const accepted = error === undefined && step.completed === true;
+    const observation = {
+        toolsInvoked: new Set(recorder.invocations.map((i) => i.name)),
+        sessionToolsInvoked,
+        actions,
+        toolCalls: recorder.invocations.length,
+        result
+    };
+    const checks = [
+        { name: "accepted", pass: accepted, detail: error }
+    ];
+    if (accepted) {
+        checks.push(...checkCodeActExpectations(observation, evalCase.expect));
+    }
+    const score = accepted
+        ? checks.filter((c) => c.pass).length / checks.length
+        : 0;
+    return {
+        caseId: evalCase.id,
+        description: evalCase.description,
+        accepted: accepted && checks.every((c) => c.pass),
+        score,
+        checks,
+        actions,
+        toolCalls: observation.toolCalls,
+        durationMs,
+        costUsd,
+        error
+    };
+}
+export async function runCodeActEval(opts) {
+    const cases = opts.cases ?? CODEACT_EVAL_CASES;
+    const results = [];
+    for (const evalCase of cases) {
+        if (opts.signal?.aborted)
+            break;
+        opts.onEvent?.(`▸ ${evalCase.id}`);
+        const result = await runCase(evalCase, opts);
+        results.push(result);
+        opts.onEvent?.(`  ${result.accepted ? "pass" : "FAIL"} score=${result.score.toFixed(2)} ` +
+            `actions=${result.actions} toolCalls=${result.toolCalls}`);
+    }
+    const accepted = results.filter((r) => r.accepted).length;
+    const meanScore = results.length > 0
+        ? results.reduce((sum, r) => sum + r.score, 0) / results.length
+        : 0;
+    return {
+        provider: opts.provider.provider,
+        model: opts.model,
+        startedAt: new Date().toISOString(),
+        cases: results,
+        summary: {
+            total: results.length,
+            accepted,
+            successRate: results.length > 0 ? accepted / results.length : 0,
+            meanScore,
+            avgActions: results.length > 0
+                ? results.reduce((s, r) => s + r.actions, 0) / results.length
+                : 0,
+            avgToolCalls: results.length > 0
+                ? results.reduce((s, r) => s + r.toolCalls, 0) / results.length
+                : 0,
+            totalCostUsd: results.reduce((s, r) => s + r.costUsd, 0)
+        }
+    };
+}
+export function formatCodeActReport(report) {
+    const lines = [];
+    lines.push(`CodeAct eval — provider=${report.provider} model=${report.model}`);
+    lines.push("");
+    const header = [
+        "case".padEnd(26),
+        "result".padEnd(7),
+        "score".padEnd(6),
+        "acts".padEnd(5),
+        "tools".padEnd(6),
+        "time".padEnd(7),
+        "cost"
+    ].join("");
+    lines.push(header);
+    lines.push("-".repeat(header.length));
+    for (const r of report.cases) {
+        lines.push([
+            r.caseId.padEnd(26),
+            (r.accepted ? "pass" : "FAIL").padEnd(7),
+            r.score.toFixed(2).padEnd(6),
+            String(r.actions).padEnd(5),
+            String(r.toolCalls).padEnd(6),
+            `${Math.round(r.durationMs / 1000)}s`.padEnd(7),
+            r.costUsd > 0 ? `$${r.costUsd.toFixed(4)}` : "-"
+        ].join(""));
+        for (const c of r.checks.filter((c) => !c.pass)) {
+            lines.push(`  ✗ ${c.name}${c.detail ? ` — ${c.detail}` : ""}`);
+        }
+    }
+    lines.push("");
+    const s = report.summary;
+    lines.push(`success ${s.accepted}/${s.total} (${(s.successRate * 100).toFixed(0)}%)  ` +
+        `meanScore ${s.meanScore.toFixed(2)}  avgActions ${s.avgActions.toFixed(1)}  ` +
+        `avgToolCalls ${s.avgToolCalls.toFixed(1)}  cost $${s.totalCostUsd.toFixed(4)}`);
+    return lines.join("\n");
+}
+//# sourceMappingURL=codeact-eval.js.map

package/dist/evals/codeact-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codeact-eval.js","sourceRoot":"","sources":["../../src/evals/codeact-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EACL,eAAe,EACf,sBAAsB,EACvB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAGnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAwDjC,8CAA8C;AAC9C,MAAM,UAAU,wBAAwB,CACtC,GAAuB,EACvB,MAA+B;IAE/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,QAAQ,IAAI,EAAE;YACpB,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,IAAI,EAAE;SACnD,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG;YACV,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;SAC3D,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,oBAAoB,IAAI,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,gBAAgB,IAAI,EAAE;YAC5B,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iBAAiB,IAAI,EAAE;SACnD,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,MAAM,CAAC,UAAU,EAAE;YACrC,IAAI,EAAE,GAAG,CAAC,OAAO,IAAI,MAAM,CAAC,UAAU;YACtC,MAAM,EAAE,QAAQ,GAAG,CAAC,OAAO,EAAE;SAC9B,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc,MAAM,CAAC,YAAY,EAAE;YACzC,IAAI,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,YAAY;YAC1C,MAAM,EAAE,QAAQ,GAAG,CAAC,SAAS,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc,MAAM,CAAC,YAAY,EAAE;YACzC,IAAI,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM,CAAC,YAAY;YAC1C,MAAM,EAAE,QAAQ,GAAG,CAAC,SAAS,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,UAAU,MAAM,CAAC,gBAAgB,IAAI,WAAW,EAAE;YACxD,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;SACvD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,QAAyB,EACzB,IAA2B;IAE3B,MAAM,QAAQ,GAAG,qBAAqB,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,kBAAkB,CAAC,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,aAAa,GACjB,QAAQ,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAC;IAEzE,6EAA6E;IAC7E,8EAA8E;IAC9E,MAAM,eAAe,GAAG,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC;IACvD,MAAM,WAAW,GAAuD;QACtE,KAAK,EAAE,gBAAgB,UAAU,EAAE,EAAE;QACrC,MAAM,EAAE,WAAW;KACpB,CAAC;IACF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,WAAW,CAAC,oBAAoB,GAAG,kBAAkB,EAAE,CAAC;IAC1D,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAS;QACjB,EAAE,EAAE,UAAU,EAAE;QAChB,YAAY,EAAE,QAAQ,CAAC,SAAS;QAChC,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,EAAE;QACb,IAAI,EAAE,EAAE;QACR,YAAY,EAAE,QAAQ,CAAC,YAAY;YACjC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC;YACvC,CAAC,CAAC,SAAS;KACd,CAAC;IACF,MAAM,IAAI,GAAS;QACjB,EAAE,EAAE,UAAU,EAAE;QAChB,KAAK,EAAE,QAAQ,CAAC,WAAW;QAC3B,KAAK,EAAE,CAAC,IAAI,CAAC;KACd,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,IAAI;QACJ,IAAI;QACJ,OAAO;QACP,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK;QACL,aAAa;QACb,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAY,IAAI,CAAC;IAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAyB,CAAC;IAC9B,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9C,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM;YAChC,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACrC,MAAM,EAAE,GAAG,IAAI,CAAC;gBAChB,IAAI,EAAE,CAAC,IAAI;oBAAE,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC9C,sCAAsC;gBACtC,IAAI,EAAE,CAAC,IAAI,KAAK,sBAAsB;oBAAE,OAAO,EAAE,CAAC;YACpD,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBAChC,MAAM,EAAE,GAAG,IAAI,CAAC;gBAChB,IAAI,EAAE,CAAC,KAAK;oBAAE,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC;;oBAC1B,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,KAAK,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,GAAG,UAAU,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;IAEhE,MAAM,WAAW,GAAuB;QACtC,YAAY,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,mBAAmB;QACnB,OAAO;QACP,SAAS,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM;QACtC,MAAM;KACP,CAAC;IAEF,MAAM,MAAM,GAAgB;QAC1B,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE;KACpD,CAAC;IACF,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ;QACpB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QACrD,CAAC,CAAC,CAAC,CAAC;IAEN,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,QAAQ,EAAE,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjD,KAAK;QACL,MAAM;QACN,OAAO;QACP,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,UAAU;QACV,OAAO;QACP,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAA2B;IAE3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC;IAC/C,MAAM,OAAO,GAAwB,EAAE,CAAC;IAExC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAChC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,EAAE,CACZ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACxE,WAAW,MAAM,CAAC,OAAO,cAAc,MAAM,CAAC,SAAS,EAAE,CAC5D,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,SAAS,GACb,OAAO,CAAC,MAAM,GAAG,CAAC;QAChB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;QAC/D,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAChC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,OAAO;QACd,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,QAAQ;YACR,WAAW,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/D,SAAS;YACT,UAAU,EACR,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;gBAC7D,CAAC,CAAC,CAAC;YACP,YAAY,EACV,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM;gBAC/D,CAAC,CAAC,CAAC;YACP,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SACzD;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAyB;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CACR,2BAA2B,MAAM,CAAC,QAAQ,UAAU,MAAM,CAAC,KAAK,EAAE,CACnE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,MAAM,MAAM,GAAG;QACb,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAChB,MAAM;KACP,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR;YACE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5B,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3B,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7B,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG;SACjD,CAAC,IAAI,CAAC,EAAE,CAAC,CACX,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;IACzB,KAAK,CAAC,IAAI,CACR,WAAW,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACzE,aAAa,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAC9E,gBAAgB,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAClF,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/evals/codeact-sandbox-pack-cases.d.ts

@@ -0,0 +1,31 @@
+/**
+ * CodeAct eval cases for sandbox packages.
+ *
+ * A CodeAct action is code the model just wrote, so the packages it may import
+ * are the session's allowlist and nothing else (`codeact/sandbox-packages.ts`).
+ * Three behaviours follow from that, and none of them was measured against a
+ * real model before: importing an allowed pack and using it correctly, reading
+ * a pack's SKILL.md through `get_sandbox_package_docs` rather than guessing its
+ * API, and recovering when an import is refused.
+ *
+ * The packs here are the shipped ones under `packages/sandbox-packs/`, resolved
+ * through the real catalog. Only **host** packs appear: a guest pack must be
+ * compiled by `@nodetool-ai/sandbox-compiler` first, which this package does not
+ * depend on.
+ *
+ * Every objective carries its own data, so a case answers from the pack or not
+ * at all — the expected values cannot be reached by reasoning over the prompt.
+ */
+import type { SandboxModuleCatalog } from "@nodetool-ai/runtime";
+import type { CodeActEvalCase } from "./codeact-cases.js";
+export declare const SANDBOX_PACK_DOCS_TOOL = "get_sandbox_package_docs";
+export declare const SANDBOX_PACK_LIST_TOOL = "list_sandbox_packages";
+/**
+ * A catalog over the shipped packs, built once per process.
+ *
+ * Discovery reads the pack directories users install; nothing is stubbed, so a
+ * case that imports a pack runs the same library a Code node would.
+ */
+export declare function shippedPackCatalog(): SandboxModuleCatalog;
+export declare const CODEACT_SANDBOX_PACK_EVAL_CASES: readonly CodeActEvalCase[];
+//# sourceMappingURL=codeact-sandbox-pack-cases.d.ts.map

package/dist/evals/codeact-sandbox-pack-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codeact-sandbox-pack-cases.d.ts","sourceRoot":"","sources":["../../src/evals/codeact-sandbox-pack-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAM1D,eAAO,MAAM,sBAAsB,6BAA6B,CAAC;AACjE,eAAO,MAAM,sBAAsB,0BAA0B,CAAC;AAiB9D;;;;;GAKG;AACH,wBAAgB,kBAAkB,IAAI,oBAAoB,CASzD;AAyBD,eAAO,MAAM,+BAA+B,EAAE,SAAS,eAAe,EAwJrE,CAAC"}

package/dist/evals/codeact-sandbox-pack-cases.js

@@ -0,0 +1,219 @@
+/**
+ * CodeAct eval cases for sandbox packages.
+ *
+ * A CodeAct action is code the model just wrote, so the packages it may import
+ * are the session's allowlist and nothing else (`codeact/sandbox-packages.ts`).
+ * Three behaviours follow from that, and none of them was measured against a
+ * real model before: importing an allowed pack and using it correctly, reading
+ * a pack's SKILL.md through `get_sandbox_package_docs` rather than guessing its
+ * API, and recovering when an import is refused.
+ *
+ * The packs here are the shipped ones under `packages/sandbox-packs/`, resolved
+ * through the real catalog. Only **host** packs appear: a guest pack must be
+ * compiled by `@nodetool-ai/sandbox-compiler` first, which this package does not
+ * depend on.
+ *
+ * Every objective carries its own data, so a case answers from the pack or not
+ * at all — the expected values cannot be reached by reasoning over the prompt.
+ */
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createSandboxModuleCatalog, discoverSandboxPack } from "@nodetool-ai/node-sdk";
+import { isObjectLike } from "../utils/type-guards.js";
+const here = dirname(fileURLToPath(import.meta.url));
+const PACKS_ROOT = join(here, "..", "..", "..", "sandbox-packs");
+export const SANDBOX_PACK_DOCS_TOOL = "get_sandbox_package_docs";
+export const SANDBOX_PACK_LIST_TOOL = "list_sandbox_packages";
+/**
+ * The packs these cases allow. Three host packs plus the workflow DSL, whose
+ * modules are plain guest JavaScript shipped in the pack — no npm dependency,
+ * so no compiler either.
+ */
+const SHIPPED_PACKS = [
+    "sandbox-csv",
+    "sandbox-xml",
+    "sandbox-zip",
+    "sandbox-dsl",
+    "sandbox-flow"
+];
+let cached;
+/**
+ * A catalog over the shipped packs, built once per process.
+ *
+ * Discovery reads the pack directories users install; nothing is stubbed, so a
+ * case that imports a pack runs the same library a Code node would.
+ */
+export function shippedPackCatalog() {
+    if (cached)
+        return cached;
+    const discoveries = SHIPPED_PACKS.map((dir) => {
+        const discovery = discoverSandboxPack(join(PACKS_ROOT, dir));
+        if (discovery === undefined)
+            throw new Error(`${dir} is not a sandbox pack`);
+        return discovery;
+    });
+    cached = createSandboxModuleCatalog(discoveries);
+    return cached;
+}
+const CSV = "@nodetool-ai/sandbox-csv";
+const XML = "@nodetool-ai/sandbox-xml";
+const ZIP = "@nodetool-ai/sandbox-zip";
+const DIFF = "@nodetool-ai/sandbox-diff";
+const ABSENT = "@acme/nope";
+/** Six orders; EMEA totals 405.50, which is the answer only parsing produces. */
+const ORDERS_CSV = [
+    "order_id,region,amount_eur",
+    "NT-1001,EMEA,120.00",
+    "NT-1002,AMER,80.50",
+    "NT-1003,EMEA,240.25",
+    "NT-1004,APAC,60.00",
+    "NT-1005,AMER,199.50",
+    "NT-1006,EMEA,45.25"
+].join("\\n");
+const TOTAL_SCHEMA = {
+    type: "object",
+    properties: { total: { type: "number" } },
+    required: ["total"]
+};
+export const CODEACT_SANDBOX_PACK_EVAL_CASES = [
+    {
+        id: "sandbox-pack-import",
+        description: "Import an allowed pack and compute from what it parses",
+        objective: "This CSV is your input:\\n\\n" +
+            ORDERS_CSV +
+            "\\n\\nParse it with the `" +
+            CSV +
+            "` sandbox package and finish with " +
+            "{total: <the summed amount_eur of the EMEA rows only>}. Parse it — do " +
+            "not add the numbers up yourself in your reply.",
+        sandboxPackages: [CSV],
+        outputSchema: TOTAL_SCHEMA,
+        expect: {
+            // 120.00 + 240.25 + 45.25. Wrong parsing gives a different sum, and the
+            // whole-file total (745.50) is the tell that the filter was skipped.
+            resultCheck: (r) => isObjectLike(r) &&
+                Math.abs(Number(r.total) - 405.5) < 0.001,
+            resultCheckLabel: "total=405.50 (EMEA only)",
+            maxActions: 4
+        }
+    },
+    {
+        id: "sandbox-pack-docs",
+        description: "Read a pack's SKILL.md instead of guessing its API",
+        objective: "You may import the `" +
+            XML +
+            "` sandbox package. Before you write any code, read that package's " +
+            "documentation to find out how attributes are represented in the parsed " +
+            "object. Then parse this feed:\\n\\n" +
+            '<?xml version="1.0"?><catalog><item sku="A-1"><name>Widget</name>' +
+            '</item></catalog>' +
+            "\\n\\nFinish with {prefix: <the exact key prefix the parser puts on " +
+            "attributes>}. It is two characters.",
+        sandboxPackages: [XML],
+        outputSchema: {
+            type: "object",
+            properties: { prefix: { type: "string" } },
+            required: ["prefix"]
+        },
+        expect: {
+            // The SKILL.md says attributes ride along prefixed `@_`. A model that
+            // guesses usually answers "@" or "$".
+            requiredSessionTools: [SANDBOX_PACK_DOCS_TOOL],
+            resultCheck: (r) => isObjectLike(r) &&
+                String(r.prefix).trim() === "@_",
+            resultCheckLabel: 'prefix="@_"',
+            maxActions: 4
+        }
+    },
+    {
+        id: "sandbox-pack-discover",
+        description: "Read the catalog instead of guessing what is installed",
+        objective: "Three sandbox packages might exist on this machine: `" +
+            ZIP +
+            "`, `" +
+            DIFF +
+            "` and `" +
+            ABSENT +
+            "`. Find out which of them are actually installed here — the answer is " +
+            "not in this prompt, and being off this session's allowlist is not the " +
+            "same as being absent. Finish with {installed: <the specifiers that are " +
+            "installed, as an array>}.",
+        sandboxPackages: [CSV],
+        namespaces: ["packs"],
+        outputSchema: {
+            type: "object",
+            properties: { installed: { type: "array", items: { type: "string" } } },
+            required: ["installed"]
+        },
+        expect: {
+            // Only zip is installed, and this session does not allow it — a listing
+            // that hid what it cannot import would answer with an empty array.
+            requiredSessionTools: [SANDBOX_PACK_LIST_TOOL],
+            resultCheck: (r) => {
+                const installed = isObjectLike(r)
+                    ? r.installed
+                    : undefined;
+                return (Array.isArray(installed) &&
+                    installed.length === 1 &&
+                    String(installed[0]) === ZIP);
+            },
+            resultCheckLabel: `installed=[${ZIP}]`,
+            maxActions: 4
+        }
+    },
+    {
+        id: "sandbox-pack-denied",
+        // Graded on the outcome, not the route. A model reaches `available: false`
+        // either by attempting the import and reading the refusal, or by reading
+        // the allowlist the prompt advertises; both are correct, and live runs do
+        // both. What the case rules out is the failure that matters — hand-parsing
+        // the text and reporting success. The refusal mechanism itself is pinned
+        // deterministically in `tests/codeact-sandbox-packs.test.ts`.
+        description: "Report a pack as unavailable instead of working around it",
+        objective: "Try to parse this YAML with the `@nodetool-ai/sandbox-yaml` sandbox " +
+            "package:\\n\\nservice: nodetool-api\\nreplicas: 3\\n\\n" +
+            "If that package cannot be imported in this session, do not work around " +
+            "it and do not hand-parse the text. Finish with " +
+            "{available: false, reason: <a short reason>} instead. If it can be " +
+            "imported, finish with {available: true, reason: 'imported'}.",
+        // The session allows csv, so the yaml import is refused by the mount check.
+        sandboxPackages: [CSV],
+        outputSchema: {
+            type: "object",
+            properties: {
+                available: { type: "boolean" },
+                reason: { type: "string" }
+            },
+            required: ["available", "reason"]
+        },
+        expect: {
+            resultCheck: (r) => isObjectLike(r) &&
+                r.available === false,
+            resultCheckLabel: "available=false",
+            maxActions: 5
+        }
+    },
+    {
+        id: "sandbox-pack-two",
+        description: "Use two allowed packs in one action",
+        objective: "This CSV is your input:\\n\\n" +
+            ORDERS_CSV +
+            "\\n\\nUsing the `" +
+            ZIP +
+            "` and `" +
+            CSV +
+            "` sandbox packages: put the CSV into a zip archive under the entry " +
+            "name `orders.csv`, read that archive back, parse the entry you " +
+            "extracted, and finish with {total: <the number of parsed data rows>}. " +
+            "Go through the archive — do not count the lines in the prompt.",
+        sandboxPackages: [ZIP, CSV],
+        outputSchema: TOTAL_SCHEMA,
+        expect: {
+            resultCheck: (r) => isObjectLike(r) &&
+                Number(r.total) === 6,
+            resultCheckLabel: "total=6 rows",
+            maxActions: 5
+        }
+    }
+];
+//# sourceMappingURL=codeact-sandbox-pack-cases.js.map

package/dist/evals/codeact-sandbox-pack-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codeact-sandbox-pack-cases.js","sourceRoot":"","sources":["../../src/evals/codeact-sandbox-pack-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EACL,0BAA0B,EAC1B,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,CAAC,CAAC;AAEjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,0BAA0B,CAAC;AACjE,MAAM,CAAC,MAAM,sBAAsB,GAAG,uBAAuB,CAAC;AAE9D;;;;GAIG;AACH,MAAM,aAAa,GAAG;IACpB,aAAa;IACb,aAAa;IACb,aAAa;IACb,aAAa;IACb,cAAc;CACf,CAAC;AAEF,IAAI,MAAwC,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC5C,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7D,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,wBAAwB,CAAC,CAAC;QAC7E,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,0BAA0B,CAAC,WAAW,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,GAAG,GAAG,0BAA0B,CAAC;AACvC,MAAM,GAAG,GAAG,0BAA0B,CAAC;AACvC,MAAM,GAAG,GAAG,0BAA0B,CAAC;AACvC,MAAM,IAAI,GAAG,2BAA2B,CAAC;AACzC,MAAM,MAAM,GAAG,YAAY,CAAC;AAE5B,iFAAiF;AACjF,MAAM,UAAU,GAAG;IACjB,4BAA4B;IAC5B,qBAAqB;IACrB,oBAAoB;IACpB,qBAAqB;IACrB,oBAAoB;IACpB,qBAAqB;IACrB,oBAAoB;CACrB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEd,MAAM,YAAY,GAAG;IACnB,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;IACzC,QAAQ,EAAE,CAAC,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,CAAC,MAAM,+BAA+B,GAA+B;IACzE;QACE,EAAE,EAAE,qBAAqB;QACzB,WAAW,EAAE,wDAAwD;QACrE,SAAS,EACP,+BAA+B;YAC/B,UAAU;YACV,2BAA2B;YAC3B,GAAG;YACH,oCAAoC;YACpC,wEAAwE;YACxE,gDAAgD;QAClD,eAAe,EAAE,CAAC,GAAG,CAAC;QACtB,YAAY,EAAE,YAAY;QAC1B,MAAM,EAAE;YACN,wEAAwE;YACxE,qEAAqE;YACrE,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CACjB,YAAY,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,GAAG,CAAC,MAAM,CAAE,CAAyB,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK;YACpE,gBAAgB,EAAE,0BAA0B;YAC5C,UAAU,EAAE,CAAC;SACd;KACF;IACD;QACE,EAAE,EAAE,mBAAmB;QACvB,WAAW,EAAE,oDAAoD;QACjE,SAAS,EACP,sBAAsB;YACtB,GAAG;YACH,oEAAoE;YACpE,yEAAyE;YACzE,qCAAqC;YACrC,mEAAmE;YACnE,mBAAmB;YACnB,sEAAsE;YACtE,qCAAqC;QACvC,eAAe,EAAE,CAAC,GAAG,CAAC;QACtB,YAAY,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YAC1C,QAAQ,EAAE,CAAC,QAAQ,CAAC;SACrB;QACD,MAAM,EAAE;YACN,sEAAsE;YACtE,sCAAsC;YACtC,oBAAoB,EAAE,CAAC,sBAAsB,CAAC;YAC9C,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CACjB,YAAY,CAAC,CAAC,CAAC;gBACf,MAAM,CAAE,CAA0B,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI;YAC5D,gBAAgB,EAAE,aAAa;YAC/B,UAAU,EAAE,CAAC;SACd;KACF;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,wDAAwD;QACrE,SAAS,EACP,uDAAuD;YACvD,GAAG;YACH,MAAM;YACN,IAAI;YACJ,SAAS;YACT,MAAM;YACN,wEAAwE;YACxE,wEAAwE;YACxE,yEAAyE;YACzE,2BAA2B;QAC7B,eAAe,EAAE,CAAC,GAAG,CAAC;QACtB,UAAU,EAAE,CAAC,OAAO,CAAC;QACrB,YAAY,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;YACvE,QAAQ,EAAE,CAAC,WAAW,CAAC;SACxB;QACD,MAAM,EAAE;YACN,wEAAwE;YACxE,mEAAmE;YACnE,oBAAoB,EAAE,CAAC,sBAAsB,CAAC;YAC9C,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE;gBACjB,MAAM,SAAS,GACb,YAAY,CAAC,CAAC,CAAC;oBACb,CAAC,CAAE,CAA6B,CAAC,SAAS;oBAC1C,CAAC,CAAC,SAAS,CAAC;gBAChB,OAAO,CACL,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;oBACxB,SAAS,CAAC,MAAM,KAAK,CAAC;oBACtB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAC7B,CAAC;YACJ,CAAC;YACD,gBAAgB,EAAE,cAAc,GAAG,GAAG;YACtC,UAAU,EAAE,CAAC;SACd;KACF;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,2EAA2E;QAC3E,yEAAyE;QACzE,0EAA0E;QAC1E,2EAA2E;QAC3E,yEAAyE;QACzE,8DAA8D;QAC9D,WAAW,EAAE,2DAA2D;QACxE,SAAS,EACP,sEAAsE;YACtE,yDAAyD;YACzD,yEAAyE;YACzE,iDAAiD;YACjD,qEAAqE;YACrE,8DAA8D;QAChE,4EAA4E;QAC5E,eAAe,EAAE,CAAC,GAAG,CAAC;QACtB,YAAY,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B;YACD,QAAQ,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC;SAClC;QACD,MAAM,EAAE;YACN,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CACjB,YAAY,CAAC,CAAC,CAAC;gBACd,CAA6B,CAAC,SAAS,KAAK,KAAK;YACpD,gBAAgB,EAAE,iBAAiB;YACnC,UAAU,EAAE,CAAC;SACd;KACF;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,WAAW,EAAE,qCAAqC;QAClD,SAAS,EACP,+BAA+B;YAC/B,UAAU;YACV,mBAAmB;YACnB,GAAG;YACH,SAAS;YACT,GAAG;YACH,qEAAqE;YACrE,iEAAiE;YACjE,wEAAwE;YACxE,gEAAgE;QAClE,eAAe,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAC3B,YAAY,EAAE,YAAY;QAC1B,MAAM,EAAE;YACN,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CACjB,YAAY,CAAC,CAAC,CAAC;gBACf,MAAM,CAAE,CAAyB,CAAC,KAAK,CAAC,KAAK,CAAC;YAChD,gBAAgB,EAAE,cAAc;YAChC,UAAU,EAAE,CAAC;SACd;KACF;CACF,CAAC"}

package/dist/evals/escalation-cases.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Workflow-tool eval cases with interactive escalation.
+ *
+ * The graph-editor suite in `tool-loop-cases.ts` scores a model on objectives
+ * that are fully specified: everything it needs is in the prompt, so guessing
+ * is never required. These cases remove that guarantee. Each one is missing
+ * something only the user can supply — a name, permission to delete, a decision
+ * between two node types, a capability the catalog doesn't have — and hands the
+ * model an `ask_user` tool backed by a scripted user.
+ *
+ * What's being measured is the pair: escalate when (and only when) the request
+ * is underdetermined, then *act on the answer*. So every case scores both the
+ * question (`escalation.mustAsk`) and the resulting graph — a model that asks
+ * the right thing and then ignores the reply fails on state, and one that
+ * silently guesses fails on the ask. The `no-escalation-needed` case guards the
+ * other direction: a fully specified objective where reaching for the user is
+ * itself the failure.
+ */
+import type { ToolLoopEvalCase } from "./tool-loop-eval.js";
+import { type ToolLoopFinalState } from "./tool-loop-bridge.js";
+export declare const WORKFLOW_ESCALATION_TOOL_LOOP_CASES: readonly ToolLoopEvalCase<ToolLoopFinalState>[];
+//# sourceMappingURL=escalation-cases.d.ts.map

package/dist/evals/escalation-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"escalation-cases.d.ts","sourceRoot":"","sources":["../../src/evals/escalation-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EACV,gBAAgB,EAEjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAEL,KAAK,kBAAkB,EAGxB,MAAM,uBAAuB,CAAC;AAuH/B,eAAO,MAAM,mCAAmC,EAAE,SAAS,gBAAgB,CAAC,kBAAkB,CAAC,EAqQ5F,CAAC"}