@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/evals/tool-loop-eval.js

@@ -0,0 +1,378 @@
+/**
+ * Tool-loop evaluation harness — provider-agnostic.
+ *
+ * Drives a multi-turn tool-calling loop over the *frontend* tool surface: the
+ * model is handed the real `ui_*` tool contract (names/descriptions/Zod schemas
+ * from `@nodetool-ai/protocol`) and an objective, each requested tool runs
+ * against a {@link createToolLoopBridge headless bridge}, and the result is fed
+ * back — repeating until the model stops calling tools or a turn cap is hit.
+ *
+ * Where {@link runGraphPlannerEval} evaluates typed-DSL graph authoring, this
+ * evaluates the incremental add-node/connect-node tool flow the browser UI
+ * exposes. It records the same efficiency
+ * metrics and emits the same result/summary shapes so providers/models can be
+ * compared on one report.
+ *
+ * Scoring is structural (see {@link checkToolLoopExpectations}): required /
+ * forbidden tool names, ordering constraints, final-state predicates, tool-call
+ * budgets, and a no-error-results check — never an exact transcript match, so
+ * many valid tool orderings pass.
+ *
+ * Checks are not weighed alike. Each carries a severity — `critical` for the
+ * behavior the case exists to test, `advisory` for efficiency budgets — and
+ * {@link scoreToolLoopChecks} weighs them accordingly and caps a run that
+ * missed a critical check, so "built the wrong thing" can never outscore "built
+ * the right thing a bit wastefully".
+ *
+ * Severity also decides the suite's headline number: `successRate` counts the
+ * cases that completed *and* passed every critical check, so a model that
+ * called no tool at all cannot pass. How often the loop merely ran to a stop is
+ * reported separately as `completionRate`.
+ */
+import { isRecord } from "../utils/type-guards.js";
+import { runToolLoop } from "../app-build/tool-loop.js";
+import { TOOL_LOOP_EVAL_CASES } from "./tool-loop-cases.js";
+import { checkEscalationExpectations, createEscalationChannel } from "./escalation.js";
+import { gateHeadlessTools } from "./tool-loop-permission.js";
+const DEFAULT_SYSTEM_PROMPT = `You are a workflow-graph building assistant operating a node-based editor through UI tools.
+
+Build the workflow the user asks for by calling the ui_* tools:
+- Discover node types with ui_search_nodes before adding them — never guess a type.
+- Add nodes with ui_add_node (choose a stable, unique id per node and a {x, y} position).
+- Wire nodes together with ui_connect_nodes using the exact output/input handle names from ui_search_nodes.
+- Inspect your work with ui_get_graph when unsure.
+
+Call one tool at a time and use the result before the next call. When the objective is fully satisfied, STOP calling tools and give a one-line summary.`;
+function buildUserPrompt(evalCase) {
+    return evalCase.userPrompt ?? `Objective: ${evalCase.objective}`;
+}
+/**
+ * Weight per severity. A flat pass-fraction made a run that built the right
+ * graph but skipped the one behavior under test (`confirm-before-delete`, no
+ * `ask_user`) score higher than a run that did everything right and overran a
+ * call budget — the two are not comparable, so they should not be weighed
+ * alike.
+ */
+const SEVERITY_WEIGHT = {
+    critical: 3,
+    standard: 2,
+    advisory: 1
+};
+/**
+ * Ceiling on a run that failed a `critical` check. Without it, a case with many
+ * passing state predicates can bury a core-behavior miss under partial credit.
+ */
+export const CRITICAL_FAILURE_SCORE_CAP = 0.5;
+/** Checks with no severity (other suites) are scored as `standard`. */
+function weightOf(check) {
+    return SEVERITY_WEIGHT[check.severity ?? "standard"];
+}
+/**
+ * Severity-weighted pass fraction, capped when any critical check failed.
+ * Exported so the scoring rule is testable on its own.
+ */
+export function scoreToolLoopChecks(checks) {
+    if (checks.length === 0)
+        return 0;
+    const total = checks.reduce((sum, c) => sum + weightOf(c), 0);
+    const earned = checks
+        .filter((c) => c.pass)
+        .reduce((sum, c) => sum + weightOf(c), 0);
+    const score = earned / total;
+    const criticalFailed = checks.some((c) => !c.pass && c.severity === "critical");
+    return criticalFailed ? Math.min(score, CRITICAL_FAILURE_SCORE_CAP) : score;
+}
+/** Failing checks marked `critical` — the core-behavior misses. */
+export function countCriticalFailures(checks) {
+    return checks.filter((c) => !c.pass && c.severity === "critical").length;
+}
+/**
+ * Score a completed run against a case's structural expectations. Pure and
+ * fully unit-testable: it takes an {@link ToolLoopObservation} and never calls
+ * a provider.
+ */
+export function checkToolLoopExpectations(observation, expect) {
+    const checks = [];
+    const sequence = observation.toolCalls.map((c) => c.name);
+    const called = new Set(sequence);
+    for (const name of expect.requiredTools ?? []) {
+        const pass = called.has(name);
+        checks.push({
+            name: `tool:${name}`,
+            pass,
+            severity: "critical",
+            detail: pass ? undefined : `never called ${name}`
+        });
+    }
+    for (const name of expect.forbiddenTools ?? []) {
+        const hit = called.has(name);
+        checks.push({
+            name: `not-tool:${name}`,
+            pass: !hit,
+            severity: "critical",
+            detail: hit ? `called forbidden tool ${name}` : undefined
+        });
+    }
+    for (const [a, b] of expect.ordering ?? []) {
+        const ia = sequence.indexOf(a);
+        const ib = sequence.indexOf(b);
+        const pass = ia !== -1 && ib !== -1 && ia < ib;
+        checks.push({
+            name: `order:${a}<${b}`,
+            pass,
+            severity: "standard",
+            detail: pass
+                ? undefined
+                : `${a} first@${ia}, ${b} first@${ib} (need ${a} before ${b})`
+        });
+    }
+    for (const predicate of expect.finalState ?? []) {
+        let pass = false;
+        let detail = predicate.detail;
+        try {
+            pass = predicate.test(observation.finalState);
+        }
+        catch (e) {
+            detail = e instanceof Error ? e.message : String(e);
+        }
+        checks.push({
+            name: `state:${predicate.name}`,
+            pass,
+            severity: "critical",
+            detail: pass
+                ? undefined
+                : (detail ?? `predicate ${predicate.name} failed`)
+        });
+    }
+    if (expect.minToolCalls !== undefined) {
+        checks.push({
+            name: `toolCalls>=${expect.minToolCalls}`,
+            pass: sequence.length >= expect.minToolCalls,
+            severity: "advisory",
+            detail: `made ${sequence.length}`
+        });
+    }
+    if (expect.maxToolCalls !== undefined) {
+        checks.push({
+            name: `toolCalls<=${expect.maxToolCalls}`,
+            pass: sequence.length <= expect.maxToolCalls,
+            severity: "advisory",
+            detail: `made ${sequence.length}`
+        });
+    }
+    if (expect.noErrorResults) {
+        const errored = observation.toolCalls.filter((c) => c.isError);
+        checks.push({
+            name: "no-error-results",
+            pass: errored.length === 0,
+            severity: "standard",
+            detail: errored.length === 0
+                ? undefined
+                : `${errored.length} errored: ${errored.map((c) => c.name).join(", ")}`
+        });
+    }
+    if (expect.escalation) {
+        checks.push(...checkEscalationExpectations(observation.escalations ?? [], sequence, expect.escalation));
+    }
+    for (const predicate of expect.permissionRequests ?? []) {
+        const requests = observation.permissionRequests ?? [];
+        let pass = false;
+        let detail = predicate.detail;
+        try {
+            pass = predicate.test(requests);
+        }
+        catch (e) {
+            detail = e instanceof Error ? e.message : String(e);
+        }
+        checks.push({
+            name: `permission:${predicate.name}`,
+            pass,
+            severity: "critical",
+            detail: pass
+                ? undefined
+                : (detail ?? `predicate ${predicate.name} failed`)
+        });
+    }
+    return checks;
+}
+async function runCase(evalCase, opts) {
+    const bridge = evalCase.createBridge();
+    // An interactive case gets one extra tool alongside the surface tools: the
+    // scripted user it can escalate to.
+    const escalation = evalCase.escalation
+        ? createEscalationChannel(evalCase.escalation)
+        : undefined;
+    const surfaceTools = escalation
+        ? [...bridge.tools, escalation.tool]
+        : bridge.tools;
+    // A case with a permission mode runs the whole belt through the gate, the
+    // scripted user included.
+    const gated = evalCase.permission
+        ? gateHeadlessTools(surfaceTools, evalCase.permission)
+        : undefined;
+    const run = await runToolLoop({
+        provider: opts.provider,
+        model: opts.model,
+        tools: gated ? gated.tools : surfaceTools,
+        // Per-case (surface-specific) prompt wins over the runner-wide override,
+        // which in turn wins over the graph-editor default.
+        systemPrompt: evalCase.systemPrompt ?? opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT,
+        userPrompt: buildUserPrompt(evalCase),
+        // A case that declares its own budget wins: an explicit per-case need
+        // beats the runner default. An explicit --max-iterations still overrides.
+        maxIterations: opts.maxIterations ?? evalCase.maxIterations,
+        signal: opts.signal,
+        onToolCall: (record) => opts.onEvent?.(`    [tool] ${record.name}${record.isError ? " (error)" : ""}`)
+    });
+    const accepted = run.completed;
+    const observation = {
+        toolCalls: run.calls,
+        finalState: bridge.finalState(),
+        escalations: escalation?.turns(),
+        permissionRequests: gated?.requests()
+    };
+    const checks = [
+        {
+            name: "accepted",
+            pass: accepted,
+            severity: "critical",
+            detail: run.error
+        }
+    ];
+    if (accepted) {
+        checks.push(...checkToolLoopExpectations(observation, evalCase.expect));
+    }
+    const score = accepted ? scoreToolLoopChecks(checks) : 0;
+    const criticalFailures = countCriticalFailures(checks);
+    const result = {
+        caseId: evalCase.id,
+        description: evalCase.description,
+        skipped: false,
+        accepted,
+        success: accepted && criticalFailures === 0,
+        score,
+        criticalFailures,
+        checks,
+        toolCalls: run.countsByName,
+        totalToolCalls: run.totalCalls,
+        permissionRequests: [...(observation.permissionRequests ?? [])],
+        durationMs: run.durationMs,
+        costUsd: run.costUsd,
+        error: run.error
+    };
+    if (isRecord(observation.finalState) && observation.finalState.visual_output_verified === false) {
+        result.visual_output_verified = false;
+    }
+    return result;
+}
+export async function runToolLoopEval(opts) {
+    const cases = (opts.cases ??
+        TOOL_LOOP_EVAL_CASES);
+    const hasModelProviders = !!opts.providers && Object.keys(opts.providers).length > 0;
+    const results = [];
+    for (const evalCase of cases) {
+        if (opts.signal?.aborted)
+            break;
+        if (evalCase.needsModelProviders && !hasModelProviders) {
+            opts.onEvent?.(`- ${evalCase.id}: SKIPPED (no model providers)`);
+            results.push({
+                caseId: evalCase.id,
+                description: evalCase.description,
+                skipped: true,
+                accepted: false,
+                success: false,
+                score: 0,
+                criticalFailures: 0,
+                checks: [],
+                toolCalls: {},
+                totalToolCalls: 0,
+                permissionRequests: [],
+                durationMs: 0,
+                costUsd: 0
+            });
+            continue;
+        }
+        opts.onEvent?.(`- ${evalCase.id}: ${evalCase.description}`);
+        const result = await runCase(evalCase, opts);
+        const failed = result.checks.filter((c) => !c.pass);
+        opts.onEvent?.(`  ${result.success ? "PASS" : "FAIL"} score=${result.score.toFixed(2)} ` +
+            `tools=${result.totalToolCalls} ${Math.round(result.durationMs / 1000)}s` +
+            (failed.length > 0
+                ? ` | failed: ${failed.map((c) => c.name).join(", ")}`
+                : ""));
+        results.push(result);
+    }
+    const ran = results.filter((r) => !r.skipped);
+    const acceptedResults = ran.filter((r) => r.accepted);
+    const successful = ran.filter((r) => r.success).length;
+    const summary = {
+        total: results.length,
+        skipped: results.length - ran.length,
+        accepted: acceptedResults.length,
+        successful,
+        successRate: ran.length > 0 ? successful / ran.length : 0,
+        completionRate: ran.length > 0 ? acceptedResults.length / ran.length : 0,
+        meanScore: ran.length > 0 ? ran.reduce((a, r) => a + r.score, 0) / ran.length : 0,
+        avgToolCalls: ran.length > 0
+            ? ran.reduce((a, r) => a + r.totalToolCalls, 0) / ran.length
+            : 0,
+        totalCostUsd: ran.reduce((a, r) => a + r.costUsd, 0)
+    };
+    return {
+        provider: opts.provider.provider,
+        model: opts.model,
+        startedAt: new Date().toISOString(),
+        cases: results,
+        summary
+    };
+}
+/** Text summary table for terminal output. */
+export function formatToolLoopReport(report) {
+    const lines = [];
+    lines.push(`Tool-loop eval — provider=${report.provider} model=${report.model}`);
+    lines.push("");
+    const header = [
+        "case".padEnd(24),
+        "result".padEnd(7),
+        "score".padEnd(6),
+        "crit".padEnd(5),
+        "tools".padEnd(6),
+        "time".padEnd(7),
+        "cost"
+    ].join("");
+    lines.push(header);
+    lines.push("-".repeat(header.length));
+    for (const r of report.cases) {
+        lines.push([
+            r.caseId.padEnd(24),
+            (r.skipped
+                ? "skip"
+                : r.success
+                    ? "pass"
+                    : r.accepted
+                        ? "FAIL"
+                        : "ERROR").padEnd(7),
+            (r.skipped ? "-" : r.score.toFixed(2)).padEnd(6),
+            (r.skipped ? "-" : String(r.criticalFailures)).padEnd(5),
+            String(r.totalToolCalls).padEnd(6),
+            `${Math.round(r.durationMs / 1000)}s`.padEnd(7),
+            r.costUsd > 0 ? `$${r.costUsd.toFixed(4)}` : "-"
+        ].join(""));
+        for (const c of r.checks.filter((c) => !c.pass)) {
+            // Severity first: a reader scanning failures needs to know which ones
+            // are the behavior under test and which are budget overruns.
+            const tag = c.severity === "critical" ? "[critical] " : "";
+            lines.push(`  ✗ ${tag}${c.name}${c.detail ? ` — ${c.detail}` : ""}`);
+        }
+    }
+    const s = report.summary;
+    lines.push("");
+    lines.push(`success ${s.successful}/${s.total - s.skipped} (${(s.successRate * 100).toFixed(0)}%)` +
+        `  completed ${(s.completionRate * 100).toFixed(0)}%` +
+        `  mean score ${s.meanScore.toFixed(2)}` +
+        `  avg tools ${s.avgToolCalls.toFixed(1)}` +
+        (s.totalCostUsd > 0 ? `  cost $${s.totalCostUsd.toFixed(4)}` : "") +
+        (s.skipped > 0 ? `  (${s.skipped} skipped)` : ""));
+    return lines.join("\n");
+}
+//# sourceMappingURL=tool-loop-eval.js.map

package/dist/evals/tool-loop-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-loop-eval.js","sourceRoot":"","sources":["../../src/evals/tool-loop-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EACL,WAAW,EAEZ,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,EAIxB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,iBAAiB,EAGlB,MAAM,2BAA2B,CAAC;AA8LnC,MAAM,qBAAqB,GAAG;;;;;;;;wJAQ0H,CAAC;AAEzJ,SAAS,eAAe,CAAC,QAGxB;IACC,OAAO,QAAQ,CAAC,UAAU,IAAI,cAAc,QAAQ,CAAC,SAAS,EAAE,CAAC;AACnE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,eAAe,GAAG;IACtB,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,CAAC;CACiD,CAAC;AAE/D;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAE9C,uEAAuE;AACvE,SAAS,QAAQ,CAAC,KAAgB;IAChC,OAAO,eAAe,CAAC,KAAK,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAA4B;IAC9D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,MAAM;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SACrB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;IAC7B,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,UAAU,CAC5C,CAAC;IACF,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC9E,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,qBAAqB,CAAC,MAA4B;IAChE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACvC,WAAwC,EACxC,MAAwC;IAExC,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,QAAQ,IAAI,EAAE;YACpB,IAAI;YACJ,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE;SAClD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,IAAI,EAAE;YACxB,IAAI,EAAE,CAAC,GAAG;YACV,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;SAC1D,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QAC3C,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE;YACvB,IAAI;YACJ,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,UAAU,CAAC,WAAW,CAAC,GAAG;SACjE,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QAChD,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS,SAAS,CAAC,IAAI,EAAE;YAC/B,IAAI;YACJ,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,CAAC,MAAM,IAAI,aAAa,SAAS,CAAC,IAAI,SAAS,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc,MAAM,CAAC,YAAY,EAAE;YACzC,IAAI,EAAE,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,YAAY;YAC5C,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;SAClC,CAAC,CAAC;IACL,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc,MAAM,CAAC,YAAY,EAAE;YACzC,IAAI,EAAE,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,YAAY;YAC5C,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;SAClC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,kBAAkB;YACxB,IAAI,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC;YAC1B,QAAQ,EAAE,UAAU;YACpB,MAAM,EACJ,OAAO,CAAC,MAAM,KAAK,CAAC;gBAClB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,aAAa,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC5E,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CACT,GAAG,2BAA2B,CAC5B,WAAW,CAAC,WAAW,IAAI,EAAE,EAC7B,QAAQ,EACR,MAAM,CAAC,UAAU,CAClB,CACF,CAAC;IACJ,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,kBAAkB,IAAI,EAAE,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,WAAW,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACtD,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc,SAAS,CAAC,IAAI,EAAE;YACpC,IAAI;YACJ,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,CAAC,MAAM,IAAI,aAAa,SAAS,CAAC,IAAI,SAAS,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,QAAkC,EAClC,IAAoC;IAEpC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,EAAE,CAAC;IACvC,2EAA2E;IAC3E,oCAAoC;IACpC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU;QACpC,CAAC,CAAC,uBAAuB,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC9C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,YAAY,GAAmB,UAAU;QAC7C,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;QACpC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;IACjB,0EAA0E;IAC1E,0BAA0B;IAC1B,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU;QAC/B,CAAC,CAAC,iBAAiB,CAAC,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC;QACtD,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC;QAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY;QACzC,yEAAyE;QACzE,oDAAoD;QACpD,YAAY,EACV,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,qBAAqB;QACrE,UAAU,EAAE,eAAe,CAAC,QAAQ,CAAC;QACrC,sEAAsE;QACtE,0EAA0E;QAC1E,aAAa,EAAE,IAAI,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC3D,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CACrB,IAAI,CAAC,OAAO,EAAE,CACZ,cAAc,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/D;KACJ,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC;IAC/B,MAAM,WAAW,GAAgC;QAC/C,SAAS,EAAE,GAAG,CAAC,KAAK;QACpB,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE;QAC/B,WAAW,EAAE,UAAU,EAAE,KAAK,EAAE;QAChC,kBAAkB,EAAE,KAAK,EAAE,QAAQ,EAAE;KACtC,CAAC;IAEF,MAAM,MAAM,GAAgB;QAC1B;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,UAAU;YACpB,MAAM,EAAE,GAAG,CAAC,KAAK;SAClB;KACF,CAAC;IACF,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAuB;QACjC,MAAM,EAAE,QAAQ,CAAC,EAAE;QACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,OAAO,EAAE,KAAK;QACd,QAAQ;QACR,OAAO,EAAE,QAAQ,IAAI,gBAAgB,KAAK,CAAC;QAC3C,KAAK;QACL,gBAAgB;QAChB,MAAM;QACN,SAAS,EAAE,GAAG,CAAC,YAAY;QAC3B,cAAc,EAAE,GAAG,CAAC,UAAU;QAC9B,kBAAkB,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC;QAC/D,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;IACF,IAAI,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,IAAI,WAAW,CAAC,UAAU,CAAC,sBAAsB,KAAK,KAAK,EAAE,CAAC;QAChG,MAAM,CAAC,sBAAsB,GAAG,KAAK,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAoC;IAEpC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK;QACvB,oBAAoB,CAAwC,CAAC;IAC/D,MAAM,iBAAiB,GACrB,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAChC,IAAI,QAAQ,CAAC,mBAAmB,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvD,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,QAAQ,CAAC,EAAE,gCAAgC,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,KAAK;gBACf,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAC;gBACR,gBAAgB,EAAE,CAAC;gBACnB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,EAAE;gBACb,cAAc,EAAE,CAAC;gBACjB,kBAAkB,EAAE,EAAE;gBACtB,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO,EAAE,CACZ,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACvE,SAAS,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG;YACzE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,cAAc,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtD,CAAC,CAAC,EAAE,CAAC,CACV,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvD,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM;QACpC,QAAQ,EAAE,eAAe,CAAC,MAAM;QAChC,UAAU;QACV,WAAW,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACzD,cAAc,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxE,SAAS,EACP,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxE,YAAY,EACV,GAAG,CAAC,MAAM,GAAG,CAAC;YACZ,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM;YAC5D,CAAC,CAAC,CAAC;QACP,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;KACrD,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ;QAChC,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,OAAO;QACd,OAAO;KACR,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,oBAAoB,CAAC,MAA0B;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CACR,6BAA6B,MAAM,CAAC,QAAQ,UAAU,MAAM,CAAC,KAAK,EAAE,CACrE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,MAAM,MAAM,GAAG;QACb,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAChB,MAAM;KACP,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACR;YACE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnB,CAAC,CAAC,CAAC,OAAO;gBACR,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,CAAC,CAAC,OAAO;oBACT,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,CAAC,CAAC,QAAQ;wBACV,CAAC,CAAC,MAAM;wBACR,CAAC,CAAC,OAAO,CACd,CAAC,MAAM,CAAC,CAAC,CAAC;YACX,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YAClC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG;SACjD,CAAC,IAAI,CAAC,EAAE,CAAC,CACX,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,sEAAsE;YACtE,6DAA6D;YAC7D,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IACD,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,WAAW,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QACrF,eAAe,CAAC,CAAC,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QACrD,gBAAgB,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QACxC,eAAe,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QAC1C,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CACpD,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/evals/tool-loop-permission.d.ts

@@ -0,0 +1,29 @@
+/** Headless evals invoke capabilities through the production permission gate. */
+import type { PermissionCategory, PermissionMode } from "../tools/tool-permissions.js";
+import type { HeadlessTool } from "./tool-loop-bridge.js";
+/** The scripted user's answer to every approval prompt. */
+export type ToolLoopApproval = "allow" | "deny";
+export interface ToolLoopPermission {
+    mode: PermissionMode;
+    /** Answer to every approval request. Defaults to `allow`. */
+    approve?: ToolLoopApproval;
+}
+/** One approval request the gate made, and the scripted answer it got. */
+export interface PermissionRequestRecord {
+    toolName: string;
+    category: PermissionCategory;
+    decision: ToolLoopApproval;
+}
+export interface GatedHeadlessTools {
+    tools: HeadlessTool[];
+    /** Every approval request so far, in order. */
+    requests: () => readonly PermissionRequestRecord[];
+}
+/**
+ * Run `tools` through the permission ladder under `permission`. The returned
+ * tools keep the inner tools' names, descriptions and schemas; only `execute`
+ * changes, and a blocked or denied call resolves to the ladder's structured
+ * error result rather than throwing.
+ */
+export declare function gateHeadlessTools(tools: HeadlessTool[], permission: ToolLoopPermission): GatedHeadlessTools;
+//# sourceMappingURL=tool-loop-permission.d.ts.map

package/dist/evals/tool-loop-permission.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-loop-permission.d.ts","sourceRoot":"","sources":["../../src/evals/tool-loop-permission.ts"],"names":[],"mappings":"AAAA,iFAAiF;AAOjF,OAAO,KAAK,EACV,kBAAkB,EAElB,cAAc,EACf,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAE1D,2DAA2D;AAC3D,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,MAAM,CAAC;AAEhD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,cAAc,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED,0EAA0E;AAC1E,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,QAAQ,EAAE,gBAAgB,CAAC;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,SAAS,uBAAuB,EAAE,CAAC;CACpD;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,YAAY,EAAE,EACrB,UAAU,EAAE,kBAAkB,GAC7B,kBAAkB,CA4CpB"}

package/dist/evals/tool-loop-permission.js

@@ -0,0 +1,58 @@
+/** Headless evals invoke capabilities through the production permission gate. */
+import { randomUUID } from "node:crypto";
+import { ProcessingContext } from "@nodetool-ai/runtime";
+import { zodToJsonSchema } from "@nodetool-ai/runtime";
+import { createCapabilityRun } from "../capabilities/invoke.js";
+import { capabilityCategoryFor } from "../capabilities/registry.js";
+import { PERMISSION_GATE_CONTEXT_KEY } from "../types.js";
+/**
+ * Run `tools` through the permission ladder under `permission`. The returned
+ * tools keep the inner tools' names, descriptions and schemas; only `execute`
+ * changes, and a blocked or denied call resolves to the ladder's structured
+ * error result rather than throwing.
+ */
+export function gateHeadlessTools(tools, permission) {
+    const requests = [];
+    const decision = permission.approve ?? "allow";
+    const gate = {
+        mode: permission.mode,
+        sessionAllow: new Set(),
+        requestApproval: async (request) => {
+            requests.push({
+                toolName: request.toolName,
+                category: request.category,
+                decision
+            });
+            return decision;
+        }
+    };
+    const context = new ProcessingContext({
+        jobId: `tool-loop-eval-${randomUUID()}`,
+        userId: "eval-user"
+    });
+    context.set(PERMISSION_GATE_CONTEXT_KEY, gate);
+    const run = createCapabilityRun({
+        context,
+        gate,
+        capabilities: tools.map((tool) => ({
+            spec: {
+                name: tool.name,
+                description: tool.description,
+                inputSchema: zodToJsonSchema(tool.parameters),
+                zodSchema: tool.parameters,
+                category: tool.category ?? capabilityCategoryFor(tool.name),
+                userMessage: tool.userMessage,
+                needsToolCallId: tool.needsToolCallId
+            },
+            impl: (_run, args) => tool.execute(args)
+        }))
+    });
+    return {
+        tools: tools.map((tool) => ({
+            ...tool,
+            execute: (args) => run.invoke(tool.name, args)
+        })),
+        requests: () => requests
+    };
+}
+//# sourceMappingURL=tool-loop-permission.js.map

package/dist/evals/tool-loop-permission.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-loop-permission.js","sourceRoot":"","sources":["../../src/evals/tool-loop-permission.ts"],"names":[],"mappings":"AAAA,iFAAiF;AAEjF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAMpE,OAAO,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAyB1D;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAqB,EACrB,UAA8B;IAE9B,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,MAAM,QAAQ,GAAqB,UAAU,CAAC,OAAO,IAAI,OAAO,CAAC;IACjE,MAAM,IAAI,GAA0B;QAClC,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,YAAY,EAAE,IAAI,GAAG,EAAU;QAC/B,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACjC,QAAQ,CAAC,IAAI,CAAC;gBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,QAAQ;aACT,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC;QAClB,CAAC;KACF,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC;QACpC,KAAK,EAAE,kBAAkB,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,WAAW;KACpB,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,IAAI,CAAC,CAAC;IAE/C,MAAM,GAAG,GAAG,mBAAmB,CAAC;QAC9B,OAAO;QACP,IAAI;QACJ,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACjC,IAAI,EAAE;gBACJ,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC7C,SAAS,EAAE,IAAI,CAAC,UAAU;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC3D,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;aACtC;YACD,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC;KACJ,CAAC,CAAC;IACH,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1B,GAAG,IAAI;YACP,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;SAC/C,CAAC,CAAC;QACH,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ;KACzB,CAAC;AACJ,CAAC"}