@nodetool-ai/cli 0.7.0-rc.23 → 0.8.2

package/dist/commands/eval.js

@@ -0,0 +1,690 @@
+import { parseNumericOption } from "../numeric-options.js";
+/**
+ * Threshold gates for one suite run: the reasons it should exit non-zero.
+ * Extracted from the command action so both gates — including the failing
+ * paths, which otherwise only a paid model run reaches — are unit-testable.
+ */
+export function evalGateFailures(result, suiteId, opts) {
+    const failures = [];
+    // Checked first, and reported in its own words: every suite reports a rate of
+    // 0 over an empty set, so a run whose cases were all skipped otherwise fails
+    // as if the cases had run and lost. It is also the one failure a rate cannot
+    // express — a suite that ran half its cases and passed them reports 1.0.
+    if (opts.minCases !== undefined) {
+        const floor = parseNumericOption(opts.minCases, "--min-cases", {
+            integer: true,
+            min: 0
+        });
+        if (result.casesRan < floor) {
+            failures.push(`Suite "${suiteId}" ran ${result.casesRan} case(s), below --min-cases ` +
+                `${floor} — a rate over that many cases says nothing`);
+        }
+    }
+    if (opts.minScore !== undefined) {
+        const threshold = parseNumericOption(opts.minScore, "--min-score", {
+            min: 0,
+            max: 1
+        });
+        if (result.meanScore === undefined) {
+            failures.push(`--min-score: suite "${suiteId}" reports no score`);
+        }
+        else if (result.meanScore < threshold) {
+            failures.push(`Mean score ${result.meanScore.toFixed(2)} below threshold ${threshold}`);
+        }
+    }
+    if (opts.minSuccess !== undefined) {
+        const threshold = parseNumericOption(opts.minSuccess, "--min-success", {
+            min: 0,
+            max: 1
+        });
+        if (result.successRate < threshold) {
+            failures.push(`Success rate ${result.successRate.toFixed(2)} below threshold ${threshold}`);
+        }
+    }
+    // Spend is summed over every case, skipped ones included (they cost 0), so
+    // the number gated is what the run billed.
+    if (opts.maxCostUsd !== undefined) {
+        const cap = parseNumericOption(opts.maxCostUsd, "--max-cost-usd", {
+            min: 0
+        });
+        if (result.costUsd > cap) {
+            failures.push(`Total cost $${result.costUsd.toFixed(4)} above --max-cost-usd ${cap}`);
+        }
+    }
+    // p95 over the cases that ran: a skipped case records 0 ms and would pull
+    // the percentile down. Nearest-rank, so the number is a duration one case
+    // actually took. A run with no timings fails the gate outright — the same
+    // rule --min-score applies when a suite reports no score.
+    if (opts.maxP95DurationMs !== undefined) {
+        const cap = parseNumericOption(opts.maxP95DurationMs, "--max-p95-duration-ms", { integer: true, min: 0 });
+        const p95 = percentile95(result.durationsMs);
+        if (p95 === undefined) {
+            failures.push(`--max-p95-duration-ms: suite "${suiteId}" ran no case, nothing to time`);
+        }
+        else if (p95 > cap) {
+            failures.push(`p95 case duration ${p95}ms above --max-p95-duration-ms ${cap}`);
+        }
+    }
+    return failures;
+}
+/** Nearest-rank p95; undefined over an empty sample. */
+function percentile95(samples) {
+    if (samples.length === 0)
+        return undefined;
+    const sorted = [...samples].sort((a, b) => a - b);
+    return sorted[Math.ceil(0.95 * sorted.length) - 1];
+}
+/**
+ * What every suite records per case and the cost/duration gates read. A
+ * skipped case (`skipped` set, or a non-empty reason string) cost nothing and
+ * took no time, so it stays in the sum and leaves the timings.
+ */
+function runMetrics(cases) {
+    return {
+        costUsd: cases.reduce((sum, c) => sum + c.costUsd, 0),
+        durationsMs: cases.filter((c) => !c.skipped).map((c) => c.durationMs)
+    };
+}
+/**
+ * Filter a suite's cases down to `--cases`, rejecting ids the suite doesn't
+ * have. Undefined ids run the whole suite.
+ */
+function selectCases(all, caseIds) {
+    if (!caseIds)
+        return all;
+    const wanted = new Set(caseIds);
+    const picked = all.filter((c) => wanted.has(c.id));
+    const pickedIds = new Set(picked.map((c) => c.id));
+    const missing = [...wanted].filter((id) => !pickedIds.has(id));
+    if (missing.length > 0) {
+        throw new Error(`Unknown case ids: ${missing.join(", ")} (see --list)`);
+    }
+    return picked;
+}
+/** Graph authoring suite — `authorGraph` over the typed DSL pack. */
+const graphPlannerSuite = {
+    id: "graph-planner",
+    description: "Run the graph authoring eval suite (authorGraph, typed DSL pack) against a provider/model and report metrics",
+    async listCases() {
+        const { GRAPH_PLANNER_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return GRAPH_PLANNER_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description,
+            needsModelProviders: c.needsModelProviders
+        }));
+    },
+    async run(deps) {
+        const { GRAPH_PLANNER_EVAL_CASES, runGraphPlannerEval, formatEvalReport } = await import("@nodetool-ai/agents");
+        const cases = selectCases(GRAPH_PLANNER_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} case(s) with ${deps.providerId}/${deps.model}` +
+            (deps.providers && Object.keys(deps.providers).length > 0
+                ? ` (find_model: ${Object.keys(deps.providers).join(", ")})`
+                : " (no model providers — model-dependent cases skipped)"));
+        const evalOptions = {
+            provider: deps.provider,
+            model: deps.model,
+            registry: deps.registry,
+            providers: deps.providers,
+            cases,
+            onEvent: deps.onEvent
+        };
+        if (deps.maxIterations) {
+            evalOptions.maxIterations = deps.maxIterations;
+        }
+        const report = await runGraphPlannerEval(evalOptions);
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatEvalReport(report),
+            successRate: report.summary.successRate,
+            meanScore: report.summary.meanScore,
+            casesRan: report.summary.total - report.summary.skipped
+        };
+    }
+};
+/**
+ * End-to-end graph suite: plan a workflow, run it on the kernel, judge the
+ * outputs against the case's goal.
+ *
+ * This is the one suite that executes what the planner produced, so it needs a
+ * `GraphRunner` — built here over `ExecutionSession` — and it costs real
+ * inference twice per case (the run, then the judge). Its success rate is the
+ * end-to-end claim: planned AND ran AND achieved the goal.
+ */
+const graphE2eSuite = {
+    id: "graph-e2e",
+    description: "Plan a workflow with the agent, execute it, and judge whether the outputs achieve the goal",
+    async listCases() {
+        const { GRAPH_E2E_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return GRAPH_E2E_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description,
+            needsModelProviders: c.needsModelProviders
+        }));
+    },
+    async run(deps) {
+        const { GRAPH_E2E_EVAL_CASES, runGraphE2eEval, formatGraphE2eReport } = await import("@nodetool-ai/agents");
+        const { createEvalGraphRunner } = await import("../evals/graph-runner.js");
+        const cases = selectCases(GRAPH_E2E_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} graph-e2e case(s) with ${deps.providerId}/${deps.model}` +
+            (deps.providers && Object.keys(deps.providers).length > 0
+                ? ` (find_model: ${Object.keys(deps.providers).join(", ")})`
+                : " (no model providers — model-dependent cases skipped)"));
+        const evalOptions = {
+            provider: deps.provider,
+            model: deps.model,
+            registry: deps.registry,
+            providers: deps.providers,
+            runGraph: createEvalGraphRunner(),
+            cases,
+            timeoutMs: deps.timeoutMs,
+            onEvent: deps.onEvent
+        };
+        // Without this the judge is the run's own provider/model grading itself.
+        if (deps.judge) {
+            evalOptions.judgeProvider = deps.judge.provider;
+            evalOptions.judgeModel = deps.judge.model;
+        }
+        if (deps.maxIterations) {
+            evalOptions.maxIterations = deps.maxIterations;
+        }
+        const report = await runGraphE2eEval(evalOptions);
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatGraphE2eReport(report),
+            successRate: report.summary.successRate,
+            casesRan: report.summary.total - report.summary.skipped
+        };
+    }
+};
+/**
+ * Code node authoring suite (`CodePlanner` + `submit_code`).
+ *
+ * Its gate reads post-repair acceptance — a submission accepted at all within
+ * the round cap — while the report also carries first-pass acceptance, which is
+ * what the model managed without the tool's feedback.
+ */
+const codeGenSuite = {
+    id: "code-gen",
+    description: "Run the Code node authoring eval suite (CodePlanner writes one typed Code node) against a provider/model, reporting first-pass and post-repair acceptance",
+    async listCases() {
+        const { CODE_GEN_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return CODE_GEN_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description
+        }));
+    },
+    async run(deps) {
+        const { CODE_GEN_EVAL_CASES, runCodeGenEval, formatCodeGenReport } = await import("@nodetool-ai/agents");
+        const cases = selectCases(CODE_GEN_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} code-gen case(s) with ${deps.providerId}/${deps.model}`);
+        const report = await runCodeGenEval({
+            provider: deps.provider,
+            model: deps.model,
+            cases,
+            onEvent: deps.onEvent
+        });
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatCodeGenReport(report),
+            successRate: report.summary.postRepairRate,
+            // The suite skips nothing: every case is a Code body the planner writes.
+            casesRan: report.summary.total
+        };
+    }
+};
+/** Sub-agent execution suite (RunSubtaskTool + inherited toolset). */
+const subtaskSuite = {
+    id: "subtask",
+    description: "Run the sub-agent execution eval suite (run_subtask delegates to a child agent that runs the inherited tools) against a provider/model",
+    async listCases() {
+        const { SUBTASK_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return SUBTASK_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description,
+            needsModelProviders: c.needsModelProviders
+        }));
+    },
+    async run(deps) {
+        const { SUBTASK_EVAL_CASES, runSubtaskEval, formatSubtaskReport } = await import("@nodetool-ai/agents");
+        const cases = selectCases(SUBTASK_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} subtask case(s) with ${deps.providerId}/${deps.model}` +
+            (deps.providers && Object.keys(deps.providers).length > 0
+                ? ""
+                : " (no model providers — model-dependent cases skipped)"));
+        const report = await runSubtaskEval({
+            provider: deps.provider,
+            model: deps.model,
+            providers: deps.providers,
+            cases,
+            maxIterations: deps.maxIterations,
+            onEvent: deps.onEvent
+        });
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatSubtaskReport(report),
+            successRate: report.summary.successRate,
+            casesRan: report.summary.total - report.summary.skipped
+        };
+    }
+};
+/** CodeAct execution suite. */
+const codeActSuite = {
+    id: "codeact",
+    description: "Run the CodeAct execution eval suite (steps act by writing sandboxed JavaScript over the toolbelt — docs/codeact-design.md) against a provider/model",
+    async listCases() {
+        const { CODEACT_EVAL_CASES, CODEACT_API_EVAL_CASES, CODEACT_SANDBOX_PACK_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return [
+            ...CODEACT_EVAL_CASES,
+            ...CODEACT_API_EVAL_CASES,
+            ...CODEACT_SANDBOX_PACK_EVAL_CASES
+        ].map((c) => ({
+            id: c.id,
+            description: c.description
+        }));
+    },
+    async run(deps) {
+        const { CODEACT_EVAL_CASES, CODEACT_API_EVAL_CASES, CODEACT_SANDBOX_PACK_EVAL_CASES, runCodeActEval, formatCodeActReport } = await import("@nodetool-ai/agents");
+        const cases = selectCases([
+            ...CODEACT_EVAL_CASES,
+            ...CODEACT_API_EVAL_CASES,
+            ...CODEACT_SANDBOX_PACK_EVAL_CASES
+        ], deps.caseIds);
+        deps.log(`Running ${cases.length} codeact case(s) with ${deps.providerId}/${deps.model}`);
+        const report = await runCodeActEval({
+            provider: deps.provider,
+            model: deps.model,
+            cases,
+            maxIterations: deps.maxIterations,
+            onEvent: deps.onEvent
+        });
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatCodeActReport(report),
+            successRate: report.summary.successRate,
+            meanScore: report.summary.meanScore,
+            // The suite skips nothing: its toolbelt is instrumented, not configured.
+            casesRan: report.summary.total
+        };
+    }
+};
+/** TaskPlanner multi-task (plan mode) DAG-quality suite. */
+const taskPlannerSuite = {
+    id: "task-planner",
+    description: "Run the TaskPlanner (plan mode) eval suite — multi-task DAG quality: parallelism, decomposition size, tool routing — against a provider/model",
+    async listCases() {
+        const { TASK_PLANNER_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return TASK_PLANNER_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description,
+            needsModelProviders: c.needsModelProviders
+        }));
+    },
+    async run(deps) {
+        const { TASK_PLANNER_EVAL_CASES, runTaskPlannerEval, formatTaskPlanReport } = await import("@nodetool-ai/agents");
+        const cases = selectCases(TASK_PLANNER_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} task-planner case(s) with ${deps.providerId}/${deps.model}`);
+        const report = await runTaskPlannerEval({
+            provider: deps.provider,
+            model: deps.model,
+            providers: deps.providers,
+            cases,
+            systemPrompt: deps.systemPrompt,
+            onEvent: deps.onEvent
+        });
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatTaskPlanReport(report),
+            successRate: report.summary.successRate,
+            meanScore: report.summary.meanScore,
+            casesRan: report.summary.total - report.summary.skipped
+        };
+    }
+};
+/**
+ * Mini-app build suite: prompt in, verified `ApplicationBundle` out.
+ *
+ * The only suite that both plans workflows and runs them, so it needs the
+ * kernel runner (`runOnServer`, the same one `nodetool app debug` uses) and a
+ * real `ProcessingContext`. Its deterministic cases author from a script and
+ * never touch the provider, which is why it is the one suite that declares a
+ * keyless set: `--keyless` runs exactly those, with no API key at all. The gate
+ * reads green-within-budget; the report also carries the one-shot rate the PRD
+ * calls the north star.
+ */
+const appBuildSuite = {
+    id: "app-build",
+    description: "Run the mini-app build eval suite (nodetool app build end to end) against a provider/model, reporting one-shot and green-within-budget rates",
+    async listCases() {
+        const { APP_BUILD_EVAL_CASES } = await import("@nodetool-ai/agents");
+        return APP_BUILD_EVAL_CASES.map((c) => ({
+            id: c.id,
+            description: c.description,
+            needsModelProviders: c.needsModelProviders
+        }));
+    },
+    async keylessCaseIds() {
+        // The suite's own marker, not a list a caller copies: a case that gains or
+        // loses `deterministic` moves `--keyless` with it.
+        const { APP_BUILD_DETERMINISTIC_CASE_IDS } = await import("@nodetool-ai/agents");
+        return APP_BUILD_DETERMINISTIC_CASE_IDS;
+    },
+    async run(deps) {
+        const [{ APP_BUILD_EVAL_CASES, runAppBuildEval, formatAppBuildReport }, { runOnServer }, { getDefaultAssetsPath }, { resolveLocalSecret }, { ProcessingContext }, { FileStorageAdapter }] = await Promise.all([
+            import("@nodetool-ai/agents"),
+            import("../debug/server-runner.js"),
+            import("@nodetool-ai/config"),
+            import("../local-secrets.js"),
+            import("@nodetool-ai/runtime"),
+            import("@nodetool-ai/storage")
+        ]);
+        const { setupLocalDb } = await import("./local-db.js");
+        await setupLocalDb();
+        const cases = selectCases(APP_BUILD_EVAL_CASES, deps.caseIds);
+        deps.log(`Running ${cases.length} app-build case(s) with ${deps.providerId}/${deps.model}` +
+            (deps.providers && Object.keys(deps.providers).length > 0
+                ? ` (find_model: ${Object.keys(deps.providers).join(", ")})`
+                : " (no model providers — prompt cases skipped)"));
+        const evalOptions = {
+            provider: deps.provider,
+            model: deps.model,
+            registry: deps.registry,
+            context: new ProcessingContext({
+                jobId: `eval-app-build-${Date.now()}`,
+                workflowId: null,
+                userId: "1",
+                secretResolver: resolveLocalSecret,
+                storage: new FileStorageAdapter(getDefaultAssetsPath())
+            }),
+            runOnServer,
+            cases,
+            onEvent: deps.onEvent
+        };
+        if (deps.providers) {
+            evalOptions.providers = deps.providers;
+        }
+        // `--judge-model` overrides `buildApp`'s own judge default.
+        if (deps.judge) {
+            evalOptions.judge = {
+                enabled: true,
+                provider: deps.judge.provider,
+                model: deps.judge.model
+            };
+        }
+        if (deps.timeoutMs !== undefined) {
+            evalOptions.timeoutMs = deps.timeoutMs;
+        }
+        const report = await runAppBuildEval(evalOptions);
+        return {
+            report,
+            ...runMetrics(report.cases),
+            formatted: formatAppBuildReport(report),
+            successRate: report.summary.greenWithinBudgetRate,
+            meanScore: report.summary.meanScore,
+            casesRan: report.summary.ran
+        };
+    }
+};
+/**
+ * Build a tool-loop suite (multi-turn `ui_*` tool calling) from a named export
+ * in `@nodetool-ai/agents`. The graph-editor suite and the five editor-surface
+ * suites (script/sketch/timeline/storyboard/3D) all share the generic runner
+ * and report shape — only the case array differs — so each is data, not code.
+ */
+function makeToolLoopSuite(id, description, casesOf) {
+    const pickCases = (mod) => {
+        // The declaration says this is an array; a stale packages/*/dist can still
+        // hand back nothing, so say which suite lost its cases rather than failing
+        // later on `.map` of undefined.
+        const picked = casesOf(mod);
+        if (!Array.isArray(picked)) {
+            throw new Error(`Eval suite "${id}" found no case array in @nodetool-ai/agents. ` +
+                `A stale packages/*/dist is the usual cause — rebuild with ` +
+                `npm run build:packages.`);
+        }
+        return picked;
+    };
+    return {
+        id,
+        description,
+        async listCases() {
+            const mod = await import("@nodetool-ai/agents");
+            return pickCases(mod).map((c) => ({
+                id: c.id,
+                description: c.description,
+                needsModelProviders: c.needsModelProviders
+            }));
+        },
+        async run(deps) {
+            const mod = await import("@nodetool-ai/agents");
+            const { runToolLoopEval, formatToolLoopReport } = mod;
+            const cases = selectCases(pickCases(mod), deps.caseIds);
+            deps.log(`Running ${cases.length} ${id} case(s) with ${deps.providerId}/${deps.model}`);
+            const report = await runToolLoopEval({
+                provider: deps.provider,
+                model: deps.model,
+                cases,
+                maxIterations: deps.maxIterations,
+                onEvent: deps.onEvent
+            });
+            return {
+                report,
+                ...runMetrics(report.cases),
+                formatted: formatToolLoopReport(report),
+                successRate: report.summary.successRate,
+                meanScore: report.summary.meanScore,
+                casesRan: report.summary.total - report.summary.skipped
+            };
+        }
+    };
+}
+/** All evaluation suites exposed under `nodetool eval <suite>`. */
+export const EVAL_SUITES = [
+    graphPlannerSuite,
+    graphE2eSuite,
+    codeGenSuite,
+    taskPlannerSuite,
+    subtaskSuite,
+    codeActSuite,
+    appBuildSuite,
+    makeToolLoopSuite("tool-loop", "Run the frontend graph-editor tool-loop eval suite (ui_* graph tools) against a provider/model and report metrics", (mod) => mod.TOOL_LOOP_EVAL_CASES),
+    makeToolLoopSuite("workflow-escalation", "Run the workflow-tool escalation eval suite (ui_* graph tools plus an ask_user channel to a scripted user) against a provider/model", (mod) => mod.WORKFLOW_ESCALATION_TOOL_LOOP_CASES),
+    makeToolLoopSuite("script-tools", "Run the Script surface tool-loop eval suite (ui_script_* tools) against a provider/model", (mod) => mod.SCRIPT_TOOL_LOOP_CASES),
+    makeToolLoopSuite("jsscript-tools", "Run the JS-script surface tool-loop eval suite (ui_jsscript_* tools, real sandbox execution) against a provider/model", (mod) => mod.JS_SCRIPT_TOOL_LOOP_CASES),
+    makeToolLoopSuite("sketch-tools", "Run the Sketch/image-editor surface tool-loop eval suite (ui_sketch_* tools) against a provider/model", (mod) => mod.SKETCH_TOOL_LOOP_CASES),
+    makeToolLoopSuite("timeline-tools", "Run the Timeline/video-editor surface tool-loop eval suite (ui_timeline_* tools) against a provider/model", (mod) => mod.TIMELINE_TOOL_LOOP_CASES),
+    makeToolLoopSuite("storyboard-tools", "Run the Storyboard surface tool-loop eval suite (ui_storyboard_* tools) against a provider/model", (mod) => mod.STORYBOARD_TOOL_LOOP_CASES),
+    makeToolLoopSuite("model3d-tools", "Run the 3D model-editor surface tool-loop eval suite (ui_3d_* tools) against a provider/model", (mod) => mod.MODEL3D_TOOL_LOOP_CASES),
+    makeToolLoopSuite("app-tools", "Run the App Builder surface tool-loop eval suite (ui_app_* tools) against a provider/model", (mod) => mod.APP_TOOL_LOOP_CASES),
+    makeToolLoopSuite("memory-tools", "Run the memory tool-loop eval suite (memory_*/asset tools, real DB) against a provider/model", (mod) => mod.MEMORY_TOOL_LOOP_CASES),
+    makeToolLoopSuite("creative-pipeline", "Run the long-horizon creative eval: one commission carried through brief, ideation, sketch, storyboard, cut and review across the composed ui_sketch_*/ui_storyboard_*/ui_timeline_* surfaces", (mod) => mod.CREATIVE_PIPELINE_TOOL_LOOP_CASES)
+];
+/**
+ * The case ids one invocation should run: `--cases` verbatim, `--keyless` from
+ * the suite's own declaration, or undefined for the whole suite.
+ */
+async function resolveCaseIds(suite, opts) {
+    if (opts.keyless) {
+        if (opts.cases) {
+            throw new Error("--keyless and --cases select cases two ways; pass one");
+        }
+        const ids = suite.keylessCaseIds ? await suite.keylessCaseIds() : [];
+        if (ids.length === 0) {
+            const offered = EVAL_SUITES.filter((s) => s.id !== suite.id && s.keylessCaseIds).map((s) => s.id);
+            throw new Error(`Suite "${suite.id}" has no keyless cases: every case it runs needs a ` +
+                `live provider.` +
+                (offered.length > 0 ? ` Suites that do: ${offered.join(", ")}.` : ""));
+        }
+        return [...ids];
+    }
+    if (!opts.cases)
+        return undefined;
+    return opts.cases
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+}
+/**
+ * Shared runner for every suite: handles `--list`, builds the provider/
+ * registry/find-model deps, runs the suite, and applies the common
+ * `--out`/`--json`/`--min-success` handling.
+ */
+async function runSuite(suite, opts) {
+    if (opts.list) {
+        const keyless = new Set(suite.keylessCaseIds ? await suite.keylessCaseIds() : []);
+        // Marked here rather than described in a doc: which cases run without a key
+        // is a property of the suite, and this is the command that reports it.
+        const cases = (await suite.listCases()).map((c) => ({
+            ...c,
+            keyless: keyless.has(c.id)
+        }));
+        if (opts.json) {
+            console.log(JSON.stringify(cases, null, 2));
+            return;
+        }
+        for (const c of cases) {
+            console.log(`${c.id.padEnd(24)} ${c.description}` +
+                (c.keyless ? " (keyless)" : "") +
+                (c.needsModelProviders ? " (needs model providers)" : ""));
+        }
+        return;
+    }
+    if (!opts.provider || !opts.model) {
+        const message = "--provider and --model are required (or use --list)";
+        console.error(message);
+        if (opts.json)
+            console.log(JSON.stringify({ error: message }, null, 2));
+        process.exitCode = 1;
+        return;
+    }
+    try {
+        const [{ createProviderStrict, buildConfiguredProviders }, registryMod] = await Promise.all([
+            import("../providers.js"),
+            import("../node-registry.js")
+        ]);
+        const provider = await createProviderStrict(opts.provider);
+        const registry = registryMod.buildFullRegistry();
+        const providers = opts.findModel === false ? undefined : await buildConfiguredProviders();
+        const caseIds = await resolveCaseIds(suite, opts);
+        const judge = opts.judgeModel
+            ? await resolveJudge(opts.judgeModel, createProviderStrict)
+            : undefined;
+        const runOptions = {
+            provider,
+            providerId: opts.provider,
+            model: opts.model,
+            registry,
+            providers,
+            caseIds,
+            maxIterations: opts.maxIterations !== undefined
+                ? parseNumericOption(opts.maxIterations, "--max-iterations", {
+                    integer: true,
+                    min: 1
+                })
+                : undefined,
+            timeoutMs: opts.timeout !== undefined
+                ? parseNumericOption(opts.timeout, "--timeout", {
+                    integer: true,
+                    min: 0
+                })
+                : undefined,
+            log: (line) => {
+                if (!opts.json)
+                    console.log(line);
+            },
+            onEvent: (line) => {
+                if (!opts.json)
+                    console.log(line);
+            }
+        };
+        // `--no-agent-prompt` scores the bare contract; `--system-prompt` swaps in
+        // another caller's. Neither given, the suite plans behind the prompt the
+        // product actually sends.
+        if (opts.agentPrompt === false) {
+            runOptions.systemPrompt = "";
+        }
+        else if (opts.systemPrompt !== undefined) {
+            runOptions.systemPrompt = opts.systemPrompt;
+        }
+        if (judge) {
+            runOptions.judge = judge;
+        }
+        const result = await suite.run(runOptions);
+        if (opts.out) {
+            const { writeFile } = await import("node:fs/promises");
+            await writeFile(opts.out, JSON.stringify(result.report, null, 2), "utf-8");
+            if (!opts.json)
+                console.log(`Report written to ${opts.out}`);
+        }
+        if (opts.json) {
+            console.log(JSON.stringify(result.report, null, 2));
+        }
+        else {
+            console.log("\n" + result.formatted);
+        }
+        const gateFailures = evalGateFailures(result, suite.id, opts);
+        for (const line of gateFailures)
+            console.error(line);
+        if (gateFailures.length > 0)
+            process.exitCode = 1;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(message);
+        // An agent reading `--json` gets parseable output on the failure path too.
+        if (opts.json)
+            console.log(JSON.stringify({ error: message }, null, 2));
+        process.exitCode = 1;
+    }
+}
+/**
+ * Resolve `--judge-model` (`provider/model`) into a live provider. Model ids
+ * carry slashes themselves, so the split is decided by the provider registry,
+ * exactly as `--supervisor-model` does.
+ */
+async function resolveJudge(spec, createProvider) {
+    const { listRegisteredProviderIds } = await import("@nodetool-ai/runtime");
+    const registered = listRegisteredProviderIds();
+    const cut = spec.indexOf("/");
+    const head = cut === -1 ? spec : spec.slice(0, cut).toLowerCase();
+    if (cut === -1 || !registered.includes(head)) {
+        throw new Error(`--judge-model must be "<provider>/<model>" with a registered provider ` +
+            `(got "${spec}"). Example: openai/gpt-5.4-mini.`);
+    }
+    return { provider: await createProvider(head), model: spec.slice(cut + 1) };
+}
+export function registerEvalCommand(program) {
+    const evalCmd = program
+        .command("eval")
+        .description("Evaluation suites for the agent system");
+    for (const suite of EVAL_SUITES) {
+        evalCmd
+            .command(suite.id)
+            .description(suite.description)
+            .option("-p, --provider <id>", "Provider id (anthropic, openai, claude_agent_sdk, ollama, ...)")
+            .option("-m, --model <id>", "Model id for the provider")
+            .option("--cases <ids>", "Comma-separated case ids to run (default: all; see --list)")
+            .option("--list", "List available cases and exit")
+            .option("--json", "Print the full report as JSON")
+            .option("--out <path>", "Write the JSON report to a file")
+            .option("--max-iterations <n>", "Turn cap per case for loop-style suites (tool-loop; default 12)")
+            .option("--timeout <ms>", "Per-case execution timeout for suites that run what they plan (graph-e2e; default 300000)")
+            .option("--judge-model <provider/model>", "Model that judges outputs for the self-judging suites (graph-e2e, app-build). Default: the run's own provider/model, which grades its own work")
+            .option("--min-success <rate>", "Exit non-zero when the success rate is below this threshold (0..1)")
+            .option("--min-cases <n>", "Exit non-zero when fewer than n cases actually ran (skipped cases do not count)")
+            .option("--keyless", "Run only the cases that need no API key and no network (see --list)")
+            .option("--no-find-model", "Run without configured model providers (skips model-dependent cases)")
+            .option("--min-score <rate>", "Exit non-zero when the mean expectation score is below this threshold (0..1)")
+            .option("--system-prompt <text>", "Caller preamble the planner suites plan behind (default: the AgentNode's own prompt)")
+            .option("--no-agent-prompt", "Plan with no caller preamble — scores the bare planner contract")
+            .option("--max-cost-usd <usd>", "Exit non-zero when the run's total provider spend exceeds this")
+            .option("--max-p95-duration-ms <ms>", "Exit non-zero when the p95 case duration (skipped cases excluded) exceeds this")
+            .action((opts) => runSuite(suite, opts));
+    }
+}
+//# sourceMappingURL=eval.js.map