@nodetool-ai/cli 0.7.0-rc.23 → 0.8.2

package/dist/commands/eval.d.ts

@@ -0,0 +1,152 @@
+/**
+ * `nodetool eval <suite>` — run a registered evaluation suite against any
+ * provider and print/save a metrics report.
+ *
+ * Suites are data, not code: each entry in `EVAL_SUITES` describes one suite
+ * (its subcommand name, how to list its cases, and how to run it). The shared
+ * runner wires up a provider from the runtime registry, the full TS node
+ * registry, and (when available) configured model providers for `find_model`,
+ * then hands them to the suite. Adding a suite means pushing an `EvalSuite`
+ * entry — not another hand-wired command block.
+ *
+ * The suites live in `@nodetool-ai/agents`: graph authoring
+ * (`GRAPH_PLANNER_EVAL_CASES`, typed-DSL CodeAct) and the frontend tool-loop
+ * (`TOOL_LOOP_EVAL_CASES`, multi-turn `ui_*` tool calling). Heavy deps are
+ * imported lazily so command registration stays light.
+ */
+import type { Command } from "commander";
+import type { BaseProvider } from "@nodetool-ai/runtime";
+import type { NodeRegistry } from "@nodetool-ai/node-sdk";
+interface EvalCliOptions {
+    provider?: string;
+    model?: string;
+    cases?: string;
+    list?: boolean;
+    json?: boolean;
+    out?: string;
+    maxIterations?: string;
+    timeout?: string;
+    minSuccess?: string;
+    /** Gate on the mean expectation score, for suites that compute one. */
+    minScore?: string;
+    /** `provider/model` for the suites that judge their own output. */
+    judgeModel?: string;
+    /** Exit non-zero when fewer cases than this actually ran. */
+    minCases?: string;
+    /** Run only the cases the suite declares keyless (no API key, no network). */
+    keyless?: boolean;
+    /** commander negated flag: `--no-find-model` sets this to false. */
+    findModel?: boolean;
+    /** Caller preamble for the planner suites; `--bare-prompt` clears it. */
+    systemPrompt?: string;
+    /** commander negated flag: `--no-agent-prompt` sets this to false. */
+    agentPrompt?: boolean;
+    /** Exit non-zero when the run's total provider spend (USD) exceeds this. */
+    maxCostUsd?: string;
+    /** Exit non-zero when the p95 case duration (ms) exceeds this. */
+    maxP95DurationMs?: string;
+}
+/** Metadata for one eval case, shown by `--list`. */
+interface EvalCaseMeta {
+    id: string;
+    description: string;
+    /** Case needs configured model providers (`find_model`) to be solvable. */
+    needsModelProviders?: boolean;
+    /** Case runs with no API key and no network — the set `--keyless` runs. */
+    keyless?: boolean;
+}
+/** Runtime deps handed to a suite's `run`, built once by the shared runner. */
+interface EvalRunDeps {
+    /** Provider constructed from `--provider`. */
+    provider: BaseProvider;
+    /** The raw `--provider` id (for display). */
+    providerId: string;
+    model: string;
+    registry: NodeRegistry;
+    /** Configured providers for `find_model`; undefined when `--no-find-model`. */
+    providers?: Record<string, BaseProvider>;
+    /** Case ids to run; undefined runs the whole suite. */
+    caseIds?: string[];
+    /** Turn cap per case, for loop-style suites (tool-loop). */
+    maxIterations?: number;
+    /** Per-case execution timeout (ms), for suites that run what they plan. */
+    timeoutMs?: number;
+    /**
+     * Judge provider/model for the suites that grade their own output
+     * (`graph-e2e`, `app-build`), from `--judge-model`. Undefined leaves each
+     * suite's own default in place.
+     */
+    judge?: {
+        provider: BaseProvider;
+        model: string;
+    };
+    /**
+     * Human-facing line (suite preamble). A no-op under `--json`, so stdout
+     * carries exactly one JSON document.
+     */
+    log: (line: string) => void;
+    /** Progress callback (one line per event, for CLI display). */
+    onEvent: (line: string) => void;
+    /**
+     * Caller system prompt the planner suites plan behind. Undefined keeps the
+     * suite's own default (the AgentNode's prompt — what the product sends);
+     * `""` scores the bare planner contract.
+     */
+    systemPrompt?: string;
+}
+/**
+ * Threshold gates for one suite run: the reasons it should exit non-zero.
+ * Extracted from the command action so both gates — including the failing
+ * paths, which otherwise only a paid model run reaches — are unit-testable.
+ */
+export declare function evalGateFailures(result: Pick<EvalRunResult, "successRate" | "meanScore" | "casesRan" | "costUsd" | "durationsMs">, suiteId: string, opts: Pick<EvalCliOptions, "minScore" | "minSuccess" | "minCases" | "maxCostUsd" | "maxP95DurationMs">): string[];
+/** Outcome of a suite run, consumed by the shared runner. */
+interface EvalRunResult {
+    /** Opaque report object; written verbatim when `--out`/`--json` are set. */
+    report: unknown;
+    /** Human-readable report for the default (non-JSON) output. */
+    formatted: string;
+    /** Overall success rate (0..1) for the `--min-success` gate. */
+    successRate: number;
+    /**
+     * Mean expectation score (0..1), where the suite computes one, for the
+     * `--min-score` gate. `successRate` alone cannot gate quality: the planner
+     * suites count a *committed* plan as a success, so a plan that lost half its
+     * parallelism still reports pass. Both gates apply when both are given.
+     */
+    meanScore?: number;
+    /**
+     * Cases that actually ran, skipped ones excluded. `--min-cases` reads it, so
+     * a gate can assert what it examined instead of trusting a rate computed
+     * over whatever survived.
+     */
+    casesRan: number;
+    /** Provider spend over the whole run, for `--max-cost-usd`. */
+    costUsd: number;
+    /** Wall clock per case that ran, for `--max-p95-duration-ms`. */
+    durationsMs: readonly number[];
+}
+/**
+ * A registered evaluation suite. Adding a suite is data — push an entry to
+ * `EVAL_SUITES` — not another hand-wired command block.
+ */
+interface EvalSuite {
+    /** Subcommand name, e.g. `graph-planner` (`nodetool eval graph-planner`). */
+    id: string;
+    description: string;
+    /** Case metadata for `--list`. */
+    listCases(): Promise<EvalCaseMeta[]>;
+    /** Run the suite against the built deps. */
+    run(deps: EvalRunDeps): Promise<EvalRunResult>;
+    /**
+     * The case ids that need neither credentials nor network — what `--keyless`
+     * runs. A suite with none omits the hook, and the flag is refused by name
+     * rather than running the whole suite against a provider nobody configured.
+     */
+    keylessCaseIds?(): Promise<readonly string[]>;
+}
+/** All evaluation suites exposed under `nodetool eval <suite>`. */
+export declare const EVAL_SUITES: readonly EvalSuite[];
+export declare function registerEvalCommand(program: Command): void;
+export {};
+//# sourceMappingURL=eval.d.ts.map

package/dist/commands/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/commands/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAI1D,UAAU,cAAc;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,yEAAyE;IACzE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,qDAAqD;AACrD,UAAU,YAAY;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,2EAA2E;IAC3E,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,+EAA+E;AAC/E,UAAU,WAAW;IACnB,8CAA8C;IAC9C,QAAQ,EAAE,YAAY,CAAC;IACvB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,CAAC;IACvB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACzC,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,YAAY,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAClD;;;OAGG;IACH,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,+DAA+D;IAC/D,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,IAAI,CACV,aAAa,EACb,aAAa,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,aAAa,CACrE,EACD,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,IAAI,CACR,cAAc,EACd,UAAU,GAAG,YAAY,GAAG,UAAU,GAAG,YAAY,GAAG,kBAAkB,CAC3E,GACA,MAAM,EAAE,CAkFV;AA2BD,6DAA6D;AAC7D,UAAU,aAAa;IACrB,4EAA4E;IAC5E,MAAM,EAAE,OAAO,CAAC;IAChB,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;CAChC;AAED;;;GAGG;AACH,UAAU,SAAS;IACjB,6EAA6E;IAC7E,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACrC,4CAA4C;IAC5C,GAAG,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C;;;;OAIG;IACH,cAAc,CAAC,IAAI,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;CAC/C;AAqfD,mEAAmE;AACnE,eAAO,MAAM,WAAW,EAAE,SAAS,SAAS,EA+D3C,CAAC;AAqLF,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAuE1D"}