@valerypopoff/rivet2-cli 2.1.9 → 2.3.0

package/README.md

@@ -10,11 +10,14 @@ npx @valerypopoff/rivet2-cli list my-project.rivet-project
 npx @valerypopoff/rivet2-cli inspect my-project.rivet-project
 npx @valerypopoff/rivet2-cli doctor my-project.rivet-project
 npx @valerypopoff/rivet2-cli run my-project.rivet-project
+npx @valerypopoff/rivet2-cli evaluations run --project my-project.rivet-project --suite-file evaluation.json
 npx @valerypopoff/rivet2-cli serve my-project.rivet-project --port 8080
 npx @valerypopoff/rivet2-cli serve-app my-project.rivet-project "My web app"
 npx @valerypopoff/rivet2-cli completion
 ```
 
+The Evaluations command consumes a **suite + dataset** bundle exported from Rivet's application-local Evaluations workspace. The project supplies its target and evaluator graphs; the bundle does not contain graph definitions, baselines, run history, or recordings. Use `--benchmark` for measurement without quality checks, `--json` for a complete `EvaluationRun`, or `--junit` for CI.
+
 The CLI package exposes the `rivet` binary when installed globally:
 
 ```bash

package/bin/cli.js

@@ -7,7 +7,9 @@ import { doctor, makeDoctorCommand } from './commands/doctor.js';
 import { makeCommand as makeRunCommand, run } from './commands/run.js';
 import { makeCommand as makeServeCommand, serve } from './commands/serve.js';
 import { makeCommand as makeServeAppCommand, serveApp } from './commands/serveApp.js';
+import { EvaluationCliError, makeEvaluationCommand, runEvaluation } from './commands/evaluations.js';
 await yargs(hideBin(process.argv))
+    .command('evaluations run', 'Run a named Evaluation suite.', (y) => makeEvaluationCommand(y), (args) => runCli(() => runEvaluation(args)))
     .command('list <projectFile>', 'List graphs, web apps, library nodes, and plugins in a project file.', (y) => makeListCommand(y), (args) => runCli(() => list(args)))
     .command('inspect <projectFile>', 'Print a machine-readable project summary.', (y) => makeInspectCommand(y), (args) => runCli(() => inspect(args.projectFile)))
     .command('doctor <projectFile>', 'Check a project file for common CLI/runtime problems.', (y) => makeDoctorCommand(y), (args) => runCli(() => doctor(args)))
@@ -23,7 +25,7 @@ async function runCli(action) {
     }
     catch (err) {
         console.error(chalk.red(formatCliError(err)));
-        process.exit(1);
+        process.exit(err instanceof EvaluationCliError ? err.exitCode : 1);
     }
 }
 function formatCliError(err) {

package/bin/commands/evaluations.js

@@ -0,0 +1,350 @@
+import { createProcessor, loadProjectAndAttachedDataFromFile, } from '@valerypopoff/rivet2-node';
+import { deserializeEvaluationProjectData, deserializeEvaluationSuiteBundleJson, EvaluationGraphExecutionError, runEvaluationSuite, validateEvaluationDataset, } from '@valerypopoff/rivet2-evaluations';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { addDatasetOptions, addProviderOptions, createDatasetProvider, getProjectFile, resolveDatasetFilePath, withCliProcessorOptions, } from '../cliRuntime.js';
+export class EvaluationCliError extends Error {
+    exitCode;
+    constructor(message, exitCode) {
+        super(message);
+        this.exitCode = exitCode;
+    }
+}
+export function makeEvaluationCommand(y) {
+    return addDatasetOptions(addProviderOptions(y))
+        .option('project', {
+        demandOption: true,
+        describe: 'The currently runnable Rivet project containing the target and evaluator graphs',
+        type: 'string',
+    })
+        .option('suite', {
+        describe: 'Evaluation suite ID or name in a legacy project attachment; omit when using --suite-file',
+        type: 'string',
+    })
+        .option('suite-file', {
+        describe: 'An Export suite + dataset JSON bundle from the Evaluations workspace',
+        type: 'string',
+    })
+        .option('trials', { describe: 'Override the suite trial count', type: 'number' })
+        .option('concurrency', { describe: 'Override the suite worker-pool concurrency (1–32)', type: 'number' })
+        .option('baseline', { describe: 'Use this saved baseline ID instead of the suite default', type: 'string' })
+        .option('benchmark', {
+        describe: 'Measure execution without requiring or applying output-quality checks',
+        type: 'boolean',
+        default: false,
+    })
+        .option('json', { describe: 'Write the complete evaluation run as JSON', type: 'boolean', default: false })
+        .option('junit', { describe: 'Write a JUnit XML report', type: 'boolean', default: false });
+}
+function findSuite(data, input) {
+    return data.suites.find((suite) => suite.id === input || suite.name === input);
+}
+async function loadEvaluationSuiteBundle(path) {
+    let source;
+    const resolvedPath = resolve(process.cwd(), path);
+    try {
+        source = await readFile(resolvedPath, 'utf8');
+    }
+    catch (error) {
+        throw new EvaluationCliError(`Could not read evaluation suite bundle "${resolvedPath}": ${error instanceof Error ? error.message : String(error)}`, 3);
+    }
+    try {
+        const bundle = deserializeEvaluationSuiteBundleJson(source, {
+            suiteId: 'cli-imported-suite',
+            datasetId: 'cli-imported-dataset',
+        });
+        return { data: { version: 1, suites: [bundle.suite], baselines: [] }, dataset: bundle.dataset };
+    }
+    catch (error) {
+        throw new EvaluationCliError(`Could not import evaluation suite bundle "${resolvedPath}": ${error instanceof Error ? error.message : String(error)}`, 3);
+    }
+}
+async function loadEvaluationDataset(projectPath, projectId, datasetId, datasetFile) {
+    const path = resolveDatasetFilePath(projectPath, datasetFile);
+    let content;
+    try {
+        content = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        throw new EvaluationCliError(`Could not read evaluation dataset file "${path}": ${error instanceof Error ? error.message : String(error)}`, 3);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        throw new EvaluationCliError(`Evaluation dataset file "${path}" is not valid JSON.`, 3);
+    }
+    const dataset = Array.isArray(parsed.evaluationDatasets)
+        ? parsed.evaluationDatasets.find((candidate) => candidate.id === datasetId && candidate.projectId === projectId)
+        : undefined;
+    if (!dataset)
+        throw new EvaluationCliError(`Evaluation dataset "${datasetId}" was not found in "${path}".`, 3);
+    return validateEvaluationDataset(dataset);
+}
+function graphInputsToOutputs(project, graphId, inputs) {
+    const graph = project.graphs[graphId];
+    if (!graph)
+        throw new Error(`Evaluation graph "${graphId}" does not exist.`);
+    const graphInputs = new Map(graph.nodes
+        .filter((node) => node.type === 'graphInput')
+        .map((node) => [node.data.id, node]));
+    return Object.fromEntries(Object.entries(inputs).map(([id, value]) => {
+        const input = graphInputs.get(id);
+        if (!input)
+            throw new Error(`Evaluation supplied unknown graph input "${id}".`);
+        return [id, { type: input.data.dataType, value }];
+    }));
+}
+export function formatEvaluationRunAsJUnit(run) {
+    const escape = (value) => value.replace(/[<>&"']/gu, (character) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', '"': '&quot;', "'": '&apos;' })[character]);
+    const cases = run.trials.map((trial) => {
+        const observationDetails = trial.observations
+            .filter((observation) => observation.status !== 'passed')
+            .map((observation) => observation.message ?? observation.name)
+            .join('; ');
+        const details = trial.error ?? (observationDetails || trial.qualityReason.message);
+        const body = trial.executionStatus === 'error'
+            ? `<error message="${escape(details || 'Evaluation execution error')}" />`
+            : trial.executionStatus === 'canceled'
+                ? `<skipped message="Evaluation execution was canceled." />`
+                : trial.qualityStatus === 'unable-to-evaluate'
+                    ? `<error message="${escape(details || 'Evaluation infrastructure error')}" />`
+                    : trial.qualityStatus === 'failed'
+                        ? `<failure message="${escape(details || 'Evaluation failed')}" />`
+                        : trial.qualityStatus === 'not-evaluated'
+                            ? `<skipped message="${escape(trial.qualityReason.message)}" />`
+                            : '';
+        return {
+            xml: `<testcase name="${escape(trial.caseName)} [trial ${trial.trialIndex + 1}]" time="${(trial.totalMetrics.durationMs / 1000).toFixed(3)}">${body}</testcase>`,
+            outcome: trial.executionStatus === 'error' ||
+                (trial.executionStatus === 'completed' && trial.qualityStatus === 'unable-to-evaluate')
+                ? 'error'
+                : trial.executionStatus === 'canceled' || trial.qualityStatus === 'not-evaluated'
+                    ? 'skipped'
+                    : trial.qualityStatus === 'failed'
+                        ? 'failure'
+                        : 'passed',
+        };
+    });
+    const representedRunQuality = cases.some((item) => run.qualityStatus === 'failed'
+        ? item.outcome === 'failure'
+        : run.qualityStatus === 'unable-to-evaluate'
+            ? item.outcome === 'error'
+            : false);
+    const representedRunExecution = cases.some((item) => run.executionStatus === 'error'
+        ? item.outcome === 'error'
+        : run.executionStatus === 'canceled'
+            ? item.outcome === 'skipped'
+            : false);
+    if (run.executionStatus !== 'completed' && !representedRunExecution) {
+        const details = run.qualityReason.message;
+        cases.push(run.executionStatus === 'canceled'
+            ? {
+                xml: `<testcase name="Evaluation execution"><skipped message="${escape(details)}" /></testcase>`,
+                outcome: 'skipped',
+            }
+            : {
+                xml: `<testcase name="Evaluation execution"><error message="${escape(details)}" /></testcase>`,
+                outcome: 'error',
+            });
+    }
+    else if (run.executionStatus === 'completed' &&
+        !representedRunQuality &&
+        !cases.some((item) => item.outcome === 'error')) {
+        const thresholdDetails = run.thresholdResults
+            .filter((result) => result.status !== 'passed')
+            .map((result) => result.message)
+            .join('; ');
+        const details = thresholdDetails || run.qualityReason.message;
+        if (run.qualityStatus === 'failed') {
+            cases.push({
+                xml: `<testcase name="Evaluation aggregate requirements"><failure message="${escape(details)}" /></testcase>`,
+                outcome: 'failure',
+            });
+        }
+        else if (run.qualityStatus === 'unable-to-evaluate' ||
+            (run.purpose === 'evaluation' && run.qualityStatus === 'not-evaluated')) {
+            cases.push({
+                xml: `<testcase name="Evaluation aggregate requirements"><error message="${escape(details)}" /></testcase>`,
+                outcome: 'error',
+            });
+        }
+    }
+    const failures = cases.filter((item) => item.outcome === 'failure').length;
+    const errors = cases.filter((item) => item.outcome === 'error').length;
+    const skipped = cases.filter((item) => item.outcome === 'skipped').length;
+    return `<?xml version="1.0" encoding="UTF-8"?><testsuite name="${escape(run.suiteName)}" tests="${cases.length}" failures="${failures}" errors="${errors}" skipped="${skipped}">${cases.map((item) => item.xml).join('')}</testsuite>`;
+}
+export function evaluationRunFailure(run) {
+    if (run.executionStatus !== 'completed') {
+        return new EvaluationCliError(`Evaluation execution ${run.executionStatus}: ${run.qualityReason.message}`, 3);
+    }
+    const erroredTrials = run.aggregate?.erroredTrialCount ?? run.trials.filter((trial) => trial.executionStatus === 'error').length;
+    const canceledTrials = run.aggregate?.canceledTrialCount ?? run.trials.filter((trial) => trial.executionStatus === 'canceled').length;
+    if (erroredTrials > 0 || canceledTrials > 0) {
+        return new EvaluationCliError(`Evaluation execution completed with ${erroredTrials} errored and ${canceledTrials} canceled trials.`, 3);
+    }
+    if (run.qualityStatus === 'failed') {
+        return new EvaluationCliError('Evaluation thresholds or required checks failed.', 2);
+    }
+    if (run.qualityStatus === 'unable-to-evaluate') {
+        return new EvaluationCliError(run.qualityReason.message, 3);
+    }
+    if (run.purpose === 'evaluation' && run.qualityStatus === 'not-evaluated') {
+        return new EvaluationCliError(run.qualityReason.message, 3);
+    }
+    return undefined;
+}
+export async function runEvaluation(args) {
+    if (!args.project || (!args.suite && !args.suiteFile)) {
+        throw new EvaluationCliError('Provide --project and either --suite-file or --suite for a legacy project attachment.', 3);
+    }
+    const projectPath = await getProjectFile(args.project);
+    const [project, attachedData] = await loadProjectAndAttachedDataFromFile(projectPath);
+    const importedBundle = args.suiteFile === undefined ? undefined : await loadEvaluationSuiteBundle(args.suiteFile);
+    const rawData = attachedData.evaluations;
+    if (!importedBundle && !rawData) {
+        throw new EvaluationCliError('This project has no legacy Evaluation definitions. Export the suite + dataset from Rivet and pass it with --suite-file.', 3);
+    }
+    const evaluationData = importedBundle?.data ?? deserializeEvaluationProjectData(rawData);
+    const suite = importedBundle?.data.suites[0] ?? findSuite(evaluationData, args.suite);
+    if (!suite)
+        throw new EvaluationCliError(`Evaluation suite "${args.suite}" was not found.`, 3);
+    if (args.baseline !== undefined && importedBundle) {
+        throw new EvaluationCliError('Exported suite bundles do not include baselines; omit --baseline or use a legacy project attachment.', 3);
+    }
+    const dataset = importedBundle?.dataset ?? (await loadEvaluationDataset(projectPath, project.metadata.id, suite.datasetId, args.datasetFile));
+    const datasetProvider = await createDatasetProvider(projectPath, args);
+    const effectiveData = args.trials === undefined && args.concurrency === undefined
+        ? evaluationData
+        : {
+            ...evaluationData,
+            suites: evaluationData.suites.map((candidate) => candidate.id === suite.id
+                ? {
+                    ...candidate,
+                    configuration: {
+                        ...candidate.configuration,
+                        ...(args.trials === undefined ? {} : { trialCount: args.trials }),
+                        ...(args.concurrency === undefined ? {} : { concurrency: args.concurrency }),
+                    },
+                }
+                : candidate),
+        };
+    const actualSuite = effectiveData.suites.find((candidate) => candidate.id === suite.id);
+    const selectedBaseline = args.baseline === undefined
+        ? undefined
+        : effectiveData.baselines.find((candidate) => candidate.id === args.baseline && candidate.suiteId === actualSuite.id);
+    if (args.baseline !== undefined && !selectedBaseline) {
+        throw new EvaluationCliError(`Baseline "${args.baseline}" was not found for evaluation suite "${actualSuite.name}".`, 3);
+    }
+    const run = await runEvaluationSuite({
+        project,
+        evaluationData: effectiveData,
+        dataset,
+        suiteId: actualSuite.id,
+        purpose: args.benchmark ? 'execution-benchmark' : 'evaluation',
+        ...(selectedBaseline === undefined ? {} : { baseline: selectedBaseline }),
+        executionMode: 'node-cli',
+        runGraph: async ({ graphId, inputs, signal, metadata }) => {
+            const startedAt = Date.now();
+            const metrics = {
+                durationMs: 0,
+                modelCallCount: 0,
+                toolCallCount: 0,
+                toolFailureCount: 0,
+            };
+            const providerAttempts = [];
+            const processorInfo = createProcessor(project, withCliProcessorOptions({ datasetProvider, projectPath }, args, {
+                graph: graphId,
+                inputs: graphInputsToOutputs(project, graphId, inputs),
+                abortSignal: signal,
+                evaluation: metadata,
+            }));
+            processorInfo.processor.on('llmCallFinished', (event) => {
+                metrics.modelCallCount = (metrics.modelCallCount ?? 0) + 1;
+                metrics.inputTokens = (metrics.inputTokens ?? 0) + (event.normalizedUsage?.promptTokens ?? 0);
+                metrics.outputTokens = (metrics.outputTokens ?? 0) + (event.normalizedUsage?.completionTokens ?? 0);
+                metrics.cachedInputTokens = (metrics.cachedInputTokens ?? 0) + (event.normalizedUsage?.cachedTokens ?? 0);
+                metrics.reasoningTokens = (metrics.reasoningTokens ?? 0) + (event.normalizedUsage?.reasoningTokens ?? 0);
+                if (event.pricing.status === 'known')
+                    metrics.costUsd = (metrics.costUsd ?? 0) + (event.pricing.costUsd ?? 0);
+                else
+                    metrics.hasUnknownCost = true;
+                providerAttempts.push({
+                    kind: 'provider-call',
+                    provider: event.provider,
+                    model: event.model,
+                    customProviderApi: event.customProviderApi ?? null,
+                    outcome: event.outcome,
+                    profileIndex: event.profileIndex ?? null,
+                    attemptIndex: event.attemptIndex,
+                    roundIndex: event.roundIndex ?? null,
+                    durationMs: event.durationMs ?? null,
+                });
+            });
+            processorInfo.processor.on('llmProfileAttempt', (event) => {
+                providerAttempts.push({
+                    kind: 'profile-decision',
+                    provider: event.provider,
+                    model: event.model,
+                    customProviderApi: event.customProviderApi ?? null,
+                    stage: event.stage,
+                    outcome: event.outcome,
+                    profileIndex: event.profileIndex ?? null,
+                    attemptIndex: event.attemptIndex ?? null,
+                    roundIndex: event.roundIndex,
+                    status: event.status ?? null,
+                    healthState: event.healthState ?? null,
+                    healthDisposition: event.healthDisposition ?? null,
+                    timeoutKind: event.timeoutKind ?? null,
+                });
+            });
+            processorInfo.processor.on('toolCallFinished', (event) => {
+                metrics.toolCallCount = (metrics.toolCallCount ?? 0) + 1;
+                if (event.outcome !== 'success')
+                    metrics.toolFailureCount = (metrics.toolFailureCount ?? 0) + 1;
+            });
+            try {
+                const outputs = await processorInfo.run();
+                metrics.durationMs = Date.now() - startedAt;
+                return {
+                    outputs: Object.fromEntries(Object.entries(outputs).map(([key, value]) => [key, value.value])),
+                    metrics,
+                    ...(providerAttempts.length === 0 ? {} : { providerAttempts }),
+                };
+            }
+            catch (error) {
+                metrics.durationMs = Math.max(metrics.durationMs, Date.now() - startedAt);
+                throw new EvaluationGraphExecutionError(error instanceof Error ? error.message : String(error), {
+                    metrics,
+                    ...(providerAttempts.length === 0 ? {} : { providerAttempts }),
+                });
+            }
+            finally {
+                processorInfo.dispose();
+            }
+        },
+    });
+    if (args.junit)
+        process.stdout.write(`${formatEvaluationRunAsJUnit(run)}\n`);
+    else if (args.json)
+        process.stdout.write(`${JSON.stringify(run, null, 2)}\n`);
+    else {
+        const measured = run.purpose === 'execution-benchmark'
+            ? `${run.aggregate?.trialCount ?? 0} trials measured`
+            : run.evaluationMode === 'scoring'
+                ? `${formatScore(run.aggregate?.meanScore)}; ${run.aggregate?.scoredTrialCount ?? 0}/${run.aggregate?.trialCount ?? 0} trials scored`
+                : `${run.aggregate?.passedTrialCount ?? 0}/${run.aggregate?.evaluatedTrialCount ?? 0} evaluated trials passed`;
+        process.stdout.write(`${run.suiteName}: ${run.qualityStatus} (${run.executionStatus}; ${measured})\n`);
+    }
+    const failure = evaluationRunFailure(run);
+    if (failure)
+        throw failure;
+}
+function formatScore(score) {
+    if (score === undefined)
+        return 'score unavailable';
+    const outOfHundred = score * 100;
+    return `${outOfHundred.toFixed(outOfHundred === Math.round(outOfHundred) ? 0 : 1)}/100`;
+}

package/dist/types/commands/evaluations.d.ts

@@ -0,0 +1,41 @@
+import { runEvaluationSuite, type EvaluationRun } from '@valerypopoff/rivet2-evaluations';
+import type * as yargs from 'yargs';
+import { type DatasetCliOptions, type ProviderCliOptions } from '../cliRuntime.js';
+export declare class EvaluationCliError extends Error {
+    readonly exitCode: number;
+    constructor(message: string, exitCode: number);
+}
+type EvaluationRunArgs = {
+    project?: string;
+    suite?: string;
+    suiteFile?: string;
+    trials?: number;
+    concurrency?: number;
+    baseline?: string;
+    json?: boolean;
+    junit?: boolean;
+    benchmark?: boolean;
+} & DatasetCliOptions & ProviderCliOptions;
+export declare function makeEvaluationCommand<T>(y: yargs.Argv<T>): yargs.Argv<T & {
+    project: string;
+} & {
+    suite: string | undefined;
+} & {
+    "suite-file": string | undefined;
+} & {
+    trials: number | undefined;
+} & {
+    concurrency: number | undefined;
+} & {
+    baseline: string | undefined;
+} & {
+    benchmark: boolean;
+} & {
+    json: boolean;
+} & {
+    junit: boolean;
+}>;
+export declare function formatEvaluationRunAsJUnit(run: Awaited<ReturnType<typeof runEvaluationSuite>>): string;
+export declare function evaluationRunFailure(run: Pick<EvaluationRun, 'purpose' | 'executionStatus' | 'qualityStatus' | 'qualityReason' | 'aggregate' | 'trials'>): EvaluationCliError | undefined;
+export declare function runEvaluation(args: EvaluationRunArgs): Promise<void>;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@valerypopoff/rivet2-cli",
-  "version": "2.1.9",
+  "version": "2.3.0",
   "license": "MIT",
   "types": "dist/types/cli.d.ts",
   "type": "module",
@@ -23,7 +23,8 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.19.14",
-    "@valerypopoff/rivet2-node": "^2.1.9",
+    "@valerypopoff/rivet2-evaluations": "^2.3.0",
+    "@valerypopoff/rivet2-node": "^2.3.0",
     "chalk": "^5.4.1",
     "didyoumean2": "^7.0.4",
     "dotenv": "^16.4.7",