@valerypopoff/rivet2-cli 2.8.0 → 2.9.0

package/bin/commands/evaluations.js

@@ -1,5 +1,5 @@
 import { createProcessor, loadProjectAndAttachedDataFromFile, } from '@valerypopoff/rivet2-node';
-import { deserializeEvaluationProjectData, deserializeEvaluationSuiteBundleJson, EvaluationGraphExecutionError, runEvaluationSuite, validateEvaluationDataset, } from '@valerypopoff/rivet2-evaluations';
+import { deserializeEvaluationProjectData, deserializeEvaluationSuiteBundleJson, createEvaluationEventCollector, EvaluationGraphExecutionError, runEvaluationSuite, validateEvaluationDataset, } from '@valerypopoff/rivet2-evaluations';
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { addDatasetOptions, addProviderOptions, createDatasetProvider, getProjectFile, resolveDatasetFilePath, withCliProcessorOptions, } from '../cliRuntime.js';
@@ -214,7 +214,8 @@ export async function runEvaluation(args) {
     if (args.baseline !== undefined && importedBundle) {
         throw new EvaluationCliError('Exported suite bundles do not include baselines; omit --baseline or use a legacy project attachment.', 3);
     }
-    const dataset = importedBundle?.dataset ?? (await loadEvaluationDataset(projectPath, project.metadata.id, suite.datasetId, args.datasetFile));
+    const dataset = importedBundle?.dataset ??
+        (await loadEvaluationDataset(projectPath, project.metadata.id, suite.datasetId, args.datasetFile));
     const datasetProvider = await createDatasetProvider(projectPath, args);
     const effectiveData = args.trials === undefined && args.concurrency === undefined
         ? evaluationData
@@ -248,77 +249,30 @@ export async function runEvaluation(args) {
         executionMode: 'node-cli',
         runGraph: async ({ graphId, inputs, signal, metadata }) => {
             const startedAt = Date.now();
-            const metrics = {
-                durationMs: 0,
-                modelCallCount: 0,
-                toolCallCount: 0,
-                toolFailureCount: 0,
-            };
-            const providerAttempts = [];
+            const captured = createEvaluationEventCollector('cli');
             const processorInfo = createProcessor(project, withCliProcessorOptions({ datasetProvider, projectPath }, args, {
                 graph: graphId,
                 inputs: graphInputsToOutputs(project, graphId, inputs),
                 abortSignal: signal,
                 evaluation: metadata,
             }));
-            processorInfo.processor.on('llmCallFinished', (event) => {
-                metrics.modelCallCount = (metrics.modelCallCount ?? 0) + 1;
-                metrics.inputTokens = (metrics.inputTokens ?? 0) + (event.normalizedUsage?.promptTokens ?? 0);
-                metrics.outputTokens = (metrics.outputTokens ?? 0) + (event.normalizedUsage?.completionTokens ?? 0);
-                metrics.cachedInputTokens = (metrics.cachedInputTokens ?? 0) + (event.normalizedUsage?.cachedTokens ?? 0);
-                metrics.reasoningTokens = (metrics.reasoningTokens ?? 0) + (event.normalizedUsage?.reasoningTokens ?? 0);
-                if (event.pricing.status === 'known')
-                    metrics.costUsd = (metrics.costUsd ?? 0) + (event.pricing.costUsd ?? 0);
-                else
-                    metrics.hasUnknownCost = true;
-                providerAttempts.push({
-                    kind: 'provider-call',
-                    provider: event.provider,
-                    model: event.model,
-                    customProviderApi: event.customProviderApi ?? null,
-                    outcome: event.outcome,
-                    profileIndex: event.profileIndex ?? null,
-                    attemptIndex: event.attemptIndex,
-                    roundIndex: event.roundIndex ?? null,
-                    durationMs: event.durationMs ?? null,
-                });
-            });
-            processorInfo.processor.on('llmProfileAttempt', (event) => {
-                providerAttempts.push({
-                    kind: 'profile-decision',
-                    provider: event.provider,
-                    model: event.model,
-                    customProviderApi: event.customProviderApi ?? null,
-                    stage: event.stage,
-                    outcome: event.outcome,
-                    profileIndex: event.profileIndex ?? null,
-                    attemptIndex: event.attemptIndex ?? null,
-                    roundIndex: event.roundIndex,
-                    status: event.status ?? null,
-                    healthState: event.healthState ?? null,
-                    healthDisposition: event.healthDisposition ?? null,
-                    timeoutKind: event.timeoutKind ?? null,
-                });
-            });
-            processorInfo.processor.on('toolCallFinished', (event) => {
-                metrics.toolCallCount = (metrics.toolCallCount ?? 0) + 1;
-                if (event.outcome !== 'success')
-                    metrics.toolFailureCount = (metrics.toolFailureCount ?? 0) + 1;
-            });
+            processorInfo.processor.on('llmCallFinished', captured.llmCallFinished);
+            processorInfo.processor.on('llmProfileAttempt', captured.llmProfileAttempt);
+            processorInfo.processor.on('toolCallFinished', captured.toolCallFinished);
             try {
                 const outputs = await processorInfo.run();
-                metrics.durationMs = Date.now() - startedAt;
+                captured.metrics.durationMs = Date.now() - startedAt;
                 return {
                     outputs: Object.fromEntries(Object.entries(outputs).map(([key, value]) => [key, value.value])),
-                    metrics,
-                    ...(providerAttempts.length === 0 ? {} : { providerAttempts }),
+                    metrics: captured.metrics,
+                    ...(captured.providerAttempts.length === 0 ? {} : { providerAttempts: captured.providerAttempts }),
                 };
             }
             catch (error) {
-                metrics.durationMs = Math.max(metrics.durationMs, Date.now() - startedAt);
+                captured.metrics.durationMs = Math.max(captured.metrics.durationMs, Date.now() - startedAt);
                 throw new EvaluationGraphExecutionError(error instanceof Error ? error.message : String(error), {
-                    metrics,
-                    ...(providerAttempts.length === 0 ? {} : { providerAttempts }),
+                    metrics: captured.metrics,
+                    ...(captured.providerAttempts.length === 0 ? {} : { providerAttempts: captured.providerAttempts }),
                 });
             }
             finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@valerypopoff/rivet2-cli",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "license": "MIT",
   "types": "dist/types/cli.d.ts",
   "type": "module",
@@ -23,8 +23,8 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.19.14",
-    "@valerypopoff/rivet2-evaluations": "^2.8.0",
-    "@valerypopoff/rivet2-node": "^2.8.0",
+    "@valerypopoff/rivet2-evaluations": "^2.9.0",
+    "@valerypopoff/rivet2-node": "^2.9.0",
     "chalk": "^5.4.1",
     "didyoumean2": "^7.0.4",
     "dotenv": "^16.4.7",