@valerypopoff/rivet2-evaluations 2.7.0 → 2.9.0

package/dist/esm/executionEventCollector.js

@@ -0,0 +1,62 @@
+export function createEvaluationEventCollector(attemptFormat) {
+    const metrics = {
+        durationMs: 0,
+        modelCallCount: 0,
+        toolCallCount: 0,
+        toolFailureCount: 0,
+    };
+    const providerAttempts = [];
+    return {
+        metrics,
+        providerAttempts,
+        llmCallFinished(event) {
+            metrics.modelCallCount = (metrics.modelCallCount ?? 0) + 1;
+            metrics.inputTokens = (metrics.inputTokens ?? 0) + (event.normalizedUsage?.promptTokens ?? 0);
+            metrics.outputTokens = (metrics.outputTokens ?? 0) + (event.normalizedUsage?.completionTokens ?? 0);
+            metrics.cachedInputTokens = (metrics.cachedInputTokens ?? 0) + (event.normalizedUsage?.cachedTokens ?? 0);
+            metrics.reasoningTokens = (metrics.reasoningTokens ?? 0) + (event.normalizedUsage?.reasoningTokens ?? 0);
+            if (event.pricing.status === 'known') {
+                metrics.costUsd = (metrics.costUsd ?? 0) + (event.pricing.costUsd ?? 0);
+            }
+            else {
+                metrics.hasUnknownCost = true;
+            }
+            providerAttempts.push({
+                kind: 'provider-call',
+                provider: event.provider,
+                model: event.model,
+                customProviderApi: event.customProviderApi ?? null,
+                outcome: event.outcome,
+                ...(attemptFormat === 'full' ? { finishReason: event.finishReason ?? null } : {}),
+                profileIndex: event.profileIndex ?? null,
+                ...(attemptFormat === 'full' ? { profileName: event.profileName ?? null } : {}),
+                attemptIndex: event.attemptIndex,
+                roundIndex: event.roundIndex ?? null,
+                durationMs: event.durationMs ?? null,
+            });
+        },
+        llmProfileAttempt(event) {
+            providerAttempts.push({
+                kind: 'profile-decision',
+                provider: event.provider,
+                model: event.model,
+                customProviderApi: event.customProviderApi ?? null,
+                stage: event.stage,
+                outcome: event.outcome,
+                profileIndex: event.profileIndex ?? null,
+                ...(attemptFormat === 'full' ? { profileName: event.profileName ?? null } : {}),
+                attemptIndex: event.attemptIndex ?? null,
+                roundIndex: event.roundIndex,
+                status: event.status ?? null,
+                healthState: event.healthState ?? null,
+                healthDisposition: event.healthDisposition ?? null,
+                timeoutKind: event.timeoutKind ?? null,
+            });
+        },
+        toolCallFinished(event) {
+            metrics.toolCallCount = (metrics.toolCallCount ?? 0) + 1;
+            if (event.outcome !== 'success')
+                metrics.toolFailureCount = (metrics.toolFailureCount ?? 0) + 1;
+        },
+    };
+}

package/dist/esm/index.js

@@ -10,3 +10,4 @@ export * from './dataTypes.js';
 export * from './normalization.js';
 export * from './library.js';
 export * from './librarySync.js';
+export * from './executionEventCollector.js';

package/dist/types/executionEventCollector.d.ts

@@ -0,0 +1,21 @@
+import type { ChatV2CallTraceEvent, LLMProfileAttemptTraceEvent, ToolCallFinishedEvent } from '@valerypopoff/rivet2-core';
+import type { EvaluationExecutionMetrics, PortableJson } from './types.js';
+/**
+ * The evidence shape used by evaluation runners. The CLI intentionally keeps
+ * its long-standing compact evidence contract, while interactive and hosted
+ * runners retain the additional display-oriented provider fields.
+ */
+export type EvaluationProviderAttemptFormat = 'full' | 'cli';
+/**
+ * Per-graph physical provider and tool accounting for an evaluation run.
+ * Duration belongs to the owning runner because it also includes graph work
+ * that does not emit a provider or tool event.
+ */
+export type EvaluationEventCollector = {
+    readonly metrics: EvaluationExecutionMetrics;
+    readonly providerAttempts: PortableJson[];
+    llmCallFinished(event: ChatV2CallTraceEvent): void;
+    llmProfileAttempt(event: LLMProfileAttemptTraceEvent): void;
+    toolCallFinished(event: ToolCallFinishedEvent): void;
+};
+export declare function createEvaluationEventCollector(attemptFormat: EvaluationProviderAttemptFormat): EvaluationEventCollector;

package/dist/types/index.d.ts

@@ -10,3 +10,4 @@ export * from './dataTypes.js';
 export * from './normalization.js';
 export * from './library.js';
 export * from './librarySync.js';
+export * from './executionEventCollector.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@valerypopoff/rivet2-evaluations",
-  "version": "2.7.0",
+  "version": "2.9.0",
   "license": "MIT",
   "main": "dist/cjs/bundle.cjs",
   "module": "dist/esm/index.js",
@@ -36,7 +36,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@valerypopoff/rivet2-core": "^2.7.0",
+    "@valerypopoff/rivet2-core": "^2.9.0",
     "yaml": "^2.9.0"
   }
 }