@valerypopoff/rivet2-evaluations 2.8.0 → 2.10.0

package/dist/cjs/bundle.cjs

@@ -46,6 +46,7 @@ __export(src_exports, {
   createEmptyEvaluationProjectData: () => createEmptyEvaluationProjectData,
   createEmptyMetrics: () => createEmptyMetrics,
   createEvaluationBaselineSnapshot: () => createEvaluationBaselineSnapshot,
+  createEvaluationEventCollector: () => createEvaluationEventCollector,
   createEvaluationRunPlan: () => createEvaluationRunPlan,
   createEvaluationRunShell: () => createEvaluationRunShell,
   deserializeEvaluationDatasetJson: () => deserializeEvaluationDatasetJson,
@@ -3464,6 +3465,69 @@ function applyEvaluationLibraryMutation(library, mutation) {
     migratedLegacyProjectIds: library.migratedLegacyProjectIds
   });
 }
+
+// src/executionEventCollector.ts
+function createEvaluationEventCollector(attemptFormat) {
+  const metrics = {
+    durationMs: 0,
+    modelCallCount: 0,
+    toolCallCount: 0,
+    toolFailureCount: 0
+  };
+  const providerAttempts = [];
+  return {
+    metrics,
+    providerAttempts,
+    llmCallFinished(event) {
+      var _a, _b, _c, _d;
+      metrics.modelCallCount = (metrics.modelCallCount ?? 0) + 1;
+      metrics.inputTokens = (metrics.inputTokens ?? 0) + (((_a = event.normalizedUsage) == null ? void 0 : _a.promptTokens) ?? 0);
+      metrics.outputTokens = (metrics.outputTokens ?? 0) + (((_b = event.normalizedUsage) == null ? void 0 : _b.completionTokens) ?? 0);
+      metrics.cachedInputTokens = (metrics.cachedInputTokens ?? 0) + (((_c = event.normalizedUsage) == null ? void 0 : _c.cachedTokens) ?? 0);
+      metrics.reasoningTokens = (metrics.reasoningTokens ?? 0) + (((_d = event.normalizedUsage) == null ? void 0 : _d.reasoningTokens) ?? 0);
+      if (event.pricing.status === "known") {
+        metrics.costUsd = (metrics.costUsd ?? 0) + (event.pricing.costUsd ?? 0);
+      } else {
+        metrics.hasUnknownCost = true;
+      }
+      providerAttempts.push({
+        kind: "provider-call",
+        provider: event.provider,
+        model: event.model,
+        customProviderApi: event.customProviderApi ?? null,
+        outcome: event.outcome,
+        ...attemptFormat === "full" ? { finishReason: event.finishReason ?? null } : {},
+        profileIndex: event.profileIndex ?? null,
+        ...attemptFormat === "full" ? { profileName: event.profileName ?? null } : {},
+        attemptIndex: event.attemptIndex,
+        roundIndex: event.roundIndex ?? null,
+        durationMs: event.durationMs ?? null
+      });
+    },
+    llmProfileAttempt(event) {
+      providerAttempts.push({
+        kind: "profile-decision",
+        provider: event.provider,
+        model: event.model,
+        customProviderApi: event.customProviderApi ?? null,
+        stage: event.stage,
+        outcome: event.outcome,
+        profileIndex: event.profileIndex ?? null,
+        ...attemptFormat === "full" ? { profileName: event.profileName ?? null } : {},
+        attemptIndex: event.attemptIndex ?? null,
+        roundIndex: event.roundIndex,
+        status: event.status ?? null,
+        healthState: event.healthState ?? null,
+        healthDisposition: event.healthDisposition ?? null,
+        timeoutKind: event.timeoutKind ?? null
+      });
+    },
+    toolCallFinished(event) {
+      metrics.toolCallCount = (metrics.toolCallCount ?? 0) + 1;
+      if (event.outcome !== "success") metrics.toolFailureCount = (metrics.toolFailureCount ?? 0) + 1;
+    }
+  };
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   EvaluationGraphExecutionError,
@@ -3482,6 +3546,7 @@ function applyEvaluationLibraryMutation(library, mutation) {
   createEmptyEvaluationProjectData,
   createEmptyMetrics,
   createEvaluationBaselineSnapshot,
+  createEvaluationEventCollector,
   createEvaluationRunPlan,
   createEvaluationRunShell,
   deserializeEvaluationDatasetJson,