@axlsdk/eval 0.23.0 → 0.23.2

package/dist/cli.js

@@ -6,7 +6,7 @@ import {
   rescore,
   runEval,
   scorerCounts
-} from "./chunk-FQHKBSZD.js";
+} from "./chunk-AFUNZE7W.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";

package/dist/index.cjs

@@ -393,6 +393,18 @@ function extractUserMetadata(result, label = "executeWorkflow") {
   }
   return meta;
 }
+function mergeMetadata(tracked, user) {
+  const merged = { ...tracked, ...user };
+  for (const [list, counts] of [
+    ["models", "modelCallCounts"],
+    ["workflows", "workflowCallCounts"]
+  ]) {
+    if (user && Object.hasOwn(user, list) && !Object.hasOwn(user, counts)) {
+      delete merged[counts];
+    }
+  }
+  return merged;
+}
 async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
@@ -476,7 +488,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
         evalItem.output = tracked.result.output;
         evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
         if (tracked.unpriced) evalItem.unpriced = true;
-        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        evalItem.metadata = mergeMetadata(tracked.metadata, extractUserMetadata(tracked.result));
         absorbTiming(tracked.modelTiming, evalItem);
         if (tracked.traces && tracked.traces.length > 0) {
           evalItem.traces = tracked.traces;
@@ -486,6 +498,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
         }
       } else {
         let trackedTiming;
+        let trackedMetadata;
         let result;
         if (typeof runtime.trackExecution === "function") {
           const tracked = await runtime.trackExecution(
@@ -494,6 +507,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
           );
           result = tracked.result;
           trackedTiming = tracked.modelTiming;
+          trackedMetadata = tracked.metadata;
         } else {
           result = await executeWorkflow(item.input, runtime);
         }
@@ -501,7 +515,7 @@ async function runEval(config, executeWorkflow, runtime, options) {
         evalItem.output = result.output;
         evalItem.cost = extractUserCost(result);
         const meta = extractUserMetadata(result);
-        if (meta) evalItem.metadata = meta;
+        if (trackedMetadata || meta) evalItem.metadata = mergeMetadata(trackedMetadata, meta);
         absorbTiming(trackedTiming, evalItem);
         if (evalItem.cost != null) {
           totalCost += evalItem.cost;