@axlsdk/axl 0.13.6 → 0.13.7

package/dist/index.cjs

@@ -5568,7 +5568,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
     return this.registeredEvals.get(name);
   }
   /** Run a registered eval by name. */
-  async runRegisteredEval(name) {
+  async runRegisteredEval(name, options) {
     const entry = this.registeredEvals.get(name);
     if (!entry) throw new Error(`Eval "${name}" is not registered`);
     let result;
@@ -5583,10 +5583,18 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       }
       const originalExecuteFn = entry.executeWorkflow;
       const wrappedExecuteFn = async (input, runtime) => {
-        const { result: result2, cost: trackedCost } = await this.trackCost(async () => {
+        const {
+          result: result2,
+          cost: trackedCost,
+          metadata
+        } = await this.trackExecution(async () => {
           return originalExecuteFn(input, runtime);
         });
-        return { output: result2.output, cost: result2.cost ?? trackedCost };
+        return {
+          output: result2.output,
+          cost: result2.cost ?? trackedCost,
+          metadata: result2.metadata ?? metadata
+        };
       };
       result = await runEvalFn(entry.config, wrappedExecuteFn, this);
     } else {
@@ -5595,6 +5603,12 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       );
     }
     const resultObj = result;
+    if (options?.metadata) {
+      resultObj.metadata = {
+        ...resultObj.metadata,
+        ...options.metadata
+      };
+    }
     try {
       await this.saveEvalResult({
         id: resultObj.id ?? (0, import_node_crypto2.randomUUID)(),
@@ -6130,10 +6144,10 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       );
     }
     const executeWorkflow = async (input) => {
-      const { result, cost } = await this.trackCost(async () => {
+      const { result, cost, metadata } = await this.trackExecution(async () => {
         return this.execute(config.workflow, input);
       });
-      return { output: result, cost };
+      return { output: result, cost, metadata };
     };
     return runEvalFn(config, executeWorkflow, this);
   }
@@ -6143,7 +6157,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
    *
    * @see Spec Section 13.6
    */
-  async evalCompare(baseline, candidate) {
+  async evalCompare(baseline, candidate, options) {
     let evalCompareFn;
     try {
       ({ evalCompare: evalCompareFn } = await import("@axlsdk/eval"));
@@ -6152,7 +6166,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         "axl-eval is required for AxlRuntime.evalCompare(). Install it with: npm install @axlsdk/eval"
       );
     }
-    return evalCompareFn(baseline, candidate);
+    return evalCompareFn(baseline, candidate, options);
   }
   /**
    * Track cost across any runtime operations within the given function.
@@ -6162,24 +6176,58 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
    * Works with both `createContext()` and `execute()` calls inside `fn`.
    */
   async trackCost(fn) {
+    const { result, cost } = await this.trackExecution(fn);
+    return { result, cost };
+  }
+  /**
+   * Track cost and execution metadata across any runtime operations within the given function.
+   * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+   * making it correct with concurrent calls.
+   *
+   * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+   * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+   *
+   * Works with both `createContext()` and `execute()` calls inside `fn`.
+   */
+  async trackExecution(fn) {
     const parentScope = costScopeStorage.getStore();
     const scope = {
       totalCost: 0,
       trackedIds: /* @__PURE__ */ new Set(),
       parent: parentScope
     };
-    const costListener = (event) => {
-      if (event.cost && scope.trackedIds.has(event.executionId)) {
-        scope.totalCost += event.cost;
+    const modelCalls = /* @__PURE__ */ new Map();
+    const tokens = { input: 0, output: 0, reasoning: 0 };
+    let agentCalls = 0;
+    const listener = (event) => {
+      if (!scope.trackedIds.has(event.executionId)) return;
+      if (event.cost) scope.totalCost += event.cost;
+      if (event.type === "agent_call") {
+        if (event.model) modelCalls.set(event.model, (modelCalls.get(event.model) ?? 0) + 1);
+        agentCalls++;
+        if (event.tokens) {
+          tokens.input += event.tokens.input ?? 0;
+          tokens.output += event.tokens.output ?? 0;
+          tokens.reasoning += event.tokens.reasoning ?? 0;
+        }
       }
     };
     this.setMaxListeners(this.getMaxListeners() + 1);
-    this.on("trace", costListener);
+    this.on("trace", listener);
     try {
       const result = await costScopeStorage.run(scope, fn);
-      return { result, cost: scope.totalCost };
+      return {
+        result,
+        cost: scope.totalCost,
+        metadata: {
+          models: [...modelCalls.keys()],
+          modelCallCounts: modelCalls.size > 0 ? Object.fromEntries(modelCalls) : void 0,
+          tokens,
+          agentCalls
+        }
+      };
     } finally {
-      this.off("trace", costListener);
+      this.off("trace", listener);
       this.setMaxListeners(this.getMaxListeners() - 1);
     }
   }