@axlsdk/axl 0.13.6 → 0.13.8

package/dist/index.d.cts

@@ -1373,10 +1373,13 @@ declare class AxlRuntime extends EventEmitter {
         executeWorkflow?: (input: unknown, runtime?: AxlRuntime) => Promise<{
             output: unknown;
             cost?: number;
+            metadata?: Record<string, unknown>;
         }>;
     } | undefined;
     /** Run a registered eval by name. */
-    runRegisteredEval(name: string): Promise<unknown>;
+    runRegisteredEval(name: string, options?: {
+        metadata?: Record<string, unknown>;
+    }): Promise<unknown>;
     /** Get all execution info (running + completed + historical). */
     getExecutions(): Promise<ExecutionInfo[]>;
     /**
@@ -1451,7 +1454,7 @@ declare class AxlRuntime extends EventEmitter {
      *
      * @see Spec Section 13.6
      */
-    evalCompare(baseline: unknown, candidate: unknown): Promise<unknown>;
+    evalCompare(baseline: unknown, candidate: unknown, options?: unknown): Promise<unknown>;
     /**
      * Track cost across any runtime operations within the given function.
      * Uses AsyncLocalStorage to scope cost attribution to specific execution IDs,
@@ -1463,6 +1466,30 @@ declare class AxlRuntime extends EventEmitter {
         result: T;
         cost: number;
     }>;
+    /**
+     * Track cost and execution metadata across any runtime operations within the given function.
+     * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+     * making it correct with concurrent calls.
+     *
+     * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+     * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+     *
+     * Works with both `createContext()` and `execute()` calls inside `fn`.
+     */
+    trackExecution<T>(fn: () => Promise<T>): Promise<{
+        result: T;
+        cost: number;
+        metadata: {
+            models: string[];
+            modelCallCounts?: Record<string, number>;
+            tokens: {
+                input: number;
+                output: number;
+                reasoning: number;
+            };
+            agentCalls: number;
+        };
+    }>;
     /** Register an execution ID with the active cost scope for trackCost() attribution. */
     private registerWithCostScope;
     /**

package/dist/index.d.ts

@@ -1373,10 +1373,13 @@ declare class AxlRuntime extends EventEmitter {
         executeWorkflow?: (input: unknown, runtime?: AxlRuntime) => Promise<{
             output: unknown;
             cost?: number;
+            metadata?: Record<string, unknown>;
         }>;
     } | undefined;
     /** Run a registered eval by name. */
-    runRegisteredEval(name: string): Promise<unknown>;
+    runRegisteredEval(name: string, options?: {
+        metadata?: Record<string, unknown>;
+    }): Promise<unknown>;
     /** Get all execution info (running + completed + historical). */
     getExecutions(): Promise<ExecutionInfo[]>;
     /**
@@ -1451,7 +1454,7 @@ declare class AxlRuntime extends EventEmitter {
      *
      * @see Spec Section 13.6
      */
-    evalCompare(baseline: unknown, candidate: unknown): Promise<unknown>;
+    evalCompare(baseline: unknown, candidate: unknown, options?: unknown): Promise<unknown>;
     /**
      * Track cost across any runtime operations within the given function.
      * Uses AsyncLocalStorage to scope cost attribution to specific execution IDs,
@@ -1463,6 +1466,30 @@ declare class AxlRuntime extends EventEmitter {
         result: T;
         cost: number;
     }>;
+    /**
+     * Track cost and execution metadata across any runtime operations within the given function.
+     * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+     * making it correct with concurrent calls.
+     *
+     * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+     * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+     *
+     * Works with both `createContext()` and `execute()` calls inside `fn`.
+     */
+    trackExecution<T>(fn: () => Promise<T>): Promise<{
+        result: T;
+        cost: number;
+        metadata: {
+            models: string[];
+            modelCallCounts?: Record<string, number>;
+            tokens: {
+                input: number;
+                output: number;
+                reasoning: number;
+            };
+            agentCalls: number;
+        };
+    }>;
     /** Register an execution ID with the active cost scope for trackCost() attribution. */
     private registerWithCostScope;
     /**

package/dist/index.js

@@ -5420,7 +5420,7 @@ var AxlRuntime = class extends EventEmitter2 {
     return this.registeredEvals.get(name);
   }
   /** Run a registered eval by name. */
-  async runRegisteredEval(name) {
+  async runRegisteredEval(name, options) {
     const entry = this.registeredEvals.get(name);
     if (!entry) throw new Error(`Eval "${name}" is not registered`);
     let result;
@@ -5435,10 +5435,18 @@ var AxlRuntime = class extends EventEmitter2 {
       }
       const originalExecuteFn = entry.executeWorkflow;
       const wrappedExecuteFn = async (input, runtime) => {
-        const { result: result2, cost: trackedCost } = await this.trackCost(async () => {
+        const {
+          result: result2,
+          cost: trackedCost,
+          metadata
+        } = await this.trackExecution(async () => {
           return originalExecuteFn(input, runtime);
         });
-        return { output: result2.output, cost: result2.cost ?? trackedCost };
+        return {
+          output: result2.output,
+          cost: result2.cost ?? trackedCost,
+          metadata: result2.metadata ?? metadata
+        };
       };
       result = await runEvalFn(entry.config, wrappedExecuteFn, this);
     } else {
@@ -5447,6 +5455,12 @@ var AxlRuntime = class extends EventEmitter2 {
       );
     }
     const resultObj = result;
+    if (options?.metadata) {
+      resultObj.metadata = {
+        ...resultObj.metadata,
+        ...options.metadata
+      };
+    }
     try {
       await this.saveEvalResult({
         id: resultObj.id ?? randomUUID2(),
@@ -5982,10 +5996,10 @@ var AxlRuntime = class extends EventEmitter2 {
       );
     }
     const executeWorkflow = async (input) => {
-      const { result, cost } = await this.trackCost(async () => {
+      const { result, cost, metadata } = await this.trackExecution(async () => {
         return this.execute(config.workflow, input);
       });
-      return { output: result, cost };
+      return { output: result, cost, metadata };
     };
     return runEvalFn(config, executeWorkflow, this);
   }
@@ -5995,7 +6009,7 @@ var AxlRuntime = class extends EventEmitter2 {
    *
    * @see Spec Section 13.6
    */
-  async evalCompare(baseline, candidate) {
+  async evalCompare(baseline, candidate, options) {
     let evalCompareFn;
     try {
       ({ evalCompare: evalCompareFn } = await import("@axlsdk/eval"));
@@ -6004,7 +6018,7 @@ var AxlRuntime = class extends EventEmitter2 {
         "axl-eval is required for AxlRuntime.evalCompare(). Install it with: npm install @axlsdk/eval"
       );
     }
-    return evalCompareFn(baseline, candidate);
+    return evalCompareFn(baseline, candidate, options);
   }
   /**
    * Track cost across any runtime operations within the given function.
@@ -6014,24 +6028,58 @@ var AxlRuntime = class extends EventEmitter2 {
    * Works with both `createContext()` and `execute()` calls inside `fn`.
    */
   async trackCost(fn) {
+    const { result, cost } = await this.trackExecution(fn);
+    return { result, cost };
+  }
+  /**
+   * Track cost and execution metadata across any runtime operations within the given function.
+   * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+   * making it correct with concurrent calls.
+   *
+   * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+   * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+   *
+   * Works with both `createContext()` and `execute()` calls inside `fn`.
+   */
+  async trackExecution(fn) {
     const parentScope = costScopeStorage.getStore();
     const scope = {
       totalCost: 0,
       trackedIds: /* @__PURE__ */ new Set(),
       parent: parentScope
     };
-    const costListener = (event) => {
-      if (event.cost && scope.trackedIds.has(event.executionId)) {
-        scope.totalCost += event.cost;
+    const modelCalls = /* @__PURE__ */ new Map();
+    const tokens = { input: 0, output: 0, reasoning: 0 };
+    let agentCalls = 0;
+    const listener = (event) => {
+      if (!scope.trackedIds.has(event.executionId)) return;
+      if (event.cost) scope.totalCost += event.cost;
+      if (event.type === "agent_call") {
+        if (event.model) modelCalls.set(event.model, (modelCalls.get(event.model) ?? 0) + 1);
+        agentCalls++;
+        if (event.tokens) {
+          tokens.input += event.tokens.input ?? 0;
+          tokens.output += event.tokens.output ?? 0;
+          tokens.reasoning += event.tokens.reasoning ?? 0;
+        }
       }
     };
     this.setMaxListeners(this.getMaxListeners() + 1);
-    this.on("trace", costListener);
+    this.on("trace", listener);
     try {
       const result = await costScopeStorage.run(scope, fn);
-      return { result, cost: scope.totalCost };
+      return {
+        result,
+        cost: scope.totalCost,
+        metadata: {
+          models: [...modelCalls.keys()],
+          modelCallCounts: modelCalls.size > 0 ? Object.fromEntries(modelCalls) : void 0,
+          tokens,
+          agentCalls
+        }
+      };
     } finally {
-      this.off("trace", costListener);
+      this.off("trace", listener);
       this.setMaxListeners(this.getMaxListeners() - 1);
     }
   }