@axlsdk/axl 0.13.5 → 0.13.7

package/dist/index.d.cts

@@ -340,6 +340,7 @@ type ExecutionInfo = {
     startedAt: number;
     completedAt?: number;
     duration: number;
+    result?: unknown;
     error?: string;
 };
 type StreamEvent = {
@@ -349,10 +350,12 @@ type StreamEvent = {
     type: 'tool_call';
     name: string;
     args: unknown;
+    callId?: string;
 } | {
     type: 'tool_result';
     name: string;
     result: unknown;
+    callId?: string;
 } | {
     type: 'tool_approval';
     name: string;
@@ -382,7 +385,7 @@ type StreamEvent = {
     data: unknown;
 } | {
     type: 'error';
-    error: Error;
+    message: string;
 };
 /** Record of an agent handoff event (persisted in session metadata). */
 type HandoffRecord = {
@@ -958,6 +961,7 @@ type WorkflowContextInit = {
     onToolCall?: (call: {
         name: string;
         args: unknown;
+        callId?: string;
     }) => void;
     pendingDecisions?: Map<string, (d: HumanDecision) => void>;
     budgetContext?: {
@@ -1369,10 +1373,13 @@ declare class AxlRuntime extends EventEmitter {
         executeWorkflow?: (input: unknown, runtime?: AxlRuntime) => Promise<{
             output: unknown;
             cost?: number;
+            metadata?: Record<string, unknown>;
         }>;
     } | undefined;
     /** Run a registered eval by name. */
-    runRegisteredEval(name: string): Promise<unknown>;
+    runRegisteredEval(name: string, options?: {
+        metadata?: Record<string, unknown>;
+    }): Promise<unknown>;
     /** Get all execution info (running + completed + historical). */
     getExecutions(): Promise<ExecutionInfo[]>;
     /**
@@ -1447,7 +1454,7 @@ declare class AxlRuntime extends EventEmitter {
      *
      * @see Spec Section 13.6
      */
-    evalCompare(baseline: unknown, candidate: unknown): Promise<unknown>;
+    evalCompare(baseline: unknown, candidate: unknown, options?: unknown): Promise<unknown>;
     /**
      * Track cost across any runtime operations within the given function.
      * Uses AsyncLocalStorage to scope cost attribution to specific execution IDs,
@@ -1459,6 +1466,30 @@ declare class AxlRuntime extends EventEmitter {
         result: T;
         cost: number;
     }>;
+    /**
+     * Track cost and execution metadata across any runtime operations within the given function.
+     * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+     * making it correct with concurrent calls.
+     *
+     * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+     * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+     *
+     * Works with both `createContext()` and `execute()` calls inside `fn`.
+     */
+    trackExecution<T>(fn: () => Promise<T>): Promise<{
+        result: T;
+        cost: number;
+        metadata: {
+            models: string[];
+            modelCallCounts?: Record<string, number>;
+            tokens: {
+                input: number;
+                output: number;
+                reasoning: number;
+            };
+            agentCalls: number;
+        };
+    }>;
     /** Register an execution ID with the active cost scope for trackCost() attribution. */
     private registerWithCostScope;
     /**

package/dist/index.d.ts

@@ -340,6 +340,7 @@ type ExecutionInfo = {
     startedAt: number;
     completedAt?: number;
     duration: number;
+    result?: unknown;
     error?: string;
 };
 type StreamEvent = {
@@ -349,10 +350,12 @@ type StreamEvent = {
     type: 'tool_call';
     name: string;
     args: unknown;
+    callId?: string;
 } | {
     type: 'tool_result';
     name: string;
     result: unknown;
+    callId?: string;
 } | {
     type: 'tool_approval';
     name: string;
@@ -382,7 +385,7 @@ type StreamEvent = {
     data: unknown;
 } | {
     type: 'error';
-    error: Error;
+    message: string;
 };
 /** Record of an agent handoff event (persisted in session metadata). */
 type HandoffRecord = {
@@ -958,6 +961,7 @@ type WorkflowContextInit = {
     onToolCall?: (call: {
         name: string;
         args: unknown;
+        callId?: string;
     }) => void;
     pendingDecisions?: Map<string, (d: HumanDecision) => void>;
     budgetContext?: {
@@ -1369,10 +1373,13 @@ declare class AxlRuntime extends EventEmitter {
         executeWorkflow?: (input: unknown, runtime?: AxlRuntime) => Promise<{
             output: unknown;
             cost?: number;
+            metadata?: Record<string, unknown>;
         }>;
     } | undefined;
     /** Run a registered eval by name. */
-    runRegisteredEval(name: string): Promise<unknown>;
+    runRegisteredEval(name: string, options?: {
+        metadata?: Record<string, unknown>;
+    }): Promise<unknown>;
     /** Get all execution info (running + completed + historical). */
     getExecutions(): Promise<ExecutionInfo[]>;
     /**
@@ -1447,7 +1454,7 @@ declare class AxlRuntime extends EventEmitter {
      *
      * @see Spec Section 13.6
      */
-    evalCompare(baseline: unknown, candidate: unknown): Promise<unknown>;
+    evalCompare(baseline: unknown, candidate: unknown, options?: unknown): Promise<unknown>;
     /**
      * Track cost across any runtime operations within the given function.
      * Uses AsyncLocalStorage to scope cost attribution to specific execution IDs,
@@ -1459,6 +1466,30 @@ declare class AxlRuntime extends EventEmitter {
         result: T;
         cost: number;
     }>;
+    /**
+     * Track cost and execution metadata across any runtime operations within the given function.
+     * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+     * making it correct with concurrent calls.
+     *
+     * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+     * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+     *
+     * Works with both `createContext()` and `execute()` calls inside `fn`.
+     */
+    trackExecution<T>(fn: () => Promise<T>): Promise<{
+        result: T;
+        cost: number;
+        metadata: {
+            models: string[];
+            modelCallCounts?: Record<string, number>;
+            tokens: {
+                input: number;
+                output: number;
+                reasoning: number;
+            };
+            agentCalls: number;
+        };
+    }>;
     /** Register an execution ID with the active cost scope for trackCost() attribution. */
     private registerWithCostScope;
     /**

package/dist/index.js

@@ -2691,7 +2691,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
               });
               continue;
             }
-            this.onToolCall?.({ name: toolName, args: toolArgs2 });
+            this.onToolCall?.({ name: toolName, args: toolArgs2, callId: toolCall.id });
             const toolStart2 = Date.now();
             const executeOverride = async () => {
               let toolResult3;
@@ -2724,7 +2724,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
               agent: agent2._name,
               tool: toolName,
               duration: Date.now() - toolStart2,
-              data: { args: toolArgs2, result: toolResult2 }
+              data: { args: toolArgs2, result: toolResult2, callId: toolCall.id }
             });
             currentMessages.push({
               role: "tool",
@@ -2755,7 +2755,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
             });
             continue;
           }
-          this.onToolCall?.({ name: toolName, args: toolArgs });
+          this.onToolCall?.({ name: toolName, args: toolArgs, callId: toolCall.id });
           const toolStart = Date.now();
           if (tool2 && tool2.requireApproval) {
             const approvalFn = async () => {
@@ -2884,7 +2884,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
             agent: agent2._name,
             tool: traceName,
             duration: Date.now() - toolStart,
-            data: { args: toolArgs, result: toolResult }
+            data: { args: toolArgs, result: toolResult, callId: toolCall.id }
           });
           currentMessages.push({
             role: "tool",
@@ -4595,6 +4595,8 @@ var AxlStream = class _AxlStream extends Readable {
     this.bus.on("__reject", (error) => rejectPromise(error));
     this.bus.on("error", () => {
     });
+    this.promise.catch(() => {
+    });
   }
   static STREAM_EVENTS = /* @__PURE__ */ new Set([
     "token",
@@ -4740,9 +4742,16 @@ var AxlStream = class _AxlStream extends Readable {
   _error(error) {
     if (this.finished) return;
     this.finished = true;
-    const errorEvent = { type: "error", error };
+    const errorEvent = { type: "error", message: error.message };
     this.bus.emit("error", errorEvent);
+    this.push(errorEvent);
     this.push(null);
+    const waiter = this.waiters.shift();
+    if (waiter) {
+      waiter({ value: errorEvent, done: false });
+    } else {
+      this.eventQueue.push(errorEvent);
+    }
     for (const w of this.waiters) {
       w({ value: void 0, done: true });
     }
@@ -5411,7 +5420,7 @@ var AxlRuntime = class extends EventEmitter2 {
     return this.registeredEvals.get(name);
   }
   /** Run a registered eval by name. */
-  async runRegisteredEval(name) {
+  async runRegisteredEval(name, options) {
     const entry = this.registeredEvals.get(name);
     if (!entry) throw new Error(`Eval "${name}" is not registered`);
     let result;
@@ -5426,10 +5435,18 @@ var AxlRuntime = class extends EventEmitter2 {
       }
       const originalExecuteFn = entry.executeWorkflow;
       const wrappedExecuteFn = async (input, runtime) => {
-        const { result: result2, cost: trackedCost } = await this.trackCost(async () => {
+        const {
+          result: result2,
+          cost: trackedCost,
+          metadata
+        } = await this.trackExecution(async () => {
           return originalExecuteFn(input, runtime);
         });
-        return { output: result2.output, cost: result2.cost ?? trackedCost };
+        return {
+          output: result2.output,
+          cost: result2.cost ?? trackedCost,
+          metadata: result2.metadata ?? metadata
+        };
       };
       result = await runEvalFn(entry.config, wrappedExecuteFn, this);
     } else {
@@ -5438,6 +5455,12 @@ var AxlRuntime = class extends EventEmitter2 {
       );
     }
     const resultObj = result;
+    if (options?.metadata) {
+      resultObj.metadata = {
+        ...resultObj.metadata,
+        ...options.metadata
+      };
+    }
     try {
       await this.saveEvalResult({
         id: resultObj.id ?? randomUUID2(),
@@ -5594,6 +5617,7 @@ var AxlRuntime = class extends EventEmitter2 {
           execInfo.status = "completed";
           execInfo.completedAt = Date.now();
           execInfo.duration = execInfo.completedAt - execInfo.startedAt;
+          execInfo.result = output;
           ctx.log("workflow_end", {
             workflow: name,
             status: "completed",
@@ -5706,7 +5730,8 @@ var AxlRuntime = class extends EventEmitter2 {
             axlStream._push({
               type: "tool_result",
               name: event.tool ?? "",
-              result: event.data?.result
+              result: event.data?.result,
+              callId: event.data?.callId
             });
           }
           axlStream._push({ type: "step", step: event.step, data: event });
@@ -5715,7 +5740,12 @@ var AxlRuntime = class extends EventEmitter2 {
           axlStream._push({ type: "token", data: token });
         },
         onToolCall: (call) => {
-          axlStream._push({ type: "tool_call", name: call.name, args: call.args });
+          axlStream._push({
+            type: "tool_call",
+            name: call.name,
+            args: call.args,
+            callId: call.callId
+          });
         },
         onAgentStart: (info) => {
           axlStream._push({ type: "agent_start", agent: info.agent, model: info.model });
@@ -5749,6 +5779,7 @@ var AxlRuntime = class extends EventEmitter2 {
             execInfo.status = "completed";
             execInfo.completedAt = Date.now();
             execInfo.duration = execInfo.completedAt - execInfo.startedAt;
+            execInfo.result = result;
             wfCtx.log("workflow_end", {
               workflow: name,
               status: "completed",
@@ -5965,10 +5996,10 @@ var AxlRuntime = class extends EventEmitter2 {
       );
     }
     const executeWorkflow = async (input) => {
-      const { result, cost } = await this.trackCost(async () => {
+      const { result, cost, metadata } = await this.trackExecution(async () => {
         return this.execute(config.workflow, input);
       });
-      return { output: result, cost };
+      return { output: result, cost, metadata };
     };
     return runEvalFn(config, executeWorkflow, this);
   }
@@ -5978,7 +6009,7 @@ var AxlRuntime = class extends EventEmitter2 {
    *
    * @see Spec Section 13.6
    */
-  async evalCompare(baseline, candidate) {
+  async evalCompare(baseline, candidate, options) {
     let evalCompareFn;
     try {
       ({ evalCompare: evalCompareFn } = await import("@axlsdk/eval"));
@@ -5987,7 +6018,7 @@ var AxlRuntime = class extends EventEmitter2 {
         "axl-eval is required for AxlRuntime.evalCompare(). Install it with: npm install @axlsdk/eval"
       );
     }
-    return evalCompareFn(baseline, candidate);
+    return evalCompareFn(baseline, candidate, options);
   }
   /**
    * Track cost across any runtime operations within the given function.
@@ -5997,24 +6028,58 @@ var AxlRuntime = class extends EventEmitter2 {
    * Works with both `createContext()` and `execute()` calls inside `fn`.
    */
   async trackCost(fn) {
+    const { result, cost } = await this.trackExecution(fn);
+    return { result, cost };
+  }
+  /**
+   * Track cost and execution metadata across any runtime operations within the given function.
+   * Uses AsyncLocalStorage to scope attribution to specific execution IDs,
+   * making it correct with concurrent calls.
+   *
+   * Returns cost (same as `trackCost`) plus metadata extracted from trace events:
+   * models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
+   *
+   * Works with both `createContext()` and `execute()` calls inside `fn`.
+   */
+  async trackExecution(fn) {
     const parentScope = costScopeStorage.getStore();
     const scope = {
       totalCost: 0,
       trackedIds: /* @__PURE__ */ new Set(),
       parent: parentScope
     };
-    const costListener = (event) => {
-      if (event.cost && scope.trackedIds.has(event.executionId)) {
-        scope.totalCost += event.cost;
+    const modelCalls = /* @__PURE__ */ new Map();
+    const tokens = { input: 0, output: 0, reasoning: 0 };
+    let agentCalls = 0;
+    const listener = (event) => {
+      if (!scope.trackedIds.has(event.executionId)) return;
+      if (event.cost) scope.totalCost += event.cost;
+      if (event.type === "agent_call") {
+        if (event.model) modelCalls.set(event.model, (modelCalls.get(event.model) ?? 0) + 1);
+        agentCalls++;
+        if (event.tokens) {
+          tokens.input += event.tokens.input ?? 0;
+          tokens.output += event.tokens.output ?? 0;
+          tokens.reasoning += event.tokens.reasoning ?? 0;
+        }
       }
     };
     this.setMaxListeners(this.getMaxListeners() + 1);
-    this.on("trace", costListener);
+    this.on("trace", listener);
     try {
       const result = await costScopeStorage.run(scope, fn);
-      return { result, cost: scope.totalCost };
+      return {
+        result,
+        cost: scope.totalCost,
+        metadata: {
+          models: [...modelCalls.keys()],
+          modelCallCounts: modelCalls.size > 0 ? Object.fromEntries(modelCalls) : void 0,
+          tokens,
+          agentCalls
+        }
+      };
     } finally {
-      this.off("trace", costListener);
+      this.off("trace", listener);
       this.setMaxListeners(this.getMaxListeners() - 1);
     }
   }