langsmith 0.4.8 → 0.4.10

package/dist/experimental/anthropic/messages.cjs

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.flattenContentBlocks = flattenContentBlocks;
+exports.convertFromAnthropicMessage = convertFromAnthropicMessage;
+exports.isTaskTool = isTaskTool;
+exports.isToolBlock = isToolBlock;
+const utils_js_1 = require("./utils.cjs");
+/**
+ * Converts SDK content blocks into serializable objects.
+ * Matches Python's flatten_content_blocks behavior.
+ */
+function flattenContentBlocks(content) {
+    if (!Array.isArray(content)) {
+        return content;
+    }
+    return content.map((block) => {
+        if (!block || typeof block !== "object" || !("type" in block)) {
+            return block;
+        }
+        const blockType = block.type;
+        switch (blockType) {
+            case "text":
+                return { type: "text", text: block.text || "" };
+            case "thinking":
+                return {
+                    type: "thinking",
+                    thinking: block.thinking || "",
+                    signature: block.signature || "",
+                };
+            case "tool_use":
+                return {
+                    type: "tool_use",
+                    id: block.id,
+                    name: block.name,
+                    input: block.input,
+                };
+            case "tool_result":
+                return {
+                    type: "tool_result",
+                    tool_use_id: block.tool_use_id,
+                    content: block.content,
+                    is_error: block.is_error || false,
+                };
+            default:
+                return block;
+        }
+    });
+}
+/**
+ * Utility function to convert an Anthropic message to a LangSmith-compatible message.
+ * @internal
+ */
+function convertFromAnthropicMessage(sdkMessage) {
+    if (sdkMessage == null)
+        return [];
+    if (typeof sdkMessage === "string") {
+        return [{ content: sdkMessage, role: "user" }];
+    }
+    if ((0, utils_js_1.isIterable)(sdkMessage)) {
+        return Array.from(sdkMessage).flatMap(convertFromAnthropicMessage);
+    }
+    if (typeof sdkMessage !== "object" ||
+        sdkMessage == null ||
+        !("message" in sdkMessage) ||
+        (sdkMessage.type !== "assistant" && sdkMessage.type !== "user")) {
+        return [];
+    }
+    const { role = sdkMessage.type, content, ...rest } = sdkMessage.message;
+    const flattened = flattenContentBlocks(content);
+    const toolResultBlocks = role === "user" && Array.isArray(flattened)
+        ? flattened.filter(isToolResultBlock)
+        : [];
+    if (toolResultBlocks.length > 0) {
+        return toolResultBlocks.map((block) => ({ ...block, role: "tool" }));
+    }
+    return [{ ...rest, content: flattened, role }];
+}
+function isToolResultBlock(block) {
+    if (typeof block !== "object" || block == null)
+        return false;
+    if (!("type" in block))
+        return false;
+    return block.type === "tool_result";
+}
+/**
+ * Type assertion to check if a tool is a Task tool
+ * @param tool - The tool to check
+ * @returns True if the tool is a Task tool, false otherwise
+ * @internal
+ */
+function isTaskTool(tool) {
+    return tool.type === "tool_use" && tool.name === "Task";
+}
+/**
+ * Type-assertion to check for tool blocks
+ * @internal
+ */
+function isToolBlock(block) {
+    if (!block || typeof block !== "object")
+        return false;
+    return block.type === "tool_use";
+}

package/dist/experimental/anthropic/messages.d.ts

@@ -0,0 +1,6 @@
+import type { BetaContentBlock } from "./types.js";
+/**
+ * Converts SDK content blocks into serializable objects.
+ * Matches Python's flatten_content_blocks behavior.
+ */
+export declare function flattenContentBlocks(content: BetaContentBlock[] | unknown): Array<Record<string, unknown>> | unknown;

package/dist/experimental/anthropic/messages.js

@@ -0,0 +1,96 @@
+import { isIterable } from "./utils.js";
+/**
+ * Converts SDK content blocks into serializable objects.
+ * Matches Python's flatten_content_blocks behavior.
+ */
+export function flattenContentBlocks(content) {
+    if (!Array.isArray(content)) {
+        return content;
+    }
+    return content.map((block) => {
+        if (!block || typeof block !== "object" || !("type" in block)) {
+            return block;
+        }
+        const blockType = block.type;
+        switch (blockType) {
+            case "text":
+                return { type: "text", text: block.text || "" };
+            case "thinking":
+                return {
+                    type: "thinking",
+                    thinking: block.thinking || "",
+                    signature: block.signature || "",
+                };
+            case "tool_use":
+                return {
+                    type: "tool_use",
+                    id: block.id,
+                    name: block.name,
+                    input: block.input,
+                };
+            case "tool_result":
+                return {
+                    type: "tool_result",
+                    tool_use_id: block.tool_use_id,
+                    content: block.content,
+                    is_error: block.is_error || false,
+                };
+            default:
+                return block;
+        }
+    });
+}
+/**
+ * Utility function to convert an Anthropic message to a LangSmith-compatible message.
+ * @internal
+ */
+export function convertFromAnthropicMessage(sdkMessage) {
+    if (sdkMessage == null)
+        return [];
+    if (typeof sdkMessage === "string") {
+        return [{ content: sdkMessage, role: "user" }];
+    }
+    if (isIterable(sdkMessage)) {
+        return Array.from(sdkMessage).flatMap(convertFromAnthropicMessage);
+    }
+    if (typeof sdkMessage !== "object" ||
+        sdkMessage == null ||
+        !("message" in sdkMessage) ||
+        (sdkMessage.type !== "assistant" && sdkMessage.type !== "user")) {
+        return [];
+    }
+    const { role = sdkMessage.type, content, ...rest } = sdkMessage.message;
+    const flattened = flattenContentBlocks(content);
+    const toolResultBlocks = role === "user" && Array.isArray(flattened)
+        ? flattened.filter(isToolResultBlock)
+        : [];
+    if (toolResultBlocks.length > 0) {
+        return toolResultBlocks.map((block) => ({ ...block, role: "tool" }));
+    }
+    return [{ ...rest, content: flattened, role }];
+}
+function isToolResultBlock(block) {
+    if (typeof block !== "object" || block == null)
+        return false;
+    if (!("type" in block))
+        return false;
+    return block.type === "tool_result";
+}
+/**
+ * Type assertion to check if a tool is a Task tool
+ * @param tool - The tool to check
+ * @returns True if the tool is a Task tool, false otherwise
+ * @internal
+ */
+export function isTaskTool(tool) {
+    return tool.type === "tool_use" && tool.name === "Task";
+}
+/**
+ * Type-assertion to check for tool blocks
+ * @internal
+ */
+export function isToolBlock(block) {
+    if (!block || typeof block !== "object")
+        return false;
+    return block.type === "tool_use";
+}

package/dist/experimental/anthropic/types.cjs

@@ -1,2 +1,3 @@
 "use strict";
+/* eslint-disable @typescript-eslint/no-explicit-any */
 Object.defineProperty(exports, "__esModule", { value: true });

package/dist/experimental/anthropic/types.d.ts

@@ -1,37 +1,50 @@
-import type { RunTree, RunTreeConfig } from "../../run_trees.js";
-export type AgentSDKContext = {
-    /**
-     * Storage for active tool runs, keyed by tool_use_id.
-     * Used to correlate PreToolUse and PostToolUse hooks.
-     */
-    activeToolRuns: Map<string, {
-        run: RunTree;
-        startTime: number;
-    }>;
-    /**
-     * Storage for client-managed runs (subagent sessions and their child tools).
-     * These are created when processing AssistantMessage content blocks and
-     * closed when PostToolUse hook fires. Keyed by tool_use_id.
-     */
-    clientManagedRuns: Map<string, RunTree>;
-    /**
-     * Storage for subagent sessions, keyed by the Task tool's tool_use_id.
-     * Used to parent LLM turns and tools to the correct subagent.
-     */
-    subagentSessions: Map<string, RunTree>;
-    /**
-     * Tracks the currently active subagent context (tool_use_id).
-     * Set when a Task tool is called, cleared when the tool result returns.
-     * Assistant messages that arrive while a subagent is active belong to that subagent.
-     */
-    activeSubagentToolUseId: string | undefined;
-    /**
-     * Reference to the current parent run tree for tool tracing.
-     * Set when a traced query starts, cleared when it ends.
-     */
-    currentParentRun: RunTree | undefined;
-};
-/**
- * Configuration options for wrapping Claude Agent SDK with LangSmith tracing.
- */
-export type WrapClaudeAgentSDKConfig = Partial<Omit<RunTreeConfig, "inputs" | "outputs" | "run_type" | "child_runs" | "parent_run" | "error">>;
+export type SDKAssistantMessage = {
+    type: "assistant";
+    message: {
+        id: string;
+        role?: string;
+        content: Record<string, any>[];
+        usage?: Record<string, any>;
+        model?: string;
+    };
+    parent_tool_use_id: string | null;
+};
+export type SDKSystemMessage = {
+    type: "system";
+};
+export type SDKUserMessage = {
+    type: "user";
+    message: {
+        role?: string;
+        content: Record<string, any> | Record<string, any>[] | string;
+        usage?: Record<string, any>;
+        model?: string;
+    };
+    session_id: string;
+    tool_use_result?: unknown;
+    parent_tool_use_id: string | null;
+};
+export type SDKResultMessage = {
+    type: "result";
+    modelUsage: ModelUsage;
+    total_cost_usd: number | null;
+    is_error: boolean | null;
+    num_turns: number | null;
+    session_id: string | null;
+    duration_ms: number | null;
+    duration_api_ms: number | null;
+    usage: Record<string, any>;
+};
+export type SDKMessage = SDKAssistantMessage | SDKUserMessage | SDKSystemMessage | SDKResultMessage;
+export type ModelUsage = {
+    [key: string]: any;
+};
+export type QueryOptions = {
+    [key: string]: any;
+};
+export type BetaContentBlock = {
+    [key: string]: any;
+};
+export type BetaToolUseBlock = {
+    [key: string]: any;
+};

package/dist/experimental/anthropic/types.js

@@ -1 +1,2 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 export {};

package/dist/experimental/anthropic/usage.cjs

@@ -0,0 +1,180 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.aggregateUsageFromModelUsage = aggregateUsageFromModelUsage;
+exports.extractUsageFromMessage = extractUsageFromMessage;
+exports.correctUsageFromResults = correctUsageFromResults;
+const usage_js_1 = require("../../utils/usage.cjs");
+const utils_js_1 = require("./utils.cjs");
+/**
+ * Aggregates usage from modelUsage breakdown (includes all models, including hidden ones).
+ * This provides accurate totals when multiple models are used.
+ * @internal
+ */
+function aggregateUsageFromModelUsage(modelUsage) {
+    const metrics = {};
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
+    let totalCacheReadTokens = 0;
+    let totalCacheCreationTokens = 0;
+    // Aggregate across all models
+    for (const modelStats of Object.values(modelUsage)) {
+        totalInputTokens += modelStats.inputTokens || 0;
+        totalOutputTokens += modelStats.outputTokens || 0;
+        totalCacheReadTokens += modelStats.cacheReadInputTokens || 0;
+        totalCacheCreationTokens += modelStats.cacheCreationInputTokens || 0;
+    }
+    // Build input_token_details if we have cache tokens
+    if (totalCacheReadTokens > 0 || totalCacheCreationTokens > 0) {
+        metrics.input_token_details = {
+            cache_read: totalCacheReadTokens,
+            cache_creation: totalCacheCreationTokens,
+        };
+    }
+    // Sum all input tokens (new + cache read + cache creation)
+    const totalPromptTokens = totalInputTokens + totalCacheReadTokens + totalCacheCreationTokens;
+    metrics.input_tokens = totalPromptTokens;
+    metrics.output_tokens = totalOutputTokens;
+    metrics.total_tokens = totalPromptTokens + totalOutputTokens;
+    return metrics;
+}
+/**
+ * Extracts and normalizes usage metrics from a Claude Agent SDK message.
+ * @internal
+ */
+function extractUsageFromMessage(message) {
+    const metrics = {};
+    // Assistant messages contain usage in message.message.usage
+    // Result messages contain usage in message.usage
+    let usage;
+    if (message.type === "assistant") {
+        usage = message.message?.usage;
+    }
+    else if (message.type === "result") {
+        usage = message.usage;
+    }
+    if (!usage || typeof usage !== "object") {
+        return metrics;
+    }
+    // Standard token counts - use LangSmith's expected field names
+    const inputTokens = (0, utils_js_1.getNumberProperty)(usage, "input_tokens") || 0;
+    const outputTokens = (0, utils_js_1.getNumberProperty)(usage, "output_tokens") || 0;
+    // Get cache tokens
+    const cacheRead = (0, utils_js_1.getNumberProperty)(usage, "cache_read_input_tokens") || 0;
+    const cacheCreation = (0, utils_js_1.getNumberProperty)(usage, "cache_creation_input_tokens") || 0;
+    // Build input_token_details if we have cache tokens
+    if (cacheRead > 0 || cacheCreation > 0) {
+        const inputTokenDetails = (0, usage_js_1.convertAnthropicUsageToInputTokenDetails)(usage);
+        if (Object.keys(inputTokenDetails).length > 0) {
+            metrics.input_token_details = inputTokenDetails;
+        }
+    }
+    // Sum cache tokens into input_tokens total (matching Python's sum_anthropic_tokens)
+    const totalInputTokens = inputTokens + cacheRead + cacheCreation;
+    metrics.input_tokens = totalInputTokens;
+    metrics.output_tokens = outputTokens;
+    metrics.total_tokens = totalInputTokens + outputTokens;
+    return metrics;
+}
+/**
+ * Corrects usage metrics for assistant runs based on the results of the runs.
+ * @internal
+ */
+function correctUsageFromResults(resultUsages, assistantRuns) {
+    const runByModel = assistantRuns.reduce((acc, run) => {
+        const modelId = run.extra?.metadata?.ls_model_name;
+        if (!modelId)
+            return acc;
+        acc[modelId] ??= [];
+        acc[modelId].push(run);
+        return acc;
+    }, {});
+    const runUsageByModel = assistantRuns.reduce((acc, run) => {
+        const modelId = run.extra?.metadata?.ls_model_name;
+        if (!modelId)
+            return acc;
+        const usageMetadata = { ...run.extra?.metadata?.usage_metadata };
+        usageMetadata.input_tokens ??= 0;
+        usageMetadata.output_tokens ??= 0;
+        usageMetadata.total_tokens ??= 0;
+        usageMetadata.input_token_details = {
+            ...usageMetadata.input_token_details,
+        };
+        usageMetadata.input_token_details.cache_read ??= 0;
+        usageMetadata.input_token_details.ephemeral_5m_input_tokens ??= 0;
+        usageMetadata.input_token_details.ephemeral_1h_input_tokens ??= 0;
+        acc[modelId] ??= {
+            input_tokens: 0,
+            output_tokens: 0,
+            total_tokens: 0,
+            input_token_details: {
+                cache_read: 0,
+                cache_creation: 0,
+            },
+        };
+        acc[modelId].input_tokens += usageMetadata.input_tokens;
+        acc[modelId].output_tokens += usageMetadata.output_tokens;
+        acc[modelId].total_tokens += usageMetadata.total_tokens;
+        acc[modelId].input_token_details.cache_read +=
+            usageMetadata.input_token_details.cache_read;
+        acc[modelId].input_token_details.cache_creation +=
+            usageMetadata.input_token_details.ephemeral_5m_input_tokens;
+        acc[modelId].input_token_details.cache_creation +=
+            usageMetadata.input_token_details.ephemeral_1h_input_tokens;
+        return acc;
+    }, {});
+    const resultUsageMap = Object.fromEntries(Object.entries(resultUsages).map(([modelId, usage]) => [
+        modelId,
+        {
+            input_tokens: usage.inputTokens +
+                usage.cacheReadInputTokens +
+                usage.cacheCreationInputTokens,
+            output_tokens: usage.outputTokens,
+            total_tokens: usage.inputTokens +
+                usage.cacheReadInputTokens +
+                usage.cacheCreationInputTokens +
+                usage.outputTokens,
+            input_token_details: {
+                cache_read: usage.cacheReadInputTokens,
+                cache_creation: usage.cacheCreationInputTokens,
+            },
+        },
+    ]));
+    for (const modelId in resultUsageMap) {
+        const lastRun = runByModel[modelId]?.at(-1);
+        const runsUsage = runUsageByModel[modelId];
+        const resultUsage = resultUsageMap[modelId];
+        if (!runsUsage || !lastRun)
+            continue;
+        const difference = {
+            input_tokens: Math.max(0, resultUsage.input_tokens - runsUsage.input_tokens),
+            output_tokens: Math.max(0, resultUsage.output_tokens - runsUsage.output_tokens),
+            total_tokens: Math.max(0, resultUsage.total_tokens - runsUsage.total_tokens),
+            cache_read: Math.max(0, resultUsage.input_token_details.cache_read -
+                runsUsage.input_token_details.cache_read),
+            cache_creation: Math.max(0, resultUsage.input_token_details.cache_creation -
+                runsUsage.input_token_details.cache_creation),
+        };
+        if (Object.values(difference).some((value) => value > 0)) {
+            // apply difference to the last run
+            lastRun.extra ??= {};
+            lastRun.extra.metadata ??= {};
+            lastRun.extra.metadata.usage_metadata ??= {};
+            lastRun.extra.metadata.usage_metadata.input_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_tokens +=
+                difference.input_tokens;
+            lastRun.extra.metadata.usage_metadata.output_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.output_tokens +=
+                difference.output_tokens;
+            lastRun.extra.metadata.usage_metadata.total_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.total_tokens +=
+                difference.total_tokens;
+            lastRun.extra.metadata.usage_metadata.input_token_details ??= {};
+            lastRun.extra.metadata.usage_metadata.input_token_details.cache_read ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_token_details.cache_read +=
+                difference.cache_read;
+            lastRun.extra.metadata.usage_metadata.input_token_details.ephemeral_5m_input_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_token_details.ephemeral_5m_input_tokens +=
+                difference.cache_creation;
+        }
+    }
+}

package/dist/experimental/anthropic/usage.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/experimental/anthropic/usage.js

@@ -0,0 +1,175 @@
+import { convertAnthropicUsageToInputTokenDetails } from "../../utils/usage.js";
+import { getNumberProperty } from "./utils.js";
+/**
+ * Aggregates usage from modelUsage breakdown (includes all models, including hidden ones).
+ * This provides accurate totals when multiple models are used.
+ * @internal
+ */
+export function aggregateUsageFromModelUsage(modelUsage) {
+    const metrics = {};
+    let totalInputTokens = 0;
+    let totalOutputTokens = 0;
+    let totalCacheReadTokens = 0;
+    let totalCacheCreationTokens = 0;
+    // Aggregate across all models
+    for (const modelStats of Object.values(modelUsage)) {
+        totalInputTokens += modelStats.inputTokens || 0;
+        totalOutputTokens += modelStats.outputTokens || 0;
+        totalCacheReadTokens += modelStats.cacheReadInputTokens || 0;
+        totalCacheCreationTokens += modelStats.cacheCreationInputTokens || 0;
+    }
+    // Build input_token_details if we have cache tokens
+    if (totalCacheReadTokens > 0 || totalCacheCreationTokens > 0) {
+        metrics.input_token_details = {
+            cache_read: totalCacheReadTokens,
+            cache_creation: totalCacheCreationTokens,
+        };
+    }
+    // Sum all input tokens (new + cache read + cache creation)
+    const totalPromptTokens = totalInputTokens + totalCacheReadTokens + totalCacheCreationTokens;
+    metrics.input_tokens = totalPromptTokens;
+    metrics.output_tokens = totalOutputTokens;
+    metrics.total_tokens = totalPromptTokens + totalOutputTokens;
+    return metrics;
+}
+/**
+ * Extracts and normalizes usage metrics from a Claude Agent SDK message.
+ * @internal
+ */
+export function extractUsageFromMessage(message) {
+    const metrics = {};
+    // Assistant messages contain usage in message.message.usage
+    // Result messages contain usage in message.usage
+    let usage;
+    if (message.type === "assistant") {
+        usage = message.message?.usage;
+    }
+    else if (message.type === "result") {
+        usage = message.usage;
+    }
+    if (!usage || typeof usage !== "object") {
+        return metrics;
+    }
+    // Standard token counts - use LangSmith's expected field names
+    const inputTokens = getNumberProperty(usage, "input_tokens") || 0;
+    const outputTokens = getNumberProperty(usage, "output_tokens") || 0;
+    // Get cache tokens
+    const cacheRead = getNumberProperty(usage, "cache_read_input_tokens") || 0;
+    const cacheCreation = getNumberProperty(usage, "cache_creation_input_tokens") || 0;
+    // Build input_token_details if we have cache tokens
+    if (cacheRead > 0 || cacheCreation > 0) {
+        const inputTokenDetails = convertAnthropicUsageToInputTokenDetails(usage);
+        if (Object.keys(inputTokenDetails).length > 0) {
+            metrics.input_token_details = inputTokenDetails;
+        }
+    }
+    // Sum cache tokens into input_tokens total (matching Python's sum_anthropic_tokens)
+    const totalInputTokens = inputTokens + cacheRead + cacheCreation;
+    metrics.input_tokens = totalInputTokens;
+    metrics.output_tokens = outputTokens;
+    metrics.total_tokens = totalInputTokens + outputTokens;
+    return metrics;
+}
+/**
+ * Corrects usage metrics for assistant runs based on the results of the runs.
+ * @internal
+ */
+export function correctUsageFromResults(resultUsages, assistantRuns) {
+    const runByModel = assistantRuns.reduce((acc, run) => {
+        const modelId = run.extra?.metadata?.ls_model_name;
+        if (!modelId)
+            return acc;
+        acc[modelId] ??= [];
+        acc[modelId].push(run);
+        return acc;
+    }, {});
+    const runUsageByModel = assistantRuns.reduce((acc, run) => {
+        const modelId = run.extra?.metadata?.ls_model_name;
+        if (!modelId)
+            return acc;
+        const usageMetadata = { ...run.extra?.metadata?.usage_metadata };
+        usageMetadata.input_tokens ??= 0;
+        usageMetadata.output_tokens ??= 0;
+        usageMetadata.total_tokens ??= 0;
+        usageMetadata.input_token_details = {
+            ...usageMetadata.input_token_details,
+        };
+        usageMetadata.input_token_details.cache_read ??= 0;
+        usageMetadata.input_token_details.ephemeral_5m_input_tokens ??= 0;
+        usageMetadata.input_token_details.ephemeral_1h_input_tokens ??= 0;
+        acc[modelId] ??= {
+            input_tokens: 0,
+            output_tokens: 0,
+            total_tokens: 0,
+            input_token_details: {
+                cache_read: 0,
+                cache_creation: 0,
+            },
+        };
+        acc[modelId].input_tokens += usageMetadata.input_tokens;
+        acc[modelId].output_tokens += usageMetadata.output_tokens;
+        acc[modelId].total_tokens += usageMetadata.total_tokens;
+        acc[modelId].input_token_details.cache_read +=
+            usageMetadata.input_token_details.cache_read;
+        acc[modelId].input_token_details.cache_creation +=
+            usageMetadata.input_token_details.ephemeral_5m_input_tokens;
+        acc[modelId].input_token_details.cache_creation +=
+            usageMetadata.input_token_details.ephemeral_1h_input_tokens;
+        return acc;
+    }, {});
+    const resultUsageMap = Object.fromEntries(Object.entries(resultUsages).map(([modelId, usage]) => [
+        modelId,
+        {
+            input_tokens: usage.inputTokens +
+                usage.cacheReadInputTokens +
+                usage.cacheCreationInputTokens,
+            output_tokens: usage.outputTokens,
+            total_tokens: usage.inputTokens +
+                usage.cacheReadInputTokens +
+                usage.cacheCreationInputTokens +
+                usage.outputTokens,
+            input_token_details: {
+                cache_read: usage.cacheReadInputTokens,
+                cache_creation: usage.cacheCreationInputTokens,
+            },
+        },
+    ]));
+    for (const modelId in resultUsageMap) {
+        const lastRun = runByModel[modelId]?.at(-1);
+        const runsUsage = runUsageByModel[modelId];
+        const resultUsage = resultUsageMap[modelId];
+        if (!runsUsage || !lastRun)
+            continue;
+        const difference = {
+            input_tokens: Math.max(0, resultUsage.input_tokens - runsUsage.input_tokens),
+            output_tokens: Math.max(0, resultUsage.output_tokens - runsUsage.output_tokens),
+            total_tokens: Math.max(0, resultUsage.total_tokens - runsUsage.total_tokens),
+            cache_read: Math.max(0, resultUsage.input_token_details.cache_read -
+                runsUsage.input_token_details.cache_read),
+            cache_creation: Math.max(0, resultUsage.input_token_details.cache_creation -
+                runsUsage.input_token_details.cache_creation),
+        };
+        if (Object.values(difference).some((value) => value > 0)) {
+            // apply difference to the last run
+            lastRun.extra ??= {};
+            lastRun.extra.metadata ??= {};
+            lastRun.extra.metadata.usage_metadata ??= {};
+            lastRun.extra.metadata.usage_metadata.input_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_tokens +=
+                difference.input_tokens;
+            lastRun.extra.metadata.usage_metadata.output_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.output_tokens +=
+                difference.output_tokens;
+            lastRun.extra.metadata.usage_metadata.total_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.total_tokens +=
+                difference.total_tokens;
+            lastRun.extra.metadata.usage_metadata.input_token_details ??= {};
+            lastRun.extra.metadata.usage_metadata.input_token_details.cache_read ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_token_details.cache_read +=
+                difference.cache_read;
+            lastRun.extra.metadata.usage_metadata.input_token_details.ephemeral_5m_input_tokens ??= 0;
+            lastRun.extra.metadata.usage_metadata.input_token_details.ephemeral_5m_input_tokens +=
+                difference.cache_creation;
+        }
+    }
+}