langsmith 0.7.0 → 0.7.2

package/dist/experimental/vercel/middleware.cjs

@@ -2,7 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LangSmithMiddleware = LangSmithMiddleware;
 const traceable_js_1 = require("../../traceable.cjs");
-const vercel_js_1 = require("../../utils/vercel.cjs");
 const utils_js_1 = require("./utils.cjs");
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const _formatTracedInputs = (params) => {
@@ -34,81 +33,6 @@ const _formatTracedOutputs = (outputs, includeHttpDetails = false) => {
     }
     return (0, utils_js_1.convertMessageToTracedFormat)(formattedOutputs);
 };
-const setUsageMetadataOnRunTree = (result, runTree) => {
-    if (result.usage == null || typeof result.usage !== "object") {
-        return;
-    }
-    const usage = result.usage;
-    let inputTokens;
-    let outputTokens;
-    let totalTokens;
-    // AI SDK 6: Check for object-based token structures first
-    if (typeof usage.inputTokens === "object" &&
-        usage.inputTokens?.total != null) {
-        // AI SDK 6 detected
-        inputTokens = usage.inputTokens.total;
-        if (typeof usage.outputTokens === "object" &&
-            usage.outputTokens?.total != null) {
-            outputTokens = usage.outputTokens.total;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    else if (typeof usage.inputTokens === "number") {
-        // AI SDK 5 detected
-        inputTokens = usage.inputTokens;
-        if (typeof usage.outputTokens === "number") {
-            outputTokens = usage.outputTokens;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    else {
-        // AI SDK 4 fallback
-        if (typeof usage.promptTokens === "number") {
-            inputTokens = usage.promptTokens;
-        }
-        if (typeof usage.completionTokens === "number") {
-            outputTokens = usage.completionTokens;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    const langsmithUsage = {
-        input_tokens: inputTokens,
-        output_tokens: outputTokens,
-        total_tokens: totalTokens,
-    };
-    const inputTokenDetails = (0, vercel_js_1.extractInputTokenDetails)(result.usage, result.providerMetadata);
-    const outputTokenDetails = (0, vercel_js_1.extractOutputTokenDetails)(result.usage, result.providerMetadata);
-    runTree.extra = {
-        ...runTree.extra,
-        metadata: {
-            ...runTree.extra?.metadata,
-            usage_metadata: {
-                ...langsmithUsage,
-                input_token_details: {
-                    ...inputTokenDetails,
-                },
-                output_token_details: {
-                    ...outputTokenDetails,
-                },
-            },
-        },
-    };
-};
 /**
  * AI SDK middleware that wraps an AI SDK 6 or 5 model and adds LangSmith tracing.
  */
@@ -120,7 +44,7 @@ function LangSmithMiddleware(config) {
                 const result = await doGenerate();
                 const currentRunTree = (0, traceable_js_1.getCurrentRunTree)(true);
                 if (currentRunTree !== undefined) {
-                    setUsageMetadataOnRunTree(result, currentRunTree);
+                    (0, utils_js_1.setUsageMetadataOnRunTree)(result, currentRunTree);
                 }
                 return result;
             }, {
@@ -250,7 +174,7 @@ function LangSmithMiddleware(config) {
                                 }
                                 else if (chunk.type === "finish") {
                                     if (runTree != null) {
-                                        setUsageMetadataOnRunTree(chunk, runTree);
+                                        (0, utils_js_1.setUsageMetadataOnRunTree)(chunk, runTree);
                                     }
                                     return {
                                         ...aggregated,

package/dist/experimental/vercel/middleware.js

@@ -1,6 +1,5 @@
 import { getCurrentRunTree, traceable } from "../../traceable.js";
-import { extractInputTokenDetails, extractOutputTokenDetails, } from "../../utils/vercel.js";
-import { convertMessageToTracedFormat } from "./utils.js";
+import { convertMessageToTracedFormat, setUsageMetadataOnRunTree, } from "./utils.js";
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const _formatTracedInputs = (params) => {
     const { prompt, ...rest } = params;
@@ -31,81 +30,6 @@ const _formatTracedOutputs = (outputs, includeHttpDetails = false) => {
     }
     return convertMessageToTracedFormat(formattedOutputs);
 };
-const setUsageMetadataOnRunTree = (result, runTree) => {
-    if (result.usage == null || typeof result.usage !== "object") {
-        return;
-    }
-    const usage = result.usage;
-    let inputTokens;
-    let outputTokens;
-    let totalTokens;
-    // AI SDK 6: Check for object-based token structures first
-    if (typeof usage.inputTokens === "object" &&
-        usage.inputTokens?.total != null) {
-        // AI SDK 6 detected
-        inputTokens = usage.inputTokens.total;
-        if (typeof usage.outputTokens === "object" &&
-            usage.outputTokens?.total != null) {
-            outputTokens = usage.outputTokens.total;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    else if (typeof usage.inputTokens === "number") {
-        // AI SDK 5 detected
-        inputTokens = usage.inputTokens;
-        if (typeof usage.outputTokens === "number") {
-            outputTokens = usage.outputTokens;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    else {
-        // AI SDK 4 fallback
-        if (typeof usage.promptTokens === "number") {
-            inputTokens = usage.promptTokens;
-        }
-        if (typeof usage.completionTokens === "number") {
-            outputTokens = usage.completionTokens;
-        }
-        totalTokens = result.usage?.totalTokens;
-        if (typeof totalTokens !== "number" &&
-            typeof inputTokens === "number" &&
-            typeof outputTokens === "number") {
-            totalTokens = inputTokens + outputTokens;
-        }
-    }
-    const langsmithUsage = {
-        input_tokens: inputTokens,
-        output_tokens: outputTokens,
-        total_tokens: totalTokens,
-    };
-    const inputTokenDetails = extractInputTokenDetails(result.usage, result.providerMetadata);
-    const outputTokenDetails = extractOutputTokenDetails(result.usage, result.providerMetadata);
-    runTree.extra = {
-        ...runTree.extra,
-        metadata: {
-            ...runTree.extra?.metadata,
-            usage_metadata: {
-                ...langsmithUsage,
-                input_token_details: {
-                    ...inputTokenDetails,
-                },
-                output_token_details: {
-                    ...outputTokenDetails,
-                },
-            },
-        },
-    };
-};
 /**
  * AI SDK middleware that wraps an AI SDK 6 or 5 model and adds LangSmith tracing.
  */

package/dist/experimental/vercel/telemetry.cjs

@@ -0,0 +1,462 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LangSmithTelemetry = LangSmithTelemetry;
+const run_trees_js_1 = require("../../run_trees.cjs");
+const traceable_js_1 = require("../../singletons/traceable.cjs");
+const env_js_1 = require("../../env.cjs");
+const utils_js_1 = require("./utils.cjs");
+const utils_js_2 = require("./utils.cjs");
+const types_js_1 = require("../../utils/types.cjs");
+function _formatMessages(messages) {
+    if (!Array.isArray(messages))
+        return messages;
+    return messages.map((msg) => (0, utils_js_1.convertMessageToTracedFormat)(msg));
+}
+// oxlint-disable-next-line typescript/no-explicit-any
+function _formatToolCalls(toolCalls) {
+    return toolCalls.map((tc) => ({
+        id: tc.toolCallId,
+        type: "function",
+        function: {
+            name: tc.toolName,
+            arguments: (0, types_js_1.isPrimitive)(tc.input)
+                ? String(tc.input)
+                : JSON.stringify(tc.input),
+        },
+    }));
+}
+function _formatStepOutput(
+// oxlint-disable-next-line typescript/no-explicit-any
+event, traceRawHttp) {
+    // Build an assistant-style message from the step result
+    const output = { role: "assistant" };
+    // Text content
+    if (event.content != null) {
+        output.content = event.content;
+    }
+    else if (event.text != null) {
+        output.content = event.text;
+    }
+    // Tool calls
+    if (Array.isArray(event.toolCalls) && event.toolCalls.length > 0) {
+        output.tool_calls = _formatToolCalls(event.toolCalls);
+    }
+    if (event.finishReason != null) {
+        output.finish_reason = event.finishReason;
+    }
+    if (traceRawHttp) {
+        if (event.request != null)
+            output.request = event.request;
+        if (event.response != null)
+            output.response = event.response;
+    }
+    return (0, utils_js_1.convertMessageToTracedFormat)(output);
+}
+function _getLsAgentType(parentRunTree) {
+    if ((0, run_trees_js_1.isRunTree)(parentRunTree) && parentRunTree.run_type === "tool") {
+        return "subagent";
+    }
+    return "root";
+}
+/**
+ * Creates a LangSmith `Telemetry` for the Vercel AI SDK.
+ *
+ * This adapter implements the Vercel AI SDK's `Telemetry` interface
+ * and maps lifecycle events to LangSmith traces. It creates a root span for
+ * the entire generation, child LLM spans for each step, and tool spans for
+ * tool calls.
+ *
+ * ```ts
+ * import { generateText, registerTelemetry } from "ai";
+ * import { LangSmithTelemetry } from "langsmith/experimental/vercel";
+ *
+ * registerTelemetry(LangSmithTelemetry());
+ *
+ * const result = await generateText({
+ *   model: openai("gpt-4o"),
+ *   prompt: "Hello!",
+ * });
+ * ```
+ *
+ * @experimental Only available in Vercel AI SDK 7.
+ */
+function LangSmithTelemetry(config) {
+    const { name: customName, runType = "chain", metadata: customMetadata, tags: customTags, client, projectName, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, traceResponseMetadata, traceRawHttp, tracingEnabled, extra: customExtra, } = config ?? {};
+    function getOpenStepOrRoot(state) {
+        let openStep;
+        state.stepRunTrees.forEach((stepRt) => {
+            if (stepRt.end_time == null) {
+                openStep = stepRt;
+            }
+        });
+        return openStep ?? state.rootRunTree;
+    }
+    async function finalizeOpenToolRuns(state, opts) {
+        const entries = Array.from(state.toolRunTrees.entries());
+        for (let i = 0; i < entries.length; i++) {
+            const [, toolRt] = entries[i];
+            if (toolRt.end_time == null) {
+                if (opts?.error != null) {
+                    await toolRt.end(undefined, opts.error);
+                }
+                else {
+                    await toolRt.end(opts?.note != null ? { note: opts.note } : undefined);
+                }
+                await toolRt.patchRun({ excludeInputs: true });
+            }
+        }
+        state.toolRunTrees.clear();
+    }
+    /** Per-generation state keyed by AI SDK `callId` (stable across nested calls). */
+    const invocationsByCallId = new Map();
+    const onStart = async (event) => {
+        if (!(0, env_js_1.isTracingEnabled)(tracingEnabled))
+            return;
+        if (!("callId" in event) || typeof event.callId !== "string")
+            return;
+        // If called within an existing traceable context, nest under it
+        const parentRunTree = (0, traceable_js_1.getCurrentRunTree)(true);
+        let inputs = {};
+        if (event.recordInputs !== false) {
+            if ("messages" in event && event.messages != null) {
+                inputs.messages = _formatMessages(event.messages);
+            }
+            if ("prompt" in event && event.prompt != null) {
+                inputs.prompt = event.prompt;
+            }
+            if ("instructions" in event && event.instructions != null) {
+                inputs.instructions = event.instructions;
+            }
+            if ("system" in event && event.system != null) {
+                inputs.system = event.system;
+            }
+            if ("tools" in event && event.tools != null) {
+                inputs.tools = Object.keys(event.tools);
+            }
+            if ("runtimeContext" in event && event.runtimeContext != null) {
+                inputs.runtimeContext = event.runtimeContext;
+            }
+            if ("toolsContext" in event && event.toolsContext != null) {
+                inputs.toolsContext = event.toolsContext;
+            }
+            // Apply user-provided input processing
+            if (processInputs) {
+                try {
+                    inputs = processInputs(inputs);
+                }
+                catch (e) {
+                    console.error("Error in processInputs, using raw inputs:", e);
+                }
+            }
+        }
+        const runTreeConfig = {
+            name: customName ?? event.functionId ?? event.provider,
+            run_type: runType,
+            inputs,
+            tracingEnabled: true,
+            extra: {
+                ...customExtra,
+                metadata: {
+                    ...customMetadata,
+                    ai_sdk_method: event.operationId,
+                    ls_agent_type: _getLsAgentType(parentRunTree),
+                    ls_model_name: event.modelId,
+                    ls_provider: event.provider,
+                    ls_integration: "vercel-ai-sdk-telemetry",
+                },
+            },
+            tags: customTags,
+            ...(client ? { client } : {}),
+            ...(projectName ? { project_name: projectName } : {}),
+        };
+        let rootRunTree;
+        if ((0, run_trees_js_1.isRunTree)(parentRunTree)) {
+            rootRunTree = parentRunTree.createChild(runTreeConfig);
+        }
+        else {
+            rootRunTree = new run_trees_js_1.RunTree(runTreeConfig);
+        }
+        await rootRunTree.postRun();
+        invocationsByCallId.set(event.callId, {
+            rootRunTree,
+            stepRunTrees: new Map(),
+            toolRunTrees: new Map(),
+        });
+    };
+    const onStepStart = async (event) => {
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const stepNumber = event.stepNumber ?? 0;
+        let inputs = {};
+        if (event.recordInputs !== false) {
+            if ("messages" in event && event.messages != null) {
+                inputs.messages = _formatMessages(event.messages);
+            }
+            if ("runtimeContext" in event && event.runtimeContext != null) {
+                inputs.runtimeContext = event.runtimeContext;
+            }
+            if ("toolsContext" in event && event.toolsContext != null) {
+                inputs.toolsContext = event.toolsContext;
+            }
+            if (processChildLLMRunInputs) {
+                try {
+                    inputs = processChildLLMRunInputs(inputs);
+                }
+                catch (e) {
+                    console.error("Error in processChildLLMRunInputs, using raw inputs:", e);
+                }
+            }
+        }
+        const stepRunTree = state.rootRunTree.createChild({
+            name: event.provider,
+            run_type: "llm",
+            inputs,
+            extra: { metadata: { step_number: stepNumber } },
+        });
+        state.stepRunTrees.set(stepNumber, stepRunTree);
+        await stepRunTree.postRun();
+    };
+    const onLanguageModelCallStart = async (event) => {
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const stepRunTree = getOpenStepOrRoot(state);
+        if (stepRunTree.run_type !== "llm")
+            return;
+        const prevParams = (0, types_js_1.isRecord)(stepRunTree.extra?.invocation_params)
+            ? stepRunTree.extra.invocation_params
+            : {};
+        const nextParams = { ...event };
+        // Remove properties that are already in the step run tree
+        delete nextParams.messages;
+        delete nextParams.provider;
+        delete nextParams.modelId;
+        // Remove telemetry options (except functionId)
+        delete nextParams.recordInputs;
+        delete nextParams.recordOutputs;
+        delete nextParams.includeToolsContext;
+        // Massage tools for LangSmith to render schema nicely
+        nextParams.tools = nextParams.tools?.map((tool) => {
+            const newTool = { ...tool };
+            if ("inputSchema" in newTool) {
+                newTool.input_schema = newTool.inputSchema;
+                delete newTool.inputSchema;
+            }
+            return newTool;
+        });
+        stepRunTree.extra = {
+            ...stepRunTree.extra,
+            invocation_params: { ...prevParams, ...nextParams },
+        };
+    };
+    const onToolExecutionStart = async (event) => {
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const parentRunTree = getOpenStepOrRoot(state);
+        let inputs = {};
+        if (event.recordInputs !== false) {
+            if ((0, types_js_1.isRecord)(event.toolCall.input)) {
+                inputs = { ...event.toolCall.input };
+            }
+            else if (typeof event.toolCall.input !== "undefined") {
+                inputs = { input: event.toolCall.input };
+            }
+            if ("toolContext" in event && event.toolContext != null) {
+                inputs.toolContext = event.toolContext;
+            }
+        }
+        else {
+            inputs = {};
+        }
+        const toolRunTree = parentRunTree.createChild({
+            name: event.toolCall.toolName,
+            run_type: "tool",
+            inputs,
+            extra: {
+                metadata: {
+                    tool_call_id: event.toolCall.toolCallId,
+                    ai_sdk_call_id: event.callId,
+                },
+            },
+        });
+        await toolRunTree.postRun();
+        state.toolRunTrees.set(event.toolCall.toolCallId, toolRunTree);
+    };
+    const onToolExecutionEnd = async (event) => {
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const toolRunTree = state.toolRunTrees.get(event.toolCall.toolCallId);
+        if (!toolRunTree)
+            return;
+        state.toolRunTrees.delete(event.toolCall.toolCallId);
+        let outputs;
+        let error;
+        if (event.recordOutputs !== false) {
+            if (event.toolOutput.type === "tool-result") {
+                outputs = { output: event.toolOutput.output };
+            }
+            else if (event.toolOutput.type === "tool-error") {
+                const err = event.toolOutput.error;
+                error = err instanceof Error ? err.message : String(err);
+            }
+        }
+        else {
+            outputs = {};
+        }
+        await toolRunTree.end(outputs, error, Math.floor(toolRunTree.start_time + event.toolExecutionMs));
+        await toolRunTree.patchRun({ excludeInputs: true });
+    };
+    const onStepFinish = async (event) => {
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const stepNumber = event.stepNumber ?? 0;
+        const stepRunTree = state.stepRunTrees.get(stepNumber);
+        if (!stepRunTree)
+            return;
+        let outputs = {};
+        if (event.recordOutputs !== false) {
+            outputs = _formatStepOutput(event, traceRawHttp);
+            if (processChildLLMRunOutputs) {
+                try {
+                    outputs = processChildLLMRunOutputs(outputs);
+                }
+                catch (e) {
+                    console.error("Error in processChildLLMRunOutputs, using raw outputs:", e);
+                }
+            }
+        }
+        // Set usage metadata
+        // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+        (0, utils_js_2.setUsageMetadataOnRunTree)(event, stepRunTree);
+        await stepRunTree.end(outputs);
+        await stepRunTree.patchRun({ excludeInputs: true });
+        state.stepRunTrees.delete(stepNumber);
+    };
+    const onEnd = async (event) => {
+        if (!("callId" in event) || typeof event.callId !== "string")
+            return;
+        const state = invocationsByCallId.get(event.callId);
+        if (!state)
+            return;
+        const { rootRunTree } = state;
+        await finalizeOpenToolRuns(state, { note: "closed on finish" });
+        // Ensure any remaining step runs are closed
+        const remainingSteps = Array.from(state.stepRunTrees.entries());
+        for (let i = 0; i < remainingSteps.length; i++) {
+            const [stepNumber, stepRt] = remainingSteps[i];
+            if (stepRt.end_time == null) {
+                await stepRt.end({ note: "closed on finish" });
+                await stepRt.patchRun({ excludeInputs: true });
+            }
+            state.stepRunTrees.delete(stepNumber);
+        }
+        let outputs = {};
+        if (event.recordOutputs !== false) {
+            // Final result output
+            if ("text" in event && event.text != null) {
+                outputs.content = event.text;
+            }
+            else if ("content" in event && event.content != null) {
+                outputs.content = event.content;
+            }
+            if (outputs.content != null) {
+                outputs.role = "assistant";
+            }
+            if ("object" in event && event.object != null) {
+                outputs.object = event.object;
+            }
+            if ("toolCalls" in event &&
+                Array.isArray(event.toolCalls) &&
+                event.toolCalls.length > 0) {
+                outputs.tool_calls = _formatToolCalls(event.toolCalls);
+            }
+            if ("finishReason" in event && event.finishReason != null) {
+                outputs.finish_reason = event.finishReason;
+            }
+            if (traceResponseMetadata &&
+                "steps" in event &&
+                Array.isArray(event.steps)) {
+                outputs.steps = event.steps.map((step, idx) => ({
+                    step_number: idx,
+                    ..._formatStepOutput(step, traceRawHttp),
+                }));
+            }
+            if (processOutputs) {
+                try {
+                    outputs = processOutputs(outputs);
+                }
+                catch (e) {
+                    console.error("Error in processOutputs, using raw outputs:", e);
+                }
+            }
+        }
+        // Set aggregated usage on root
+        if ("totalUsage" in event && event.totalUsage != null) {
+            (0, utils_js_2.setUsageMetadataOnRunTree)(
+            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+            { usage: event.totalUsage, providerMetadata: event.providerMetadata }, rootRunTree);
+        }
+        else if ("usage" in event && event.usage != null) {
+            // @ts-expect-error SharedV4ProviderMetadata is not assignable to SharedV2ProviderMetadata
+            (0, utils_js_2.setUsageMetadataOnRunTree)(event, rootRunTree);
+        }
+        await rootRunTree.end(outputs);
+        await rootRunTree.patchRun({ excludeInputs: true });
+        invocationsByCallId.delete(event.callId);
+    };
+    const onError = async (payload) => {
+        const callId = typeof payload === "object" &&
+            payload !== null &&
+            "callId" in payload &&
+            typeof payload.callId === "string"
+            ? payload.callId
+            : undefined;
+        const error = typeof payload === "object" && payload !== null && "error" in payload
+            ? payload.error
+            : payload;
+        if (callId === undefined)
+            return;
+        const state = invocationsByCallId.get(callId);
+        if (!state)
+            return;
+        const { rootRunTree } = state;
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        await finalizeOpenToolRuns(state, { error: errorMsg });
+        // Close any open step runs with error
+        const errorSteps = Array.from(state.stepRunTrees.entries());
+        for (let i = 0; i < errorSteps.length; i++) {
+            const [stepNumber, stepRt] = errorSteps[i];
+            if (stepRt.end_time == null) {
+                await stepRt.end(undefined, errorMsg);
+                await stepRt.patchRun({ excludeInputs: true });
+            }
+            state.stepRunTrees.delete(stepNumber);
+        }
+        await rootRunTree.end(undefined, errorMsg);
+        await rootRunTree.patchRun({ excludeInputs: true });
+        invocationsByCallId.delete(callId);
+    };
+    const executeTool = async (params) => {
+        const state = invocationsByCallId.get(params.callId);
+        const toolRunTree = state?.toolRunTrees.get(params.toolCallId);
+        if (toolRunTree != null) {
+            return (0, traceable_js_1.withRunTree)(toolRunTree, () => params.execute());
+        }
+        return params.execute();
+    };
+    return {
+        onStart,
+        onStepStart,
+        onLanguageModelCallStart,
+        onToolExecutionStart,
+        onToolExecutionEnd,
+        onStepFinish,
+        onEnd,
+        onError,
+        executeTool,
+    };
+}