langsmith 0.3.62-rc.2 → 0.3.63

package/dist/evaluation/evaluate_comparative.cjs

@@ -45,6 +45,7 @@ async function loadTraces(client, experiment, options) {
     }
     return results;
 }
+/** @deprecated Use `evaluate` and pass two experiments as targets. */
 async function evaluateComparative(experiments, options) {
     if (experiments.length < 2) {
         throw new Error("Comparative evaluation requires at least 2 experiments.");

package/dist/evaluation/evaluate_comparative.d.ts

@@ -57,5 +57,6 @@ export interface ComparisonEvaluationResults {
     experimentName: string;
     results: ComparisonEvaluationResultRow[];
 }
+/** @deprecated Use `evaluate` and pass two experiments as targets. */
 export declare function evaluateComparative(experiments: Array<string> | Array<Promise<ExperimentResults> | ExperimentResults>, options: EvaluateComparativeOptions): Promise<ComparisonEvaluationResults>;
 export {};

package/dist/evaluation/evaluate_comparative.js

@@ -39,6 +39,7 @@ async function loadTraces(client, experiment, options) {
     }
     return results;
 }
+/** @deprecated Use `evaluate` and pass two experiments as targets. */
 export async function evaluateComparative(experiments, options) {
     if (experiments.length < 2) {
         throw new Error("Comparative evaluation requires at least 2 experiments.");

package/dist/experimental/vercel/index.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.wrapAISDK = void 0;
+exports.wrapAISDK = exports.createLangSmithProviderOptions = void 0;
 const middleware_js_1 = require("./middleware.cjs");
 const utils_js_1 = require("./utils.cjs");
 const traceable_js_1 = require("../../traceable.cjs");
@@ -58,6 +58,70 @@ const _formatTracedInputs = (params) => {
         return { ...rest, prompt, messages };
     }
 };
+const _mergeConfig = (baseConfig, runtimeConfig) => {
+    return {
+        ...baseConfig,
+        ...runtimeConfig,
+        metadata: {
+            ...baseConfig?.metadata,
+            ...runtimeConfig?.metadata,
+        },
+    };
+};
+const _extractChildRunConfig = (lsConfig) => {
+    const { id, name, parent_run_id, start_time, end_time, attachments, dotted_order, processInputs, processOutputs, processChildLLMRunInputs, processChildLLMRunOutputs, ...inheritedConfig } = lsConfig ?? {};
+    const childConfig = inheritedConfig;
+    if (processChildLLMRunInputs) {
+        childConfig.processInputs = processChildLLMRunInputs;
+    }
+    if (processChildLLMRunOutputs) {
+        childConfig.processOutputs = processChildLLMRunOutputs;
+    }
+    return childConfig;
+};
+const _resolveConfigs = (baseLsConfig, runtimeLsConfig) => {
+    const baseChildRunConfig = _extractChildRunConfig(baseLsConfig);
+    const runtimeChildLLMRunConfig = _extractChildRunConfig(runtimeLsConfig);
+    const resolvedLsConfig = _mergeConfig(baseLsConfig, runtimeLsConfig);
+    const resolvedChildLLMRunConfig = _mergeConfig(baseChildRunConfig, runtimeChildLLMRunConfig);
+    const { processInputs: _processInputs, processOutputs: _processOutputs, ...resolvedToolConfig } = resolvedChildLLMRunConfig;
+    return {
+        resolvedLsConfig,
+        resolvedChildLLMRunConfig,
+        resolvedToolConfig,
+    };
+};
+/**
+ * Wraps LangSmith config in a way that matches AI SDK provider types.
+ *
+ * ```ts
+ * import { createLangSmithProviderOptions } from "langsmith/experimental/vercel";
+ * import * as ai from "ai";
+ *
+ * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+ *   // Will have appropriate typing
+ *   processInputs: (inputs) => {
+ *     const { messages } = inputs;
+ *     return {
+ *       messages: messages?.map((message) => ({
+ *         ...message,
+ *         content: "REDACTED",
+ *       })),
+ *       prompt: "REDACTED",
+ *     };
+ *   },
+ * });
+ * ```
+ *
+ * Note: AI SDK expects only JSON values in an object for
+ * provider options, but LangSmith's config may contain non-JSON values.
+ * These are not passed to the underlying AI SDK model, so it is safe to
+ * cast the typing here.
+ */
+const createLangSmithProviderOptions = (lsConfig) => {
+    return (lsConfig ?? {});
+};
+exports.createLangSmithProviderOptions = createLangSmithProviderOptions;
 /**
  * Wraps Vercel AI SDK 5 functions with LangSmith tracing capabilities.
  *
@@ -74,8 +138,7 @@ const _formatTracedInputs = (params) => {
  * @returns returns.streamText - Wrapped streamText function that traces calls to LangSmith
  * @returns returns.streamObject - Wrapped streamObject function that traces calls to LangSmith
  */
-const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject, generateObject, }, lsConfig) => {
-    const { id, name, parent_run_id, start_time, end_time, attachments, dotted_order, ...inheritedConfig } = lsConfig ?? {};
+const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject, generateObject, }, baseLsConfig) => {
     /**
      * Wrapped version of AI SDK 5's generateText with LangSmith tracing.
      *
@@ -96,6 +159,8 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
      */
     const wrappedGenerateText = async (...args) => {
         const params = args[0];
+        const { langsmith: runtimeLsConfig, ...providerOptions } = params.providerOptions ?? {};
+        const { resolvedLsConfig, resolvedChildLLMRunConfig, resolvedToolConfig } = _resolveConfigs(baseLsConfig, runtimeLsConfig);
         const traceableFunc = (0, traceable_js_1.traceable)(async (...args) => {
             const [params, ...rest] = args;
             const wrappedModel = wrapLanguageModel({
@@ -103,23 +168,30 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 middleware: (0, middleware_js_1.LangSmithMiddleware)({
                     name: _getModelDisplayName(params.model),
                     modelId: _getModelId(params.model),
-                    lsConfig: inheritedConfig,
+                    lsConfig: resolvedChildLLMRunConfig,
                 }),
             });
             return generateText({
                 ...params,
-                tools: _wrapTools(params.tools, inheritedConfig),
+                providerOptions,
+                tools: _wrapTools(params.tools, resolvedToolConfig),
                 model: wrappedModel,
             }, ...rest);
         }, {
             name: _getModelDisplayName(params.model),
-            ...lsConfig,
+            ...resolvedLsConfig,
             metadata: {
                 ai_sdk_method: "ai.generateText",
-                ...lsConfig?.metadata,
+                ...resolvedLsConfig?.metadata,
+            },
+            processInputs: (inputs) => {
+                const inputFormatter = resolvedLsConfig?.processInputs ?? _formatTracedInputs;
+                return inputFormatter(inputs);
             },
-            processInputs: (inputs) => _formatTracedInputs(inputs),
             processOutputs: (outputs) => {
+                if (resolvedLsConfig?.processOutputs) {
+                    return resolvedLsConfig.processOutputs(outputs);
+                }
                 if (outputs.outputs == null || typeof outputs.outputs !== "object") {
                     return outputs;
                 }
@@ -162,6 +234,8 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
      */
     const wrappedGenerateObject = async (...args) => {
         const params = args[0];
+        const { langsmith: runtimeLsConfig, ...providerOptions } = params.providerOptions ?? {};
+        const { resolvedLsConfig, resolvedChildLLMRunConfig } = _resolveConfigs(baseLsConfig, runtimeLsConfig);
         const traceableFunc = (0, traceable_js_1.traceable)(async (...args) => {
             const [params, ...rest] = args;
             const wrappedModel = wrapLanguageModel({
@@ -169,22 +243,29 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 middleware: (0, middleware_js_1.LangSmithMiddleware)({
                     name: _getModelDisplayName(params.model),
                     modelId: _getModelId(params.model),
-                    lsConfig: inheritedConfig,
+                    lsConfig: resolvedChildLLMRunConfig,
                 }),
             });
             return generateObject({
                 ...params,
+                providerOptions,
                 model: wrappedModel,
             }, ...rest);
         }, {
             name: _getModelDisplayName(params.model),
-            ...lsConfig,
+            ...resolvedLsConfig,
             metadata: {
                 ai_sdk_method: "ai.generateObject",
-                ...lsConfig?.metadata,
+                ...resolvedLsConfig?.metadata,
+            },
+            processInputs: (inputs) => {
+                const inputFormatter = resolvedLsConfig?.processInputs ?? _formatTracedInputs;
+                return inputFormatter(inputs);
             },
-            processInputs: (inputs) => _formatTracedInputs(inputs),
             processOutputs: (outputs) => {
+                if (resolvedLsConfig?.processOutputs) {
+                    return resolvedLsConfig.processOutputs(outputs);
+                }
                 if (outputs.outputs == null || typeof outputs.outputs !== "object") {
                     return outputs;
                 }
@@ -214,6 +295,8 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
      */
     const wrappedStreamText = (...args) => {
         const params = args[0];
+        const { langsmith: runtimeLsConfig, ...providerOptions } = params.providerOptions ?? {};
+        const { resolvedLsConfig, resolvedChildLLMRunConfig, resolvedToolConfig } = _resolveConfigs(baseLsConfig, runtimeLsConfig);
         const traceableFunc = (0, traceable_js_1.traceable)((...args) => {
             const [params, ...rest] = args;
             const wrappedModel = wrapLanguageModel({
@@ -221,23 +304,30 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 middleware: (0, middleware_js_1.LangSmithMiddleware)({
                     name: _getModelDisplayName(params.model),
                     modelId: _getModelId(params.model),
-                    lsConfig: inheritedConfig,
+                    lsConfig: resolvedChildLLMRunConfig,
                 }),
             });
             return streamText({
                 ...params,
-                tools: _wrapTools(params.tools, inheritedConfig),
+                providerOptions,
+                tools: _wrapTools(params.tools, resolvedToolConfig),
                 model: wrappedModel,
             }, ...rest);
         }, {
             name: _getModelDisplayName(params.model),
-            ...lsConfig,
+            ...resolvedLsConfig,
             metadata: {
                 ai_sdk_method: "ai.streamText",
-                ...lsConfig?.metadata,
+                ...resolvedLsConfig?.metadata,
+            },
+            processInputs: (inputs) => {
+                const inputFormatter = resolvedLsConfig?.processInputs ?? _formatTracedInputs;
+                return inputFormatter(inputs);
             },
-            processInputs: (inputs) => _formatTracedInputs(inputs),
             processOutputs: async (outputs) => {
+                if (resolvedLsConfig?.processOutputs) {
+                    return resolvedLsConfig.processOutputs(outputs);
+                }
                 if (outputs.outputs == null || typeof outputs.outputs !== "object") {
                     return outputs;
                 }
@@ -274,6 +364,8 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
      */
     const wrappedStreamObject = (...args) => {
         const params = args[0];
+        const { langsmith: runtimeLsConfig, ...providerOptions } = params.providerOptions ?? {};
+        const { resolvedLsConfig, resolvedChildLLMRunConfig } = _resolveConfigs(baseLsConfig, runtimeLsConfig);
         const traceableFunc = (0, traceable_js_1.traceable)((...args) => {
             const [params, ...rest] = args;
             const wrappedModel = wrapLanguageModel({
@@ -281,22 +373,29 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 middleware: (0, middleware_js_1.LangSmithMiddleware)({
                     name: _getModelDisplayName(params.model),
                     modelId: _getModelId(params.model),
-                    lsConfig: inheritedConfig,
+                    lsConfig: resolvedChildLLMRunConfig,
                 }),
             });
             return streamObject({
                 ...params,
+                providerOptions,
                 model: wrappedModel,
             }, ...rest);
         }, {
             name: _getModelDisplayName(params.model),
-            ...lsConfig,
+            ...resolvedLsConfig,
             metadata: {
                 ai_sdk_method: "ai.streamObject",
-                ...lsConfig?.metadata,
+                ...resolvedLsConfig?.metadata,
+            },
+            processInputs: (inputs) => {
+                const inputFormatter = resolvedLsConfig?.processInputs ?? _formatTracedInputs;
+                return inputFormatter(inputs);
             },
-            processInputs: (inputs) => _formatTracedInputs(inputs),
             processOutputs: async (outputs) => {
+                if (resolvedLsConfig?.processOutputs) {
+                    return resolvedLsConfig.processOutputs(outputs);
+                }
                 if (outputs.outputs == null || typeof outputs.outputs !== "object") {
                     return outputs;
                 }

package/dist/experimental/vercel/index.d.ts

@@ -1,4 +1,214 @@
+import type { JSONValue } from "ai";
+import type { LanguageModelV2, LanguageModelV2CallOptions } from "@ai-sdk/provider";
+import { type AggregatedDoStreamOutput } from "./middleware.js";
 import { RunTreeConfig } from "../../run_trees.js";
+export type { AggregatedDoStreamOutput };
+export type WrapAISDKConfig<T extends (...args: any[]) => any = (...args: any[]) => any> = Partial<Omit<RunTreeConfig, "inputs" | "outputs" | "run_type" | "child_runs" | "parent_run" | "error" | "serialized">> & {
+    /**
+     * Apply transformations to AI SDK inputs before logging.
+     * This function should NOT mutate the inputs.
+     * Receives both "raw" and LangSmith-suggested "formatted" inputs,
+     * and should combine them into a single LangSmith-formatted input.
+     *
+     * ```ts
+     * import {
+     *   wrapAISDK,
+     *   createLangSmithProviderOptions,
+     * } from "langsmith/experimental/vercel";
+     * import * as ai from "ai";
+     * import { openai } from "@ai-sdk/openai";
+     *
+     * const { generateText } = wrapAISDK(ai);
+     *
+     * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+     *   processInputs: (inputs) => {
+     *     const { messages } = inputs;
+     *     return {
+     *       messages: messages?.map((message) => ({
+     *         providerMetadata: message.providerOptions,
+     *         role: "assistant",
+     *         content: "REDACTED",
+     *       })),
+     *       prompt: "REDACTED",
+     *     };
+     *   },
+     * });
+     * const { text } = await generateText({
+     *   model: openai("gpt-5-nano"),
+     *   prompt: "What is the capital of France?",
+     *   providerOptions: {
+     *     langsmith: lsConfig,
+     *   },
+     * });
+     * ```
+     *
+     * This function is not inherited by nested LLM runs or tool calls.
+     * Pass `processChildLLMRunInputs` to override child LLM run
+     * input processing or wrap your tool's `execute` method in a
+     * separate `traceable` for tool calls.
+     *
+     * @param inputs Key-value map of the function inputs.
+     * @param inputs.formatted - Inputs formatted for LangSmith.
+     * @param inputs.raw - Raw inputs from the AI SDK.
+     * @returns A single combined key-value map of processed inputs.
+     */
+    processInputs?: (inputs: Parameters<T>[0]) => Record<string, unknown>;
+    /**
+     * Apply transformations to AI SDK outputs before logging.
+     * This function should NOT mutate the outputs.
+     * Receives both "raw" and LangSmith-suggested "formatted" outputs,
+     * and should combine them into a single LangSmith-formatted output.
+     *
+     * ```ts
+     * import {
+     *   wrapAISDK,
+     *   createLangSmithProviderOptions,
+     * } from "langsmith/experimental/vercel";
+     * import * as ai from "ai";
+     * import { openai } from "@ai-sdk/openai";
+     *
+     * const { generateText } = wrapAISDK(ai);
+     *
+     * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+     *   processOutputs: (outputs) => {
+     *     return {
+     *       providerMetadata: outputs.providerMetadata,
+     *       role: "assistant",
+     *       content: "REDACTED",
+     *     };
+     *   },
+     * });
+     * const { text } = await generateText({
+     *   model: openai("gpt-5-nano"),
+     *   prompt: "What is the capital of France?",
+     *   providerOptions: {
+     *     langsmith: lsConfig,
+     *   },
+     * });
+     * ```
+     *
+     * This function is not inherited by nested LLM runs or tool calls.
+     * Pass `processChildLLMRunOutputs` to override child LLM run
+     * output processing or wrap your tool's `execute` method in a
+     * separate `traceable` for tool calls.
+     *
+     * @param outputs Key-value map of the function inputs.
+     * @param outputs.formatted - Outputs formatted for LangSmith.
+     * @param outputs.raw - Raw outputs from the AI SDK.
+     * @returns A single combined key-value map of processed outputs.
+     */
+    processOutputs?: (outputs: Awaited<ReturnType<T>>) => Record<string, unknown>;
+    /**
+     * Apply transformations to AI SDK child LLM run inputs before logging.
+     * This function should NOT mutate the inputs.
+     * Receives both "raw" and LangSmith-suggested "formatted" inputs,
+     * and should combine them into a single LangSmith-formatted input.
+     *
+     * ```ts
+     * import {
+     *   wrapAISDK,
+     *   createLangSmithProviderOptions,
+     * } from "langsmith/experimental/vercel";
+     * import * as ai from "ai";
+     * import { openai } from "@ai-sdk/openai";
+     *
+     * const { generateText } = wrapAISDK(ai);
+     *
+     * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+     *   processChildLLMRunInputs: (inputs) => {
+     *     const { prompt } = inputs;
+     *     return {
+     *       messages: prompt.map((message) => ({
+     *         ...message,
+     *         content: "REDACTED CHILD INPUTS",
+     *       })),
+     *     };
+     *   },
+     * });
+     * const { text } = await generateText({
+     *   model: openai("gpt-5-nano"),
+     *   prompt: "What is the capital of France?",
+     *   providerOptions: {
+     *     langsmith: lsConfig,
+     *   },
+     * });
+     * ```
+     *
+     * @param inputs Key-value map of the function inputs.
+     * @param inputs.formatted - Inputs formatted for LangSmith.
+     * @param inputs.raw - Raw inputs from the AI SDK.
+     * @returns A single combined key-value map of processed inputs.
+     */
+    processChildLLMRunInputs?: (inputs: LanguageModelV2CallOptions) => Record<string, unknown>;
+    /**
+     * Apply transformations to AI SDK child LLM run outputs before logging.
+     * This function should NOT mutate the outputs.
+     * Receives both "raw" and LangSmith-suggested "formatted" outputs,
+     * and should combine them into a single LangSmith-formatted output.
+     *
+     * ```ts
+     * import {
+     *   wrapAISDK,
+     *   createLangSmithProviderOptions,
+     * } from "langsmith/experimental/vercel";
+     * import * as ai from "ai";
+     * import { openai } from "@ai-sdk/openai";
+     *
+     * const { generateText } = wrapAISDK(ai);
+     *
+     * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+     *   processChildLLMRunOutputs: (outputs) => {
+     *     return {
+     *       providerMetadata: outputs.providerMetadata,
+     *       content: "REDACTED CHILD OUTPUTS",
+     *       role: "assistant",
+     *     };
+     *   },
+     * });
+     * const { text } = await generateText({
+     *   model: openai("gpt-5-nano"),
+     *   prompt: "What is the capital of France?",
+     *   providerOptions: {
+     *     langsmith: lsConfig,
+     *   },
+     * });
+     * ```
+     *
+     * @param outputs Key-value map of the function inputs.
+     * @param outputs.formatted - Outputs formatted for LangSmith.
+     * @param outputs.raw - Raw outputs from the AI SDK.
+     * @returns A single combined key-value map of processed outputs.
+     */
+    processChildLLMRunOutputs?: (outputs: "fullStream" extends keyof Awaited<ReturnType<T>> ? AggregatedDoStreamOutput : Awaited<ReturnType<LanguageModelV2["doGenerate"]>>) => Record<string, unknown>;
+};
+/**
+ * Wraps LangSmith config in a way that matches AI SDK provider types.
+ *
+ * ```ts
+ * import { createLangSmithProviderOptions } from "langsmith/experimental/vercel";
+ * import * as ai from "ai";
+ *
+ * const lsConfig = createLangSmithProviderOptions<typeof ai.generateText>({
+ *   // Will have appropriate typing
+ *   processInputs: (inputs) => {
+ *     const { messages } = inputs;
+ *     return {
+ *       messages: messages?.map((message) => ({
+ *         ...message,
+ *         content: "REDACTED",
+ *       })),
+ *       prompt: "REDACTED",
+ *     };
+ *   },
+ * });
+ * ```
+ *
+ * Note: AI SDK expects only JSON values in an object for
+ * provider options, but LangSmith's config may contain non-JSON values.
+ * These are not passed to the underlying AI SDK model, so it is safe to
+ * cast the typing here.
+ */
+export declare const createLangSmithProviderOptions: <T extends (...args: any[]) => any>(lsConfig?: WrapAISDKConfig<T>) => Record<string, JSONValue>;
 /**
  * Wraps Vercel AI SDK 5 functions with LangSmith tracing capabilities.
  *
@@ -21,7 +231,7 @@ declare const wrapAISDK: <GenerateTextType extends (...args: any[]) => any, Stre
     streamText: StreamTextType;
     streamObject: StreamObjectType;
     generateObject: GenerateObjectType;
-}, lsConfig?: Partial<Omit<RunTreeConfig, "inputs" | "outputs" | "run_type" | "child_runs" | "parent_run" | "error" | "serialized">>) => {
+}, baseLsConfig?: WrapAISDKConfig) => {
     generateText: GenerateTextType;
     generateObject: GenerateObjectType;
     streamText: StreamTextType;