langsmith 0.3.57 → 0.3.59

package/dist/experimental/vercel/index.cjs

@@ -225,6 +225,19 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 ...lsConfig?.metadata,
             },
             processInputs: (inputs) => _formatTracedInputs(inputs),
+            processOutputs: async (outputs) => {
+                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
+                    return outputs;
+                }
+                const content = await outputs.outputs.content;
+                if (content == null || typeof content !== "object") {
+                    return outputs;
+                }
+                return (0, middleware_js_1.populateToolCallsForTracing)({
+                    content,
+                    role: "assistant",
+                });
+            },
         });
         return traceableFunc(params);
     };
@@ -268,6 +281,16 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 ...lsConfig?.metadata,
             },
             processInputs: (inputs) => _formatTracedInputs(inputs),
+            processOutputs: async (outputs) => {
+                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
+                    return outputs;
+                }
+                const object = await outputs.outputs.object;
+                if (object == null || typeof object !== "object") {
+                    return outputs;
+                }
+                return object;
+            },
         });
         return traceableFunc(params);
     };

package/dist/experimental/vercel/index.d.ts

@@ -21,7 +21,7 @@ declare const wrapAISDK: <WrapLanguageModelType extends (...args: any[]) => any,
     streamText: StreamTextType;
     streamObject: StreamObjectType;
     generateObject: GenerateObjectType;
-}, lsConfig?: Partial<Omit<RunTreeConfig, "inputs" | "outputs">>) => {
+}, lsConfig?: Partial<Omit<RunTreeConfig, "inputs" | "outputs" | "run_type">>) => {
     generateText: (params: Parameters<GenerateTextType>[0]) => Promise<ReturnType<GenerateTextType>>;
     generateObject: (params: Parameters<GenerateObjectType>[0]) => Promise<ReturnType<GenerateObjectType>>;
     streamText: (params: Parameters<StreamTextType>[0]) => Promise<ReturnType<StreamTextType>>;

package/dist/experimental/vercel/index.js

@@ -222,6 +222,19 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 ...lsConfig?.metadata,
             },
             processInputs: (inputs) => _formatTracedInputs(inputs),
+            processOutputs: async (outputs) => {
+                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
+                    return outputs;
+                }
+                const content = await outputs.outputs.content;
+                if (content == null || typeof content !== "object") {
+                    return outputs;
+                }
+                return populateToolCallsForTracing({
+                    content,
+                    role: "assistant",
+                });
+            },
         });
         return traceableFunc(params);
     };
@@ -265,6 +278,16 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 ...lsConfig?.metadata,
             },
             processInputs: (inputs) => _formatTracedInputs(inputs),
+            processOutputs: async (outputs) => {
+                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
+                    return outputs;
+                }
+                const object = await outputs.outputs.object;
+                if (object == null || typeof object !== "object") {
+                    return outputs;
+                }
+                return object;
+            },
         });
         return traceableFunc(params);
     };

package/dist/experimental/vercel/middleware.cjs

@@ -142,6 +142,17 @@ function LangSmithMiddleware(config) {
                 const chunks = [];
                 const transformStream = new TransformStream({
                     async transform(chunk, controller) {
+                        if (chunk.type === "tool-input-start" ||
+                            chunk.type === "text-start") {
+                            // Only necessary to log the first token event
+                            if (runTree?.events == null ||
+                                (Array.isArray(runTree.events) && runTree.events.length === 0)) {
+                                runTree?.addEvent({ name: "new_token" });
+                            }
+                        }
+                        else if (chunk.type === "finish") {
+                            runTree?.addEvent({ name: "end" });
+                        }
                         chunks.push(chunk);
                         controller.enqueue(chunk);
                     },

package/dist/experimental/vercel/middleware.js

@@ -137,6 +137,17 @@ export function LangSmithMiddleware(config) {
                 const chunks = [];
                 const transformStream = new TransformStream({
                     async transform(chunk, controller) {
+                        if (chunk.type === "tool-input-start" ||
+                            chunk.type === "text-start") {
+                            // Only necessary to log the first token event
+                            if (runTree?.events == null ||
+                                (Array.isArray(runTree.events) && runTree.events.length === 0)) {
+                                runTree?.addEvent({ name: "new_token" });
+                            }
+                        }
+                        else if (chunk.type === "finish") {
+                            runTree?.addEvent({ name: "end" });
+                        }
                         chunks.push(chunk);
                         controller.enqueue(chunk);
                     },

package/dist/index.cjs

@@ -10,4 +10,4 @@ Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true
 var project_js_1 = require("./utils/project.cjs");
 Object.defineProperty(exports, "getDefaultProjectName", { enumerable: true, get: function () { return project_js_1.getDefaultProjectName; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.57";
+exports.__version__ = "0.3.59";

package/dist/index.d.ts

@@ -3,4 +3,4 @@ export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, }
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
-export declare const __version__ = "0.3.57";
+export declare const __version__ = "0.3.59";

package/dist/index.js

@@ -3,4 +3,4 @@ export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.57";
+export const __version__ = "0.3.59";

package/dist/traceable.cjs

@@ -100,7 +100,7 @@ const handleRunInputs = (inputs, processInputs) => {
 const _extractUsage = (runData) => {
     const usageMetadataFromMetadata = (runData.runTree.extra.metadata ?? {})
         .usage_metadata;
-    return runData.outputs.usage_metadata ?? usageMetadataFromMetadata;
+    return runData.outputs?.usage_metadata ?? usageMetadataFromMetadata;
 };
 function validateExtractedUsageMetadata(data) {
     const allowedKeys = new Set([
@@ -121,26 +121,25 @@ function validateExtractedUsageMetadata(data) {
     }
     return data;
 }
-// Note: This mutates the run tree
-function handleRunOutputs(params) {
-    const { runTree, rawOutputs, processOutputsFn } = params;
-    let outputs;
-    if ((0, asserts_js_1.isKVMap)(rawOutputs)) {
-        outputs = { ...rawOutputs };
-    }
-    else {
-        outputs = { outputs: rawOutputs };
-    }
-    try {
-        outputs = processOutputsFn(outputs);
-    }
-    catch (e) {
-        console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+async function handleEnd(params) {
+    const { runTree, on_end, postRunPromise } = params;
+    const onEnd = on_end;
+    if (onEnd) {
+        if (!runTree) {
+            console.warn("Can not call 'on_end' if currentRunTree is undefined");
+        }
+        else {
+            onEnd(runTree);
+        }
     }
+    await postRunPromise;
+    await runTree?.patchRun();
+}
+const _populateUsageMetadata = (processedOutputs, runTree) => {
     if (runTree !== undefined) {
         let usageMetadata;
         try {
-            usageMetadata = _extractUsage({ runTree, outputs });
+            usageMetadata = _extractUsage({ runTree, outputs: processedOutputs });
         }
         catch (e) {
             console.error("Error occurred while extracting usage metadata:", e);
@@ -150,10 +149,52 @@ function handleRunOutputs(params) {
                 ...runTree.extra.metadata,
                 usage_metadata: validateExtractedUsageMetadata(usageMetadata),
             };
-            outputs.usage_metadata = usageMetadata;
+            processedOutputs.usage_metadata = usageMetadata;
+        }
+    }
+};
+function isAsyncFn(fn) {
+    return (fn != null &&
+        typeof fn === "function" &&
+        fn.constructor.name === "AsyncFunction");
+}
+// Note: This mutates the run tree
+async function handleRunOutputs(params) {
+    const { runTree, rawOutputs, processOutputsFn, on_end, postRunPromise } = params;
+    let outputs;
+    if ((0, asserts_js_1.isKVMap)(rawOutputs)) {
+        outputs = { ...rawOutputs };
+    }
+    else {
+        outputs = { outputs: rawOutputs };
+    }
+    try {
+        outputs = processOutputsFn(outputs);
+        // TODO: Investigate making this behavior for all returned promises
+        // on next minor bump.
+        if (isAsyncFn(processOutputsFn)) {
+            void outputs
+                .then(async (processedOutputs) => {
+                _populateUsageMetadata(processedOutputs, runTree);
+                await runTree?.end(processedOutputs);
+            })
+                .catch(async (e) => {
+                console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+                await runTree?.end(outputs);
+            })
+                .finally(async () => {
+                await handleEnd({ runTree, postRunPromise, on_end });
+            });
+            return;
         }
     }
-    return outputs;
+    catch (e) {
+        console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+    }
+    _populateUsageMetadata(outputs, runTree);
+    await runTree?.end(outputs);
+    await handleEnd({ runTree, postRunPromise, on_end });
+    return;
 }
 const handleRunAttachments = (rawInputs, extractAttachments) => {
     if (!extractAttachments) {
@@ -473,13 +514,13 @@ function traceable(wrappedFunc, config) {
                                 : otel_context.with(capturedOtelContext, () => reader.read()));
                             if (result.done) {
                                 finished = true;
-                                const processedOutputs = handleRunOutputs({
+                                await handleRunOutputs({
                                     runTree: currentRunTree,
                                     rawOutputs: await handleChunks(chunks),
                                     processOutputsFn,
+                                    on_end: config?.on_end,
+                                    postRunPromise,
                                 });
-                                await currentRunTree?.end(processedOutputs);
-                                await handleEnd();
                                 controller.close();
                                 break;
                             }
@@ -497,13 +538,13 @@ function traceable(wrappedFunc, config) {
                     async cancel(reason) {
                         if (!finished)
                             await currentRunTree?.end(undefined, "Cancelled");
-                        const processedOutputs = handleRunOutputs({
+                        await handleRunOutputs({
                             runTree: currentRunTree,
                             rawOutputs: await handleChunks(chunks),
                             processOutputsFn,
+                            on_end: config?.on_end,
+                            postRunPromise,
                         });
-                        await currentRunTree?.end(processedOutputs);
-                        await handleEnd();
                         return reader.cancel(reason);
                     },
                 });
@@ -540,13 +581,13 @@ function traceable(wrappedFunc, config) {
                 finally {
                     if (!finished)
                         await currentRunTree?.end(undefined, "Cancelled");
-                    const processedOutputs = handleRunOutputs({
+                    await handleRunOutputs({
                         runTree: currentRunTree,
                         rawOutputs: await handleChunks(chunks),
                         processOutputsFn,
+                        on_end: config?.on_end,
+                        postRunPromise,
                     });
-                    await currentRunTree?.end(processedOutputs);
-                    await handleEnd();
                 }
             }
             function wrapAsyncGeneratorForTracing(iterable, snapshot) {
@@ -558,19 +599,6 @@ function traceable(wrappedFunc, config) {
                 iterable[Symbol.asyncIterator] = () => wrappedIterator;
                 return iterable;
             }
-            async function handleEnd() {
-                const onEnd = config?.on_end;
-                if (onEnd) {
-                    if (!currentRunTree) {
-                        console.warn("Can not call 'on_end' if currentRunTree is undefined");
-                    }
-                    else {
-                        onEnd(currentRunTree);
-                    }
-                }
-                await postRunPromise;
-                await currentRunTree?.patchRun();
-            }
             function gatherAll(iterator) {
                 const chunks = [];
                 // eslint-disable-next-line no-constant-condition
@@ -625,7 +653,7 @@ function traceable(wrappedFunc, config) {
                     if ((0, asserts_js_1.isGenerator)(wrappedFunc) && (0, asserts_js_1.isIteratorLike)(rawOutput)) {
                         const chunks = gatherAll(rawOutput);
                         try {
-                            const processedOutputs = handleRunOutputs({
+                            await handleRunOutputs({
                                 runTree: currentRunTree,
                                 rawOutputs: await handleChunks(chunks.reduce((memo, { value, done }) => {
                                     if (!done || typeof value !== "undefined") {
@@ -634,12 +662,12 @@ function traceable(wrappedFunc, config) {
                                     return memo;
                                 }, [])),
                                 processOutputsFn,
+                                on_end: config?.on_end,
+                                postRunPromise,
                             });
-                            await currentRunTree?.end(processedOutputs);
-                            await handleEnd();
                         }
                         catch (e) {
-                            console.error("Error occurred during handleEnd:", e);
+                            console.error("[LANGSMITH]: Error occurred while handling run outputs:", e);
                         }
                         return (function* () {
                             for (const ret of chunks) {
@@ -650,13 +678,13 @@ function traceable(wrappedFunc, config) {
                         })();
                     }
                     try {
-                        const processedOutputs = handleRunOutputs({
+                        await handleRunOutputs({
                             runTree: currentRunTree,
                             rawOutputs: rawOutput,
                             processOutputsFn,
+                            on_end: config?.on_end,
+                            postRunPromise,
                         });
-                        await currentRunTree?.end(processedOutputs);
-                        await handleEnd();
                     }
                     finally {
                         // eslint-disable-next-line no-unsafe-finally
@@ -664,7 +692,11 @@ function traceable(wrappedFunc, config) {
                     }
                 }, async (error) => {
                     await currentRunTree?.end(undefined, String(error));
-                    await handleEnd();
+                    await handleEnd({
+                        runTree: currentRunTree,
+                        postRunPromise,
+                        on_end: config?.on_end,
+                    });
                     throw error;
                 })
                     .then(resolve, reject);

package/dist/traceable.d.ts

@@ -39,7 +39,7 @@ export type TraceableConfig<Func extends (...args: any[]) => any> = Partial<Omit
      * @param outputs Key-value map of the function outputs
      * @returns Transformed key-value map
      */
-    processOutputs?: (outputs: Readonly<KVMap>) => KVMap;
+    processOutputs?: (outputs: Readonly<KVMap>) => KVMap | Promise<KVMap>;
 };
 /**
  * Higher-order function that takes function as input and returns a

package/dist/traceable.js

@@ -96,7 +96,7 @@ const handleRunInputs = (inputs, processInputs) => {
 const _extractUsage = (runData) => {
     const usageMetadataFromMetadata = (runData.runTree.extra.metadata ?? {})
         .usage_metadata;
-    return runData.outputs.usage_metadata ?? usageMetadataFromMetadata;
+    return runData.outputs?.usage_metadata ?? usageMetadataFromMetadata;
 };
 function validateExtractedUsageMetadata(data) {
     const allowedKeys = new Set([
@@ -117,26 +117,25 @@ function validateExtractedUsageMetadata(data) {
     }
     return data;
 }
-// Note: This mutates the run tree
-function handleRunOutputs(params) {
-    const { runTree, rawOutputs, processOutputsFn } = params;
-    let outputs;
-    if (isKVMap(rawOutputs)) {
-        outputs = { ...rawOutputs };
-    }
-    else {
-        outputs = { outputs: rawOutputs };
-    }
-    try {
-        outputs = processOutputsFn(outputs);
-    }
-    catch (e) {
-        console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+async function handleEnd(params) {
+    const { runTree, on_end, postRunPromise } = params;
+    const onEnd = on_end;
+    if (onEnd) {
+        if (!runTree) {
+            console.warn("Can not call 'on_end' if currentRunTree is undefined");
+        }
+        else {
+            onEnd(runTree);
+        }
     }
+    await postRunPromise;
+    await runTree?.patchRun();
+}
+const _populateUsageMetadata = (processedOutputs, runTree) => {
     if (runTree !== undefined) {
         let usageMetadata;
         try {
-            usageMetadata = _extractUsage({ runTree, outputs });
+            usageMetadata = _extractUsage({ runTree, outputs: processedOutputs });
         }
         catch (e) {
             console.error("Error occurred while extracting usage metadata:", e);
@@ -146,10 +145,52 @@ function handleRunOutputs(params) {
                 ...runTree.extra.metadata,
                 usage_metadata: validateExtractedUsageMetadata(usageMetadata),
             };
-            outputs.usage_metadata = usageMetadata;
+            processedOutputs.usage_metadata = usageMetadata;
+        }
+    }
+};
+function isAsyncFn(fn) {
+    return (fn != null &&
+        typeof fn === "function" &&
+        fn.constructor.name === "AsyncFunction");
+}
+// Note: This mutates the run tree
+async function handleRunOutputs(params) {
+    const { runTree, rawOutputs, processOutputsFn, on_end, postRunPromise } = params;
+    let outputs;
+    if (isKVMap(rawOutputs)) {
+        outputs = { ...rawOutputs };
+    }
+    else {
+        outputs = { outputs: rawOutputs };
+    }
+    try {
+        outputs = processOutputsFn(outputs);
+        // TODO: Investigate making this behavior for all returned promises
+        // on next minor bump.
+        if (isAsyncFn(processOutputsFn)) {
+            void outputs
+                .then(async (processedOutputs) => {
+                _populateUsageMetadata(processedOutputs, runTree);
+                await runTree?.end(processedOutputs);
+            })
+                .catch(async (e) => {
+                console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+                await runTree?.end(outputs);
+            })
+                .finally(async () => {
+                await handleEnd({ runTree, postRunPromise, on_end });
+            });
+            return;
         }
     }
-    return outputs;
+    catch (e) {
+        console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
+    }
+    _populateUsageMetadata(outputs, runTree);
+    await runTree?.end(outputs);
+    await handleEnd({ runTree, postRunPromise, on_end });
+    return;
 }
 const handleRunAttachments = (rawInputs, extractAttachments) => {
     if (!extractAttachments) {
@@ -469,13 +510,13 @@ export function traceable(wrappedFunc, config) {
                                 : otel_context.with(capturedOtelContext, () => reader.read()));
                             if (result.done) {
                                 finished = true;
-                                const processedOutputs = handleRunOutputs({
+                                await handleRunOutputs({
                                     runTree: currentRunTree,
                                     rawOutputs: await handleChunks(chunks),
                                     processOutputsFn,
+                                    on_end: config?.on_end,
+                                    postRunPromise,
                                 });
-                                await currentRunTree?.end(processedOutputs);
-                                await handleEnd();
                                 controller.close();
                                 break;
                             }
@@ -493,13 +534,13 @@ export function traceable(wrappedFunc, config) {
                     async cancel(reason) {
                         if (!finished)
                             await currentRunTree?.end(undefined, "Cancelled");
-                        const processedOutputs = handleRunOutputs({
+                        await handleRunOutputs({
                             runTree: currentRunTree,
                             rawOutputs: await handleChunks(chunks),
                             processOutputsFn,
+                            on_end: config?.on_end,
+                            postRunPromise,
                         });
-                        await currentRunTree?.end(processedOutputs);
-                        await handleEnd();
                         return reader.cancel(reason);
                     },
                 });
@@ -536,13 +577,13 @@ export function traceable(wrappedFunc, config) {
                 finally {
                     if (!finished)
                         await currentRunTree?.end(undefined, "Cancelled");
-                    const processedOutputs = handleRunOutputs({
+                    await handleRunOutputs({
                         runTree: currentRunTree,
                         rawOutputs: await handleChunks(chunks),
                         processOutputsFn,
+                        on_end: config?.on_end,
+                        postRunPromise,
                     });
-                    await currentRunTree?.end(processedOutputs);
-                    await handleEnd();
                 }
             }
             function wrapAsyncGeneratorForTracing(iterable, snapshot) {
@@ -554,19 +595,6 @@ export function traceable(wrappedFunc, config) {
                 iterable[Symbol.asyncIterator] = () => wrappedIterator;
                 return iterable;
             }
-            async function handleEnd() {
-                const onEnd = config?.on_end;
-                if (onEnd) {
-                    if (!currentRunTree) {
-                        console.warn("Can not call 'on_end' if currentRunTree is undefined");
-                    }
-                    else {
-                        onEnd(currentRunTree);
-                    }
-                }
-                await postRunPromise;
-                await currentRunTree?.patchRun();
-            }
             function gatherAll(iterator) {
                 const chunks = [];
                 // eslint-disable-next-line no-constant-condition
@@ -621,7 +649,7 @@ export function traceable(wrappedFunc, config) {
                     if (isGenerator(wrappedFunc) && isIteratorLike(rawOutput)) {
                         const chunks = gatherAll(rawOutput);
                         try {
-                            const processedOutputs = handleRunOutputs({
+                            await handleRunOutputs({
                                 runTree: currentRunTree,
                                 rawOutputs: await handleChunks(chunks.reduce((memo, { value, done }) => {
                                     if (!done || typeof value !== "undefined") {
@@ -630,12 +658,12 @@ export function traceable(wrappedFunc, config) {
                                     return memo;
                                 }, [])),
                                 processOutputsFn,
+                                on_end: config?.on_end,
+                                postRunPromise,
                             });
-                            await currentRunTree?.end(processedOutputs);
-                            await handleEnd();
                         }
                         catch (e) {
-                            console.error("Error occurred during handleEnd:", e);
+                            console.error("[LANGSMITH]: Error occurred while handling run outputs:", e);
                         }
                         return (function* () {
                             for (const ret of chunks) {
@@ -646,13 +674,13 @@ export function traceable(wrappedFunc, config) {
                         })();
                     }
                     try {
-                        const processedOutputs = handleRunOutputs({
+                        await handleRunOutputs({
                             runTree: currentRunTree,
                             rawOutputs: rawOutput,
                             processOutputsFn,
+                            on_end: config?.on_end,
+                            postRunPromise,
                         });
-                        await currentRunTree?.end(processedOutputs);
-                        await handleEnd();
                     }
                     finally {
                         // eslint-disable-next-line no-unsafe-finally
@@ -660,7 +688,11 @@ export function traceable(wrappedFunc, config) {
                     }
                 }, async (error) => {
                     await currentRunTree?.end(undefined, String(error));
-                    await handleEnd();
+                    await handleEnd({
+                        runTree: currentRunTree,
+                        postRunPromise,
+                        on_end: config?.on_end,
+                    });
                     throw error;
                 })
                     .then(resolve, reject);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.57",
+  "version": "0.3.59",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [