langsmith 0.3.68 → 0.3.70

package/dist/client.cjs

@@ -165,7 +165,7 @@ class AutoBatchQueue {
         this.sizeBytes += size;
         return itemPromise;
     }
-    pop(upToSizeBytes) {
+    pop({ upToSizeBytes, upToSize, }) {
         if (upToSizeBytes < 1) {
             throw new Error("Number of bytes to pop off may not be less than 1.");
         }
@@ -173,7 +173,8 @@ class AutoBatchQueue {
         let poppedSizeBytes = 0;
         // Pop items until we reach or exceed the size limit
         while (poppedSizeBytes + (this.peek()?.size ?? 0) < upToSizeBytes &&
-            this.items.length > 0) {
+            this.items.length > 0 &&
+            popped.length < upToSize) {
             const item = this.items.shift();
             if (item) {
                 popped.push(item);
@@ -204,6 +205,8 @@ class AutoBatchQueue {
 exports.AutoBatchQueue = AutoBatchQueue;
 exports.DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES = 24 * 1024 * 1024;
 const SERVER_INFO_REQUEST_TIMEOUT_MS = 10000;
+/** Maximum number of operations to batch in a single request. */
+const DEFAULT_BATCH_SIZE_LIMIT = 100;
 const DEFAULT_API_URL = "https://api.smith.langchain.com";
 class Client {
     get _fetch() {
@@ -312,6 +315,12 @@ class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "batchSizeLimit", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "fetchOptions", {
             enumerable: true,
             configurable: true,
@@ -419,6 +428,7 @@ class Client {
         this.blockOnRootRunFinalization =
             config.blockOnRootRunFinalization ?? this.blockOnRootRunFinalization;
         this.batchSizeBytesLimit = config.batchSizeBytesLimit;
+        this.batchSizeLimit = config.batchSizeLimit;
         this.fetchOptions = config.fetchOptions || {};
         this.manualFlushMode = config.manualFlushMode ?? this.manualFlushMode;
         if ((0, env_js_1.getOtelEnabled)()) {
@@ -660,14 +670,27 @@ class Client {
             serverInfo.batch_ingest_config?.size_limit_bytes ??
             exports.DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES);
     }
+    /**
+     * Get the maximum number of operations to batch in a single request.
+     */
+    async _getBatchSizeLimit() {
+        const serverInfo = await this._ensureServerInfo();
+        return (this.batchSizeLimit ??
+            serverInfo.batch_ingest_config?.size_limit ??
+            DEFAULT_BATCH_SIZE_LIMIT);
+    }
     async _getDatasetExamplesMultiPartSupport() {
         const serverInfo = await this._ensureServerInfo();
         return (serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false);
     }
-    drainAutoBatchQueue(batchSizeLimit) {
+    drainAutoBatchQueue({ batchSizeLimitBytes, batchSizeLimit, }) {
         const promises = [];
         while (this.autoBatchQueue.items.length > 0) {
-            const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
+            const [batch, done] = this.autoBatchQueue.pop({
+                upToSizeBytes: batchSizeLimitBytes,
+                upToSize: batchSizeLimit,
+            });
+            console.log("batch", batch?.length);
             if (!batch.length) {
                 done();
                 break;
@@ -766,13 +789,21 @@ class Client {
             return itemPromise;
         }
         const sizeLimitBytes = await this._getBatchSizeLimitBytes();
-        if (this.autoBatchQueue.sizeBytes > sizeLimitBytes) {
-            void this.drainAutoBatchQueue(sizeLimitBytes);
+        const sizeLimit = await this._getBatchSizeLimit();
+        if (this.autoBatchQueue.sizeBytes > sizeLimitBytes ||
+            this.autoBatchQueue.items.length > sizeLimit) {
+            void this.drainAutoBatchQueue({
+                batchSizeLimitBytes: sizeLimitBytes,
+                batchSizeLimit: sizeLimit,
+            });
         }
         if (this.autoBatchQueue.items.length > 0) {
             this.autoBatchTimeout = setTimeout(() => {
                 this.autoBatchTimeout = undefined;
-                void this.drainAutoBatchQueue(sizeLimitBytes);
+                void this.drainAutoBatchQueue({
+                    batchSizeLimitBytes: sizeLimitBytes,
+                    batchSizeLimit: sizeLimit,
+                });
             }, this.autoBatchAggregationDelayMs);
         }
         return itemPromise;
@@ -828,7 +859,11 @@ class Client {
      */
     async flush() {
         const sizeLimitBytes = await this._getBatchSizeLimitBytes();
-        await this.drainAutoBatchQueue(sizeLimitBytes);
+        const sizeLimit = await this._getBatchSizeLimit();
+        await this.drainAutoBatchQueue({
+            batchSizeLimitBytes: sizeLimitBytes,
+            batchSizeLimit: sizeLimit,
+        });
     }
     _cloneCurrentOTELContext() {
         const otel_trace = (0, otel_js_1.getOTELTrace)();

package/dist/client.d.ts

@@ -12,7 +12,10 @@ export interface ClientConfig {
     hideInputs?: boolean | ((inputs: KVMap) => KVMap | Promise<KVMap>);
     hideOutputs?: boolean | ((outputs: KVMap) => KVMap | Promise<KVMap>);
     autoBatchTracing?: boolean;
+    /** Maximum size of a batch of runs in bytes. */
     batchSizeBytesLimit?: number;
+    /** Maximum number of operations to batch in a single request. */
+    batchSizeLimit?: number;
     blockOnRootRunFinalization?: boolean;
     traceBatchConcurrency?: number;
     fetchOptions?: RequestInit;
@@ -292,7 +295,10 @@ export declare class AutoBatchQueue {
         apiUrl?: string;
     };
     push(item: AutoBatchQueueItem): Promise<void>;
-    pop(upToSizeBytes: number): [AutoBatchQueueItem[], () => void];
+    pop({ upToSizeBytes, upToSize, }: {
+        upToSizeBytes: number;
+        upToSize: number;
+    }): [AutoBatchQueueItem[], () => void];
 }
 export declare const DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES: number;
 export declare class Client implements LangSmithTracingClientInterface {
@@ -313,6 +319,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     private autoBatchTimeout;
     private autoBatchAggregationDelayMs;
     private batchSizeBytesLimit?;
+    private batchSizeLimit?;
     private fetchOptions;
     private settings;
     private blockOnRootRunFinalization;
@@ -346,6 +353,10 @@ export declare class Client implements LangSmithTracingClientInterface {
     private _shouldSample;
     private _filterForSampling;
     private _getBatchSizeLimitBytes;
+    /**
+     * Get the maximum number of operations to batch in a single request.
+     */
+    private _getBatchSizeLimit;
     private _getDatasetExamplesMultiPartSupport;
     private drainAutoBatchQueue;
     private _processBatch;

package/dist/client.js

@@ -128,7 +128,7 @@ export class AutoBatchQueue {
         this.sizeBytes += size;
         return itemPromise;
     }
-    pop(upToSizeBytes) {
+    pop({ upToSizeBytes, upToSize, }) {
         if (upToSizeBytes < 1) {
             throw new Error("Number of bytes to pop off may not be less than 1.");
         }
@@ -136,7 +136,8 @@ export class AutoBatchQueue {
         let poppedSizeBytes = 0;
         // Pop items until we reach or exceed the size limit
         while (poppedSizeBytes + (this.peek()?.size ?? 0) < upToSizeBytes &&
-            this.items.length > 0) {
+            this.items.length > 0 &&
+            popped.length < upToSize) {
             const item = this.items.shift();
             if (item) {
                 popped.push(item);
@@ -166,6 +167,8 @@ export class AutoBatchQueue {
 }
 export const DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES = 24 * 1024 * 1024;
 const SERVER_INFO_REQUEST_TIMEOUT_MS = 10000;
+/** Maximum number of operations to batch in a single request. */
+const DEFAULT_BATCH_SIZE_LIMIT = 100;
 const DEFAULT_API_URL = "https://api.smith.langchain.com";
 export class Client {
     get _fetch() {
@@ -274,6 +277,12 @@ export class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "batchSizeLimit", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "fetchOptions", {
             enumerable: true,
             configurable: true,
@@ -381,6 +390,7 @@ export class Client {
         this.blockOnRootRunFinalization =
             config.blockOnRootRunFinalization ?? this.blockOnRootRunFinalization;
         this.batchSizeBytesLimit = config.batchSizeBytesLimit;
+        this.batchSizeLimit = config.batchSizeLimit;
         this.fetchOptions = config.fetchOptions || {};
         this.manualFlushMode = config.manualFlushMode ?? this.manualFlushMode;
         if (getOtelEnabled()) {
@@ -622,14 +632,27 @@ export class Client {
             serverInfo.batch_ingest_config?.size_limit_bytes ??
             DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES);
     }
+    /**
+     * Get the maximum number of operations to batch in a single request.
+     */
+    async _getBatchSizeLimit() {
+        const serverInfo = await this._ensureServerInfo();
+        return (this.batchSizeLimit ??
+            serverInfo.batch_ingest_config?.size_limit ??
+            DEFAULT_BATCH_SIZE_LIMIT);
+    }
     async _getDatasetExamplesMultiPartSupport() {
         const serverInfo = await this._ensureServerInfo();
         return (serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false);
     }
-    drainAutoBatchQueue(batchSizeLimit) {
+    drainAutoBatchQueue({ batchSizeLimitBytes, batchSizeLimit, }) {
         const promises = [];
         while (this.autoBatchQueue.items.length > 0) {
-            const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
+            const [batch, done] = this.autoBatchQueue.pop({
+                upToSizeBytes: batchSizeLimitBytes,
+                upToSize: batchSizeLimit,
+            });
+            console.log("batch", batch?.length);
             if (!batch.length) {
                 done();
                 break;
@@ -728,13 +751,21 @@ export class Client {
             return itemPromise;
         }
         const sizeLimitBytes = await this._getBatchSizeLimitBytes();
-        if (this.autoBatchQueue.sizeBytes > sizeLimitBytes) {
-            void this.drainAutoBatchQueue(sizeLimitBytes);
+        const sizeLimit = await this._getBatchSizeLimit();
+        if (this.autoBatchQueue.sizeBytes > sizeLimitBytes ||
+            this.autoBatchQueue.items.length > sizeLimit) {
+            void this.drainAutoBatchQueue({
+                batchSizeLimitBytes: sizeLimitBytes,
+                batchSizeLimit: sizeLimit,
+            });
         }
         if (this.autoBatchQueue.items.length > 0) {
             this.autoBatchTimeout = setTimeout(() => {
                 this.autoBatchTimeout = undefined;
-                void this.drainAutoBatchQueue(sizeLimitBytes);
+                void this.drainAutoBatchQueue({
+                    batchSizeLimitBytes: sizeLimitBytes,
+                    batchSizeLimit: sizeLimit,
+                });
             }, this.autoBatchAggregationDelayMs);
         }
         return itemPromise;
@@ -790,7 +821,11 @@ export class Client {
      */
     async flush() {
         const sizeLimitBytes = await this._getBatchSizeLimitBytes();
-        await this.drainAutoBatchQueue(sizeLimitBytes);
+        const sizeLimit = await this._getBatchSizeLimit();
+        await this.drainAutoBatchQueue({
+            batchSizeLimitBytes: sizeLimitBytes,
+            batchSizeLimit: sizeLimit,
+        });
     }
     _cloneCurrentOTELContext() {
         const otel_trace = getOTELTrace();

package/dist/experimental/vercel/index.cjs

@@ -325,32 +325,34 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 return inputFormatter(inputs);
             },
             processOutputs: async (outputs) => {
-                if (resolvedLsConfig?.processOutputs) {
-                    return resolvedLsConfig.processOutputs(outputs);
-                }
-                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
-                    return outputs;
-                }
-                let content;
-                if ("content" in outputs.outputs && outputs.outputs.content != null) {
-                    content = await outputs.outputs.content;
-                }
-                else if ("text" in outputs.outputs &&
-                    outputs.outputs.text != null) {
-                    // AI SDK 4 shim
-                    content = await outputs.outputs.text;
-                }
-                else {
-                    return outputs;
+                try {
+                    if (resolvedLsConfig?.processOutputs) {
+                        return resolvedLsConfig.processOutputs(outputs);
+                    }
+                    if (outputs.outputs == null ||
+                        typeof outputs.outputs !== "object") {
+                        return outputs;
+                    }
+                    // Important: Even accessing this property creates a promise.
+                    // This must be awaited.
+                    let content = await outputs.outputs.content;
+                    if (content == null) {
+                        // AI SDK 4 shim
+                        content = await outputs.outputs.text;
+                    }
+                    if (content == null ||
+                        !["object", "string"].includes(typeof content)) {
+                        return outputs;
+                    }
+                    return (0, utils_js_1.convertMessageToTracedFormat)({
+                        content,
+                        role: "assistant",
+                    });
                 }
-                if (content == null ||
-                    !["object", "string"].includes(typeof content)) {
+                catch (e) {
+                    // Handle parsing failures without a log
                     return outputs;
                 }
-                return (0, utils_js_1.convertMessageToTracedFormat)({
-                    content,
-                    role: "assistant",
-                });
             },
         });
         return traceableFunc(...args);
@@ -405,17 +407,24 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 return inputFormatter(inputs);
             },
             processOutputs: async (outputs) => {
-                if (resolvedLsConfig?.processOutputs) {
-                    return resolvedLsConfig.processOutputs(outputs);
-                }
-                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
-                    return outputs;
+                try {
+                    if (resolvedLsConfig?.processOutputs) {
+                        return resolvedLsConfig.processOutputs(outputs);
+                    }
+                    if (outputs.outputs == null ||
+                        typeof outputs.outputs !== "object") {
+                        return outputs;
+                    }
+                    const object = await outputs.outputs.object;
+                    if (object == null || typeof object !== "object") {
+                        return outputs;
+                    }
+                    return object;
                 }
-                const object = await outputs.outputs.object;
-                if (object == null || typeof object !== "object") {
+                catch (e) {
+                    // Handle parsing failures without a log
                     return outputs;
                 }
-                return object;
             },
         });
         return traceableFunc(...args);

package/dist/experimental/vercel/index.js

@@ -321,32 +321,34 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 return inputFormatter(inputs);
             },
             processOutputs: async (outputs) => {
-                if (resolvedLsConfig?.processOutputs) {
-                    return resolvedLsConfig.processOutputs(outputs);
-                }
-                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
-                    return outputs;
-                }
-                let content;
-                if ("content" in outputs.outputs && outputs.outputs.content != null) {
-                    content = await outputs.outputs.content;
-                }
-                else if ("text" in outputs.outputs &&
-                    outputs.outputs.text != null) {
-                    // AI SDK 4 shim
-                    content = await outputs.outputs.text;
-                }
-                else {
-                    return outputs;
+                try {
+                    if (resolvedLsConfig?.processOutputs) {
+                        return resolvedLsConfig.processOutputs(outputs);
+                    }
+                    if (outputs.outputs == null ||
+                        typeof outputs.outputs !== "object") {
+                        return outputs;
+                    }
+                    // Important: Even accessing this property creates a promise.
+                    // This must be awaited.
+                    let content = await outputs.outputs.content;
+                    if (content == null) {
+                        // AI SDK 4 shim
+                        content = await outputs.outputs.text;
+                    }
+                    if (content == null ||
+                        !["object", "string"].includes(typeof content)) {
+                        return outputs;
+                    }
+                    return convertMessageToTracedFormat({
+                        content,
+                        role: "assistant",
+                    });
                 }
-                if (content == null ||
-                    !["object", "string"].includes(typeof content)) {
+                catch (e) {
+                    // Handle parsing failures without a log
                     return outputs;
                 }
-                return convertMessageToTracedFormat({
-                    content,
-                    role: "assistant",
-                });
             },
         });
         return traceableFunc(...args);
@@ -401,17 +403,24 @@ const wrapAISDK = ({ wrapLanguageModel, generateText, streamText, streamObject,
                 return inputFormatter(inputs);
             },
             processOutputs: async (outputs) => {
-                if (resolvedLsConfig?.processOutputs) {
-                    return resolvedLsConfig.processOutputs(outputs);
-                }
-                if (outputs.outputs == null || typeof outputs.outputs !== "object") {
-                    return outputs;
+                try {
+                    if (resolvedLsConfig?.processOutputs) {
+                        return resolvedLsConfig.processOutputs(outputs);
+                    }
+                    if (outputs.outputs == null ||
+                        typeof outputs.outputs !== "object") {
+                        return outputs;
+                    }
+                    const object = await outputs.outputs.object;
+                    if (object == null || typeof object !== "object") {
+                        return outputs;
+                    }
+                    return object;
                 }
-                const object = await outputs.outputs.object;
-                if (object == null || typeof object !== "object") {
+                catch (e) {
+                    // Handle parsing failures without a log
                     return outputs;
                 }
-                return object;
             },
         });
         return traceableFunc(...args);

package/dist/experimental/vercel/middleware.cjs

@@ -204,7 +204,7 @@ function LangSmithMiddleware(config) {
                                 tool_calls: [],
                             });
                             const outputFormatter = lsConfig?.processOutputs ?? utils_js_1.convertMessageToTracedFormat;
-                            const formattedOutputs = outputFormatter(output);
+                            const formattedOutputs = await outputFormatter(output);
                             await runTree?.end(formattedOutputs);
                         }
                         catch (error) {

package/dist/experimental/vercel/middleware.d.ts

@@ -22,6 +22,6 @@ export declare function LangSmithMiddleware(config?: {
     modelId?: string;
     lsConfig?: Partial<Omit<RunTreeConfig, "inputs" | "outputs" | "run_type">> & {
         processInputs?: (inputs: Record<string, unknown>) => Record<string, unknown>;
-        processOutputs?: (outputs: Record<string, unknown>) => Record<string, unknown>;
+        processOutputs?: (outputs: Record<string, unknown>) => Record<string, unknown> | Promise<Record<string, unknown>>;
     };
 }): LanguageModelV2Middleware;

package/dist/experimental/vercel/middleware.js

@@ -201,7 +201,7 @@ export function LangSmithMiddleware(config) {
                                 tool_calls: [],
                             });
                             const outputFormatter = lsConfig?.processOutputs ?? convertMessageToTracedFormat;
-                            const formattedOutputs = outputFormatter(output);
+                            const formattedOutputs = await outputFormatter(output);
                             await runTree?.end(formattedOutputs);
                         }
                         catch (error) {

package/dist/index.cjs

@@ -10,4 +10,4 @@ Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true
 var project_js_1 = require("./utils/project.cjs");
 Object.defineProperty(exports, "getDefaultProjectName", { enumerable: true, get: function () { return project_js_1.getDefaultProjectName; } });
 // Update using yarn bump-version
-exports.__version__ = "0.3.68";
+exports.__version__ = "0.3.70";

package/dist/index.d.ts

@@ -3,4 +3,4 @@ export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, }
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
-export declare const __version__ = "0.3.68";
+export declare const __version__ = "0.3.70";

package/dist/index.js

@@ -3,4 +3,4 @@ export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 export { getDefaultProjectName } from "./utils/project.js";
 // Update using yarn bump-version
-export const __version__ = "0.3.68";
+export const __version__ = "0.3.70";

package/dist/traceable.cjs

@@ -166,10 +166,20 @@ async function handleRunOutputs(params) {
             })
                 .catch(async (e) => {
                 console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
-                await runTree?.end(outputs);
+                try {
+                    await runTree?.end(outputs);
+                }
+                catch (e) {
+                    console.error("Error occurred during runTree?.end.", e);
+                }
             })
                 .finally(async () => {
-                await handleEnd({ runTree, postRunPromise, on_end, excludeInputs });
+                try {
+                    await handleEnd({ runTree, postRunPromise, on_end, excludeInputs });
+                }
+                catch (e) {
+                    console.error("Error occurred during handleEnd.", e);
+                }
             });
             return;
         }

package/dist/traceable.js

@@ -162,10 +162,20 @@ async function handleRunOutputs(params) {
             })
                 .catch(async (e) => {
                 console.error("Error occurred during processOutputs. Sending unprocessed outputs:", e);
-                await runTree?.end(outputs);
+                try {
+                    await runTree?.end(outputs);
+                }
+                catch (e) {
+                    console.error("Error occurred during runTree?.end.", e);
+                }
             })
                 .finally(async () => {
-                await handleEnd({ runTree, postRunPromise, on_end, excludeInputs });
+                try {
+                    await handleEnd({ runTree, postRunPromise, on_end, excludeInputs });
+                }
+                catch (e) {
+                    console.error("Error occurred during handleEnd.", e);
+                }
             });
             return;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.68",
+  "version": "0.3.70",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -114,6 +114,7 @@
     "test:integration": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
     "watch:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --watch --config jest.config.cjs --testTimeout 100000",
+    "test:vitest": "vitest run --config vitest.config.ts",
     "test:eval:vitest": "vitest run --config ls.vitest.config.ts",
     "lint": "NODE_OPTIONS=--max-old-space-size=4096 eslint --cache --ext .ts,.js src/",
     "lint:fix": "yarn lint --fix",
@@ -148,7 +149,7 @@
     "uuid": "^10.0.0"
   },
   "devDependencies": {
-    "@ai-sdk/anthropic": "^2.0.1",
+    "@ai-sdk/anthropic": "^2.0.17",
     "@ai-sdk/openai": "^2.0.23",
     "@babel/preset-env": "^7.22.4",
     "@faker-js/faker": "^8.4.1",
@@ -167,7 +168,7 @@
     "@types/node-fetch": "^2.6.12",
     "@typescript-eslint/eslint-plugin": "^5.59.8",
     "@typescript-eslint/parser": "^5.59.8",
-    "ai": "^5.0.10",
+    "ai": "^5.0.44",
     "babel-jest": "^29.5.0",
     "cross-env": "^7.0.3",
     "dotenv": "^16.1.3",
@@ -178,6 +179,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^29.5.0",
     "langchain": "^0.3.29",
+    "msw": "^2.11.2",
     "node-fetch": "^2.7.0",
     "openai": "^5.8.2",
     "prettier": "^2.8.8",