langsmith 0.1.5 → 0.1.7

package/dist/client.cjs

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Client = exports.Queue = void 0;
+exports.Client = exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = exports.Queue = void 0;
 const uuid = __importStar(require("uuid"));
 const async_caller_js_1 = require("./utils/async_caller.cjs");
 const messages_js_1 = require("./utils/messages.cjs");
@@ -98,6 +98,18 @@ function assertUuid(str) {
         throw new Error(`Invalid UUID: ${str}`);
     }
 }
+const handle429 = async (response) => {
+    if (response?.status === 429) {
+        const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000;
+        if (retryAfter > 0) {
+            await new Promise((resolve) => setTimeout(resolve, retryAfter));
+            // Return directly after calling this check
+            return true;
+        }
+    }
+    // Fall back to existing status checks
+    return false;
+};
 class Queue {
     constructor() {
         Object.defineProperty(this, "items", {
@@ -135,6 +147,8 @@ class Queue {
     }
 }
 exports.Queue = Queue;
+// 20 MB
+exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
 class Client {
     constructor(config = {}) {
         Object.defineProperty(this, "apiKey", {
@@ -161,6 +175,12 @@ class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "batchIngestCaller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "timeout_ms", {
             enumerable: true,
             configurable: true,
@@ -239,6 +259,12 @@ class Client {
             writable: true,
             value: 50
         });
+        Object.defineProperty(this, "serverInfo", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         const defaultConfig = Client.getDefaultClientConfig();
         this.tracingSampleRate = getTracingSamplingRate();
         this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? "";
@@ -246,6 +272,10 @@ class Client {
         this.webUrl = trimQuotes(config.webUrl ?? defaultConfig.webUrl);
         this.timeout_ms = config.timeout_ms ?? 12000;
         this.caller = new async_caller_js_1.AsyncCaller(config.callerOptions ?? {});
+        this.batchIngestCaller = new async_caller_js_1.AsyncCaller({
+            ...(config.callerOptions ?? {}),
+            onFailedResponseHook: handle429,
+        });
         this.hideInputs = config.hideInputs ?? defaultConfig.hideInputs;
         this.hideOutputs = config.hideOutputs ?? defaultConfig.hideOutputs;
         this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
@@ -448,14 +478,16 @@ class Client {
         if (this.autoBatchQueue.size > 0) {
             this.autoBatchTimeout = setTimeout(() => {
                 this.autoBatchTimeout = undefined;
-                void this.drainAutoBatchQueue();
+                // This error would happen in the background and is uncatchable
+                // from the outside. So just log instead.
+                void this.drainAutoBatchQueue().catch(console.error);
             }, oldTimeout
                 ? this.autoBatchAggregationDelayMs
                 : this.autoBatchInitialDelayMs);
         }
         return itemPromise;
     }
-    async batchEndpointIsSupported() {
+    async _getServerInfo() {
         const response = await fetch(`${this.apiUrl}/info`, {
             method: "GET",
             headers: { Accept: "application/json" },
@@ -465,6 +497,15 @@ class Client {
             // consume the response body to release the connection
             // https://undici.nodejs.org/#/?id=garbage-collection
             await response.text();
+            throw new Error("Failed to retrieve server info.");
+        }
+        return response.json();
+    }
+    async batchEndpointIsSupported() {
+        try {
+            this.serverInfo = await this._getServerInfo();
+        }
+        catch (e) {
             return false;
         }
         return true;
@@ -487,7 +528,7 @@ class Client {
             void this.processRunOperation({
                 action: "create",
                 item: runCreate,
-            });
+            }).catch(console.error);
             return;
         }
         const mergedRunCreateParams = await mergeRuntimeEnvIntoRunCreates([
@@ -534,11 +575,11 @@ class Client {
             preparedCreateParams = Object.values(createById);
             preparedUpdateParams = standaloneUpdates;
         }
-        const body = {
+        const rawBatch = {
             post: this._filterForSampling(preparedCreateParams),
             patch: this._filterForSampling(preparedUpdateParams, true),
         };
-        if (!body.post.length && !body.patch.length) {
+        if (!rawBatch.post.length && !rawBatch.patch.length) {
             return;
         }
         preparedCreateParams = await mergeRuntimeEnvIntoRunCreates(preparedCreateParams);
@@ -547,33 +588,58 @@ class Client {
         }
         if (!this.batchEndpointSupported) {
             this.autoBatchTracing = false;
-            for (const preparedCreateParam of body.post) {
+            for (const preparedCreateParam of rawBatch.post) {
                 await this.createRun(preparedCreateParam);
             }
-            for (const preparedUpdateParam of body.patch) {
+            for (const preparedUpdateParam of rawBatch.patch) {
                 if (preparedUpdateParam.id !== undefined) {
                     await this.updateRun(preparedUpdateParam.id, preparedUpdateParam);
                 }
             }
             return;
         }
+        const sizeLimitBytes = this.serverInfo?.batch_ingest_config?.size_limit_bytes ??
+            exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES;
+        const batchChunks = {
+            post: [],
+            patch: [],
+        };
+        let currentBatchSizeBytes = 0;
+        for (const k of ["post", "patch"]) {
+            const key = k;
+            const batchItems = rawBatch[key].reverse();
+            let batchItem = batchItems.pop();
+            while (batchItem !== undefined) {
+                const stringifiedBatchItem = JSON.stringify(batchItem);
+                if (currentBatchSizeBytes > 0 &&
+                    currentBatchSizeBytes + stringifiedBatchItem.length > sizeLimitBytes) {
+                    await this._postBatchIngestRuns(JSON.stringify(batchChunks));
+                    currentBatchSizeBytes = 0;
+                    batchChunks.post = [];
+                    batchChunks.patch = [];
+                }
+                currentBatchSizeBytes += stringifiedBatchItem.length;
+                batchChunks[key].push(batchItem);
+                batchItem = batchItems.pop();
+            }
+        }
+        if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
+            await this._postBatchIngestRuns(JSON.stringify(batchChunks));
+        }
+    }
+    async _postBatchIngestRuns(body) {
         const headers = {
             ...this.headers,
             "Content-Type": "application/json",
             Accept: "application/json",
         };
-        try {
-            const response = await this.caller.call(fetch, `${this.apiUrl}/runs/batch`, {
-                method: "POST",
-                headers,
-                body: JSON.stringify(body),
-                signal: AbortSignal.timeout(this.timeout_ms),
-            });
-            await raiseForStatus(response, "batch create run");
-        }
-        catch (e) {
-            console.error(`Failed to batch create runs: ${e}`);
-        }
+        const response = await this.batchIngestCaller.call(fetch, `${this.apiUrl}/runs/batch`, {
+            method: "POST",
+            headers,
+            body: body,
+            signal: AbortSignal.timeout(this.timeout_ms),
+        });
+        await raiseForStatus(response, "batch create run");
     }
     async updateRun(runId, run) {
         assertUuid(runId);
@@ -598,7 +664,7 @@ class Client {
                 return;
             }
             else {
-                void this.processRunOperation({ action: "update", item: data });
+                void this.processRunOperation({ action: "update", item: data }).catch(console.error);
             }
             return;
         }

package/dist/client.d.ts

@@ -72,11 +72,13 @@ export declare class Queue<T> {
     push(item: T): Promise<void>;
     pop(upToN: number): [T[], () => void];
 }
+export declare const DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
 export declare class Client {
     private apiKey?;
     private apiUrl;
     private webUrl?;
     private caller;
+    private batchIngestCaller;
     private timeout_ms;
     private _tenantId;
     private hideInputs?;
@@ -90,6 +92,7 @@ export declare class Client {
     private autoBatchTimeout;
     private autoBatchInitialDelayMs;
     private autoBatchAggregationDelayMs;
+    private serverInfo;
     constructor(config?: ClientConfig);
     static getDefaultClientConfig(): {
         apiUrl: string;
@@ -110,6 +113,7 @@ export declare class Client {
     private _filterForSampling;
     private drainAutoBatchQueue;
     private processRunOperation;
+    protected _getServerInfo(): Promise<any>;
     protected batchEndpointIsSupported(): Promise<boolean>;
     createRun(run: CreateRunParams): Promise<void>;
     /**
@@ -120,6 +124,7 @@ export declare class Client {
         runCreates?: RunCreate[];
         runUpdates?: RunUpdate[];
     }): Promise<void>;
+    private _postBatchIngestRuns;
     updateRun(runId: string, run: RunUpdate): Promise<void>;
     readRun(runId: string, { loadChildRuns }?: {
         loadChildRuns: boolean;

package/dist/client.js

@@ -72,6 +72,18 @@ function assertUuid(str) {
         throw new Error(`Invalid UUID: ${str}`);
     }
 }
+const handle429 = async (response) => {
+    if (response?.status === 429) {
+        const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000;
+        if (retryAfter > 0) {
+            await new Promise((resolve) => setTimeout(resolve, retryAfter));
+            // Return directly after calling this check
+            return true;
+        }
+    }
+    // Fall back to existing status checks
+    return false;
+};
 export class Queue {
     constructor() {
         Object.defineProperty(this, "items", {
@@ -108,6 +120,8 @@ export class Queue {
         return [popped.map((it) => it[0]), () => popped.forEach((it) => it[1]())];
     }
 }
+// 20 MB
+export const DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
 export class Client {
     constructor(config = {}) {
         Object.defineProperty(this, "apiKey", {
@@ -134,6 +148,12 @@ export class Client {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "batchIngestCaller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "timeout_ms", {
             enumerable: true,
             configurable: true,
@@ -212,6 +232,12 @@ export class Client {
             writable: true,
             value: 50
         });
+        Object.defineProperty(this, "serverInfo", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         const defaultConfig = Client.getDefaultClientConfig();
         this.tracingSampleRate = getTracingSamplingRate();
         this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? "";
@@ -219,6 +245,10 @@ export class Client {
         this.webUrl = trimQuotes(config.webUrl ?? defaultConfig.webUrl);
         this.timeout_ms = config.timeout_ms ?? 12000;
         this.caller = new AsyncCaller(config.callerOptions ?? {});
+        this.batchIngestCaller = new AsyncCaller({
+            ...(config.callerOptions ?? {}),
+            onFailedResponseHook: handle429,
+        });
         this.hideInputs = config.hideInputs ?? defaultConfig.hideInputs;
         this.hideOutputs = config.hideOutputs ?? defaultConfig.hideOutputs;
         this.autoBatchTracing = config.autoBatchTracing ?? this.autoBatchTracing;
@@ -421,14 +451,16 @@ export class Client {
         if (this.autoBatchQueue.size > 0) {
             this.autoBatchTimeout = setTimeout(() => {
                 this.autoBatchTimeout = undefined;
-                void this.drainAutoBatchQueue();
+                // This error would happen in the background and is uncatchable
+                // from the outside. So just log instead.
+                void this.drainAutoBatchQueue().catch(console.error);
             }, oldTimeout
                 ? this.autoBatchAggregationDelayMs
                 : this.autoBatchInitialDelayMs);
         }
         return itemPromise;
     }
-    async batchEndpointIsSupported() {
+    async _getServerInfo() {
         const response = await fetch(`${this.apiUrl}/info`, {
             method: "GET",
             headers: { Accept: "application/json" },
@@ -438,6 +470,15 @@ export class Client {
             // consume the response body to release the connection
             // https://undici.nodejs.org/#/?id=garbage-collection
             await response.text();
+            throw new Error("Failed to retrieve server info.");
+        }
+        return response.json();
+    }
+    async batchEndpointIsSupported() {
+        try {
+            this.serverInfo = await this._getServerInfo();
+        }
+        catch (e) {
             return false;
         }
         return true;
@@ -460,7 +501,7 @@ export class Client {
             void this.processRunOperation({
                 action: "create",
                 item: runCreate,
-            });
+            }).catch(console.error);
             return;
         }
         const mergedRunCreateParams = await mergeRuntimeEnvIntoRunCreates([
@@ -507,11 +548,11 @@ export class Client {
             preparedCreateParams = Object.values(createById);
             preparedUpdateParams = standaloneUpdates;
         }
-        const body = {
+        const rawBatch = {
             post: this._filterForSampling(preparedCreateParams),
             patch: this._filterForSampling(preparedUpdateParams, true),
         };
-        if (!body.post.length && !body.patch.length) {
+        if (!rawBatch.post.length && !rawBatch.patch.length) {
             return;
         }
         preparedCreateParams = await mergeRuntimeEnvIntoRunCreates(preparedCreateParams);
@@ -520,33 +561,58 @@ export class Client {
         }
         if (!this.batchEndpointSupported) {
             this.autoBatchTracing = false;
-            for (const preparedCreateParam of body.post) {
+            for (const preparedCreateParam of rawBatch.post) {
                 await this.createRun(preparedCreateParam);
             }
-            for (const preparedUpdateParam of body.patch) {
+            for (const preparedUpdateParam of rawBatch.patch) {
                 if (preparedUpdateParam.id !== undefined) {
                     await this.updateRun(preparedUpdateParam.id, preparedUpdateParam);
                 }
             }
             return;
         }
+        const sizeLimitBytes = this.serverInfo?.batch_ingest_config?.size_limit_bytes ??
+            DEFAULT_BATCH_SIZE_LIMIT_BYTES;
+        const batchChunks = {
+            post: [],
+            patch: [],
+        };
+        let currentBatchSizeBytes = 0;
+        for (const k of ["post", "patch"]) {
+            const key = k;
+            const batchItems = rawBatch[key].reverse();
+            let batchItem = batchItems.pop();
+            while (batchItem !== undefined) {
+                const stringifiedBatchItem = JSON.stringify(batchItem);
+                if (currentBatchSizeBytes > 0 &&
+                    currentBatchSizeBytes + stringifiedBatchItem.length > sizeLimitBytes) {
+                    await this._postBatchIngestRuns(JSON.stringify(batchChunks));
+                    currentBatchSizeBytes = 0;
+                    batchChunks.post = [];
+                    batchChunks.patch = [];
+                }
+                currentBatchSizeBytes += stringifiedBatchItem.length;
+                batchChunks[key].push(batchItem);
+                batchItem = batchItems.pop();
+            }
+        }
+        if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
+            await this._postBatchIngestRuns(JSON.stringify(batchChunks));
+        }
+    }
+    async _postBatchIngestRuns(body) {
         const headers = {
             ...this.headers,
             "Content-Type": "application/json",
             Accept: "application/json",
         };
-        try {
-            const response = await this.caller.call(fetch, `${this.apiUrl}/runs/batch`, {
-                method: "POST",
-                headers,
-                body: JSON.stringify(body),
-                signal: AbortSignal.timeout(this.timeout_ms),
-            });
-            await raiseForStatus(response, "batch create run");
-        }
-        catch (e) {
-            console.error(`Failed to batch create runs: ${e}`);
-        }
+        const response = await this.batchIngestCaller.call(fetch, `${this.apiUrl}/runs/batch`, {
+            method: "POST",
+            headers,
+            body: body,
+            signal: AbortSignal.timeout(this.timeout_ms),
+        });
+        await raiseForStatus(response, "batch create run");
     }
     async updateRun(runId, run) {
         assertUuid(runId);
@@ -571,7 +637,7 @@ export class Client {
                 return;
             }
             else {
-                void this.processRunOperation({ action: "update", item: data });
+                void this.processRunOperation({ action: "update", item: data }).catch(console.error);
             }
             return;
         }

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.5";
+exports.__version__ = "0.1.7";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.5";
+export declare const __version__ = "0.1.7";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.5";
+export const __version__ = "0.1.7";

package/dist/utils/async_caller.cjs

@@ -52,13 +52,21 @@ class AsyncCaller {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "onFailedResponseHook", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.maxConcurrency = params.maxConcurrency ?? Infinity;
         this.maxRetries = params.maxRetries ?? 6;
         const PQueue = "default" in p_queue_1.default ? p_queue_1.default.default : p_queue_1.default;
         this.queue = new PQueue({ concurrency: this.maxConcurrency });
+        this.onFailedResponseHook = params?.onFailedResponseHook;
     }
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     call(callable, ...args) {
+        const onFailedResponseHook = this.onFailedResponseHook;
         return this.queue.add(() => (0, p_retry_1.default)(() => callable(...args).catch((error) => {
             // eslint-disable-next-line no-instanceof/no-instanceof
             if (error instanceof Error) {
@@ -68,7 +76,7 @@ class AsyncCaller {
                 throw new Error(error);
             }
         }), {
-            onFailedAttempt(error) {
+            async onFailedAttempt(error) {
                 if (error.message.startsWith("Cancel") ||
                     error.message.startsWith("TimeoutError") ||
                     error.message.startsWith("AbortError")) {
@@ -79,7 +87,8 @@ class AsyncCaller {
                     throw error;
                 }
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const status = error?.response?.status;
+                const response = error?.response;
+                const status = response?.status;
                 if (status) {
                     if (STATUS_NO_RETRY.includes(+status)) {
                         throw error;
@@ -87,6 +96,9 @@ class AsyncCaller {
                     else if (STATUS_IGNORE.includes(+status)) {
                         return;
                     }
+                    if (onFailedResponseHook) {
+                        await onFailedResponseHook(response);
+                    }
                 }
             },
             // If needed we can change some of the defaults here,

package/dist/utils/async_caller.d.ts

@@ -1,3 +1,4 @@
+type ResponseCallback = (response?: Response) => Promise<boolean>;
 export interface AsyncCallerParams {
     /**
      * The maximum number of concurrent calls that can be made.
@@ -9,6 +10,7 @@ export interface AsyncCallerParams {
      * with an exponential backoff between each attempt. Defaults to 6.
      */
     maxRetries?: number;
+    onFailedResponseHook?: ResponseCallback;
 }
 export interface AsyncCallerCallOptions {
     signal?: AbortSignal;
@@ -30,8 +32,10 @@ export declare class AsyncCaller {
     protected maxConcurrency: AsyncCallerParams["maxConcurrency"];
     protected maxRetries: AsyncCallerParams["maxRetries"];
     private queue;
+    private onFailedResponseHook?;
     constructor(params: AsyncCallerParams);
     call<A extends any[], T extends (...args: A) => Promise<any>>(callable: T, ...args: Parameters<T>): Promise<Awaited<ReturnType<T>>>;
     callWithOptions<A extends any[], T extends (...args: A) => Promise<any>>(options: AsyncCallerCallOptions, callable: T, ...args: Parameters<T>): Promise<Awaited<ReturnType<T>>>;
     fetch(...args: Parameters<typeof fetch>): ReturnType<typeof fetch>;
 }
+export {};

package/dist/utils/async_caller.js

@@ -46,13 +46,21 @@ export class AsyncCaller {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "onFailedResponseHook", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.maxConcurrency = params.maxConcurrency ?? Infinity;
         this.maxRetries = params.maxRetries ?? 6;
         const PQueue = "default" in PQueueMod ? PQueueMod.default : PQueueMod;
         this.queue = new PQueue({ concurrency: this.maxConcurrency });
+        this.onFailedResponseHook = params?.onFailedResponseHook;
     }
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     call(callable, ...args) {
+        const onFailedResponseHook = this.onFailedResponseHook;
         return this.queue.add(() => pRetry(() => callable(...args).catch((error) => {
             // eslint-disable-next-line no-instanceof/no-instanceof
             if (error instanceof Error) {
@@ -62,7 +70,7 @@ export class AsyncCaller {
                 throw new Error(error);
             }
         }), {
-            onFailedAttempt(error) {
+            async onFailedAttempt(error) {
                 if (error.message.startsWith("Cancel") ||
                     error.message.startsWith("TimeoutError") ||
                     error.message.startsWith("AbortError")) {
@@ -73,7 +81,8 @@ export class AsyncCaller {
                     throw error;
                 }
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                const status = error?.response?.status;
+                const response = error?.response;
+                const status = response?.status;
                 if (status) {
                     if (STATUS_NO_RETRY.includes(+status)) {
                         throw error;
@@ -81,6 +90,9 @@ export class AsyncCaller {
                     else if (STATUS_IGNORE.includes(+status)) {
                         return;
                     }
+                    if (onFailedResponseHook) {
+                        await onFailedResponseHook(response);
+                    }
                 }
             },
             // If needed we can change some of the defaults here,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [