langsmith 0.2.11 → 0.2.13

package/dist/client.cjs

@@ -533,6 +533,10 @@ class Client {
             serverInfo.batch_ingest_config?.size_limit_bytes ??
             exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES);
     }
+    async _getMultiPartSupport() {
+        const serverInfo = await this._ensureServerInfo();
+        return (serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false);
+    }
     drainAutoBatchQueue(batchSizeLimit) {
         while (this.autoBatchQueue.items.length > 0) {
             const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
@@ -830,7 +834,16 @@ class Client {
                     const attachments = allAttachments[payload.id];
                     if (attachments) {
                         delete allAttachments[payload.id];
-                        for (const [name, [contentType, content]] of Object.entries(attachments)) {
+                        for (const [name, attachment] of Object.entries(attachments)) {
+                            let contentType;
+                            let content;
+                            if (Array.isArray(attachment)) {
+                                [contentType, content] = attachment;
+                            }
+                            else {
+                                contentType = attachment.mimeType;
+                                content = attachment.data;
+                            }
                             // Validate that the attachment name doesn't contain a '.'
                             if (name.includes(".")) {
                                 console.warn(`Skipping attachment '${name}' for run ${payload.id}: Invalid attachment name. ` +
@@ -1938,9 +1951,21 @@ class Client {
     async readExample(exampleId) {
         (0, _uuid_js_1.assertUuid)(exampleId);
         const path = `/examples/${exampleId}`;
-        return await this._get(path);
+        const rawExample = await this._get(path);
+        const { attachment_urls, ...rest } = rawExample;
+        const example = rest;
+        if (attachment_urls) {
+            // add attachments back to the example
+            example.attachments = Object.entries(attachment_urls).reduce((acc, [key, value]) => {
+                acc[key.slice("attachment.".length)] = {
+                    presigned_url: value.presigned_url,
+                };
+                return acc;
+            }, {});
+        }
+        return example;
     }
-    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, } = {}) {
+    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, includeAttachments, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -1989,9 +2014,22 @@ class Client {
         if (filter !== undefined) {
             params.append("filter", filter);
         }
+        if (includeAttachments === true) {
+            ["attachment_urls", "outputs", "metadata"].forEach((field) => params.append("select", field));
+        }
         let i = 0;
-        for await (const examples of this._getPaginated("/examples", params)) {
-            for (const example of examples) {
+        for await (const rawExamples of this._getPaginated("/examples", params)) {
+            for (const rawExample of rawExamples) {
+                const { attachment_urls, ...rest } = rawExample;
+                const example = rest;
+                if (attachment_urls) {
+                    example.attachments = Object.entries(attachment_urls).reduce((acc, [key, value]) => {
+                        acc[key.slice("attachment.".length)] = {
+                            presigned_url: value.presigned_url,
+                        };
+                        return acc;
+                    }, {});
+                }
                 yield example;
                 i++;
             }
@@ -2662,6 +2700,144 @@ class Client {
         const result = await response.json();
         return this._getPromptUrl(`${owner}/${promptName}${result.commit_hash ? `:${result.commit_hash}` : ""}`);
     }
+    /**
+     * Update examples with attachments using multipart form data.
+     * @param updates List of ExampleUpdateWithAttachments objects to upsert
+     * @returns Promise with the update response
+     */
+    async updateExamplesMultipart(datasetId, updates = []) {
+        if (!(await this._getMultiPartSupport())) {
+            throw new Error("Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version.");
+        }
+        const formData = new FormData();
+        for (const example of updates) {
+            const exampleId = example.id;
+            // Prepare the main example body
+            const exampleBody = {
+                ...(example.metadata && { metadata: example.metadata }),
+                ...(example.split && { split: example.split }),
+            };
+            // Add main example data
+            const stringifiedExample = (0, index_js_2.stringify)(exampleBody);
+            const exampleBlob = new Blob([stringifiedExample], {
+                type: "application/json",
+            });
+            formData.append(exampleId, exampleBlob);
+            // Add inputs
+            if (example.inputs) {
+                const stringifiedInputs = (0, index_js_2.stringify)(example.inputs);
+                const inputsBlob = new Blob([stringifiedInputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.inputs`, inputsBlob);
+            }
+            // Add outputs if present
+            if (example.outputs) {
+                const stringifiedOutputs = (0, index_js_2.stringify)(example.outputs);
+                const outputsBlob = new Blob([stringifiedOutputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.outputs`, outputsBlob);
+            }
+            // Add attachments if present
+            if (example.attachments) {
+                for (const [name, attachment] of Object.entries(example.attachments)) {
+                    let mimeType;
+                    let data;
+                    if (Array.isArray(attachment)) {
+                        [mimeType, data] = attachment;
+                    }
+                    else {
+                        mimeType = attachment.mimeType;
+                        data = attachment.data;
+                    }
+                    const attachmentBlob = new Blob([data], {
+                        type: `${mimeType}; length=${data.byteLength}`,
+                    });
+                    formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
+                }
+            }
+            if (example.attachments_operations) {
+                const stringifiedAttachmentsOperations = (0, index_js_2.stringify)(example.attachments_operations);
+                const attachmentsOperationsBlob = new Blob([stringifiedAttachmentsOperations], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.attachments_operations`, attachmentsOperationsBlob);
+            }
+        }
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`, {
+            method: "PATCH",
+            headers: this.headers,
+            body: formData,
+        });
+        const result = await response.json();
+        return result;
+    }
+    /**
+     * Upload examples with attachments using multipart form data.
+     * @param uploads List of ExampleUploadWithAttachments objects to upload
+     * @returns Promise with the upload response
+     */
+    async uploadExamplesMultipart(datasetId, uploads = []) {
+        if (!(await this._getMultiPartSupport())) {
+            throw new Error("Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version.");
+        }
+        const formData = new FormData();
+        for (const example of uploads) {
+            const exampleId = (example.id ?? uuid.v4()).toString();
+            // Prepare the main example body
+            const exampleBody = {
+                created_at: example.created_at,
+                ...(example.metadata && { metadata: example.metadata }),
+                ...(example.split && { split: example.split }),
+            };
+            // Add main example data
+            const stringifiedExample = (0, index_js_2.stringify)(exampleBody);
+            const exampleBlob = new Blob([stringifiedExample], {
+                type: "application/json",
+            });
+            formData.append(exampleId, exampleBlob);
+            // Add inputs
+            const stringifiedInputs = (0, index_js_2.stringify)(example.inputs);
+            const inputsBlob = new Blob([stringifiedInputs], {
+                type: "application/json",
+            });
+            formData.append(`${exampleId}.inputs`, inputsBlob);
+            // Add outputs if present
+            if (example.outputs) {
+                const stringifiedOutputs = (0, index_js_2.stringify)(example.outputs);
+                const outputsBlob = new Blob([stringifiedOutputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.outputs`, outputsBlob);
+            }
+            // Add attachments if present
+            if (example.attachments) {
+                for (const [name, attachment] of Object.entries(example.attachments)) {
+                    let mimeType;
+                    let data;
+                    if (Array.isArray(attachment)) {
+                        [mimeType, data] = attachment;
+                    }
+                    else {
+                        mimeType = attachment.mimeType;
+                        data = attachment.data;
+                    }
+                    const attachmentBlob = new Blob([data], {
+                        type: `${mimeType}; length=${data.byteLength}`,
+                    });
+                    formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
+                }
+            }
+        }
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`, {
+            method: "POST",
+            headers: this.headers,
+            body: formData,
+        });
+        const result = await response.json();
+        return result;
+    }
     async updatePrompt(promptIdentifier, options) {
         if (!(await this.promptExists(promptIdentifier))) {
             throw new Error("Prompt does not exist, you must create it first.");

package/dist/client.d.ts

@@ -1,5 +1,5 @@
 import { AsyncCallerParams } from "./utils/async_caller.js";
-import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments } from "./schemas.js";
+import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo, Attachments, ExampleUploadWithAttachments, UploadExamplesResponse, ExampleUpdateWithAttachments, UpdateExamplesResponse } from "./schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
 export interface ClientConfig {
     apiUrl?: string;
@@ -226,6 +226,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     private _getCursorPaginatedList;
     private _filterForSampling;
     private _getBatchSizeLimitBytes;
+    private _getMultiPartSupport;
     private drainAutoBatchQueue;
     private _processBatch;
     private processRunOperation;
@@ -531,7 +532,7 @@ export declare class Client implements LangSmithTracingClientInterface {
     createLLMExample(input: string, generation: string | undefined, options: CreateExampleOptions): Promise<Example>;
     createChatExample(input: KVMap[] | LangChainBaseMessage[], generations: KVMap | LangChainBaseMessage | undefined, options: CreateExampleOptions): Promise<Example>;
     readExample(exampleId: string): Promise<Example>;
-    listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, }?: {
+    listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, includeAttachments, }?: {
         datasetId?: string;
         datasetName?: string;
         exampleIds?: string[];
@@ -542,6 +543,7 @@ export declare class Client implements LangSmithTracingClientInterface {
         limit?: number;
         offset?: number;
         filter?: string;
+        includeAttachments?: boolean;
     }): AsyncIterable<Example>;
     deleteExample(exampleId: string): Promise<void>;
     updateExample(exampleId: string, update: ExampleUpdate): Promise<object>;
@@ -726,6 +728,18 @@ export declare class Client implements LangSmithTracingClientInterface {
     createCommit(promptIdentifier: string, object: any, options?: {
         parentCommitHash?: string;
     }): Promise<string>;
+    /**
+     * Update examples with attachments using multipart form data.
+     * @param updates List of ExampleUpdateWithAttachments objects to upsert
+     * @returns Promise with the update response
+     */
+    updateExamplesMultipart(datasetId: string, updates?: ExampleUpdateWithAttachments[]): Promise<UpdateExamplesResponse>;
+    /**
+     * Upload examples with attachments using multipart form data.
+     * @param uploads List of ExampleUploadWithAttachments objects to upload
+     * @returns Promise with the upload response
+     */
+    uploadExamplesMultipart(datasetId: string, uploads?: ExampleUploadWithAttachments[]): Promise<UploadExamplesResponse>;
     updatePrompt(promptIdentifier: string, options?: {
         description?: string;
         readme?: string;

package/dist/client.js

@@ -505,6 +505,10 @@ export class Client {
             serverInfo.batch_ingest_config?.size_limit_bytes ??
             DEFAULT_BATCH_SIZE_LIMIT_BYTES);
     }
+    async _getMultiPartSupport() {
+        const serverInfo = await this._ensureServerInfo();
+        return (serverInfo.instance_flags?.dataset_examples_multipart_enabled ?? false);
+    }
     drainAutoBatchQueue(batchSizeLimit) {
         while (this.autoBatchQueue.items.length > 0) {
             const [batch, done] = this.autoBatchQueue.pop(batchSizeLimit);
@@ -802,7 +806,16 @@ export class Client {
                     const attachments = allAttachments[payload.id];
                     if (attachments) {
                         delete allAttachments[payload.id];
-                        for (const [name, [contentType, content]] of Object.entries(attachments)) {
+                        for (const [name, attachment] of Object.entries(attachments)) {
+                            let contentType;
+                            let content;
+                            if (Array.isArray(attachment)) {
+                                [contentType, content] = attachment;
+                            }
+                            else {
+                                contentType = attachment.mimeType;
+                                content = attachment.data;
+                            }
                             // Validate that the attachment name doesn't contain a '.'
                             if (name.includes(".")) {
                                 console.warn(`Skipping attachment '${name}' for run ${payload.id}: Invalid attachment name. ` +
@@ -1910,9 +1923,21 @@ export class Client {
     async readExample(exampleId) {
         assertUuid(exampleId);
         const path = `/examples/${exampleId}`;
-        return await this._get(path);
+        const rawExample = await this._get(path);
+        const { attachment_urls, ...rest } = rawExample;
+        const example = rest;
+        if (attachment_urls) {
+            // add attachments back to the example
+            example.attachments = Object.entries(attachment_urls).reduce((acc, [key, value]) => {
+                acc[key.slice("attachment.".length)] = {
+                    presigned_url: value.presigned_url,
+                };
+                return acc;
+            }, {});
+        }
+        return example;
     }
-    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, } = {}) {
+    async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter, includeAttachments, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -1961,9 +1986,22 @@ export class Client {
         if (filter !== undefined) {
             params.append("filter", filter);
         }
+        if (includeAttachments === true) {
+            ["attachment_urls", "outputs", "metadata"].forEach((field) => params.append("select", field));
+        }
         let i = 0;
-        for await (const examples of this._getPaginated("/examples", params)) {
-            for (const example of examples) {
+        for await (const rawExamples of this._getPaginated("/examples", params)) {
+            for (const rawExample of rawExamples) {
+                const { attachment_urls, ...rest } = rawExample;
+                const example = rest;
+                if (attachment_urls) {
+                    example.attachments = Object.entries(attachment_urls).reduce((acc, [key, value]) => {
+                        acc[key.slice("attachment.".length)] = {
+                            presigned_url: value.presigned_url,
+                        };
+                        return acc;
+                    }, {});
+                }
                 yield example;
                 i++;
             }
@@ -2634,6 +2672,144 @@ export class Client {
         const result = await response.json();
         return this._getPromptUrl(`${owner}/${promptName}${result.commit_hash ? `:${result.commit_hash}` : ""}`);
     }
+    /**
+     * Update examples with attachments using multipart form data.
+     * @param updates List of ExampleUpdateWithAttachments objects to upsert
+     * @returns Promise with the update response
+     */
+    async updateExamplesMultipart(datasetId, updates = []) {
+        if (!(await this._getMultiPartSupport())) {
+            throw new Error("Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version.");
+        }
+        const formData = new FormData();
+        for (const example of updates) {
+            const exampleId = example.id;
+            // Prepare the main example body
+            const exampleBody = {
+                ...(example.metadata && { metadata: example.metadata }),
+                ...(example.split && { split: example.split }),
+            };
+            // Add main example data
+            const stringifiedExample = stringifyForTracing(exampleBody);
+            const exampleBlob = new Blob([stringifiedExample], {
+                type: "application/json",
+            });
+            formData.append(exampleId, exampleBlob);
+            // Add inputs
+            if (example.inputs) {
+                const stringifiedInputs = stringifyForTracing(example.inputs);
+                const inputsBlob = new Blob([stringifiedInputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.inputs`, inputsBlob);
+            }
+            // Add outputs if present
+            if (example.outputs) {
+                const stringifiedOutputs = stringifyForTracing(example.outputs);
+                const outputsBlob = new Blob([stringifiedOutputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.outputs`, outputsBlob);
+            }
+            // Add attachments if present
+            if (example.attachments) {
+                for (const [name, attachment] of Object.entries(example.attachments)) {
+                    let mimeType;
+                    let data;
+                    if (Array.isArray(attachment)) {
+                        [mimeType, data] = attachment;
+                    }
+                    else {
+                        mimeType = attachment.mimeType;
+                        data = attachment.data;
+                    }
+                    const attachmentBlob = new Blob([data], {
+                        type: `${mimeType}; length=${data.byteLength}`,
+                    });
+                    formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
+                }
+            }
+            if (example.attachments_operations) {
+                const stringifiedAttachmentsOperations = stringifyForTracing(example.attachments_operations);
+                const attachmentsOperationsBlob = new Blob([stringifiedAttachmentsOperations], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.attachments_operations`, attachmentsOperationsBlob);
+            }
+        }
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`, {
+            method: "PATCH",
+            headers: this.headers,
+            body: formData,
+        });
+        const result = await response.json();
+        return result;
+    }
+    /**
+     * Upload examples with attachments using multipart form data.
+     * @param uploads List of ExampleUploadWithAttachments objects to upload
+     * @returns Promise with the upload response
+     */
+    async uploadExamplesMultipart(datasetId, uploads = []) {
+        if (!(await this._getMultiPartSupport())) {
+            throw new Error("Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version.");
+        }
+        const formData = new FormData();
+        for (const example of uploads) {
+            const exampleId = (example.id ?? uuid.v4()).toString();
+            // Prepare the main example body
+            const exampleBody = {
+                created_at: example.created_at,
+                ...(example.metadata && { metadata: example.metadata }),
+                ...(example.split && { split: example.split }),
+            };
+            // Add main example data
+            const stringifiedExample = stringifyForTracing(exampleBody);
+            const exampleBlob = new Blob([stringifiedExample], {
+                type: "application/json",
+            });
+            formData.append(exampleId, exampleBlob);
+            // Add inputs
+            const stringifiedInputs = stringifyForTracing(example.inputs);
+            const inputsBlob = new Blob([stringifiedInputs], {
+                type: "application/json",
+            });
+            formData.append(`${exampleId}.inputs`, inputsBlob);
+            // Add outputs if present
+            if (example.outputs) {
+                const stringifiedOutputs = stringifyForTracing(example.outputs);
+                const outputsBlob = new Blob([stringifiedOutputs], {
+                    type: "application/json",
+                });
+                formData.append(`${exampleId}.outputs`, outputsBlob);
+            }
+            // Add attachments if present
+            if (example.attachments) {
+                for (const [name, attachment] of Object.entries(example.attachments)) {
+                    let mimeType;
+                    let data;
+                    if (Array.isArray(attachment)) {
+                        [mimeType, data] = attachment;
+                    }
+                    else {
+                        mimeType = attachment.mimeType;
+                        data = attachment.data;
+                    }
+                    const attachmentBlob = new Blob([data], {
+                        type: `${mimeType}; length=${data.byteLength}`,
+                    });
+                    formData.append(`${exampleId}.attachment.${name}`, attachmentBlob);
+                }
+            }
+        }
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/v1/platform/datasets/${datasetId}/examples`, {
+            method: "POST",
+            headers: this.headers,
+            body: formData,
+        });
+        const result = await response.json();
+        return result;
+    }
     async updatePrompt(promptIdentifier, options) {
         if (!(await this.promptExists(promptIdentifier))) {
             throw new Error("Prompt does not exist, you must create it first.");

package/dist/evaluation/_runner.cjs

@@ -38,7 +38,10 @@ class _ExperimentManager {
             if (!this._data) {
                 throw new Error("Data not provided in this experiment.");
             }
-            const unresolvedData = _resolveData(this._data, { client: this.client });
+            const unresolvedData = _resolveData(this._data, {
+                client: this.client,
+                includeAttachments: this._includeAttachments,
+            });
             if (!this._examples) {
                 this._examples = [];
             }
@@ -169,6 +172,12 @@ class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_includeAttachments", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.client = args.client ?? new index_js_1.Client();
         if (!args.experiment) {
             this._experimentName = (0, _random_name_js_1.randomName)();
@@ -202,6 +211,7 @@ class _ExperimentManager {
         this._evaluationResults = args.evaluationResults;
         this._summaryResults = args.summaryResults;
         this._numRepetitions = args.numRepetitions;
+        this._includeAttachments = args.includeAttachments;
     }
     _getExperiment() {
         if (!this._experiment) {
@@ -289,6 +299,7 @@ class _ExperimentManager {
             client: this.client,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
+            includeAttachments: this._includeAttachments,
         });
     }
     async withPredictions(target, options) {
@@ -303,6 +314,7 @@ class _ExperimentManager {
                     yield pred.run;
                 }
             })(),
+            includeAttachments: this._includeAttachments,
         });
     }
     async withEvaluators(evaluators, options) {
@@ -325,6 +337,7 @@ class _ExperimentManager {
                 }
             })(),
             summaryResults: this._summaryResults,
+            includeAttachments: this._includeAttachments,
         });
     }
     async withSummaryEvaluators(summaryEvaluators) {
@@ -338,6 +351,7 @@ class _ExperimentManager {
             _runsArray: this._runsArray,
             evaluationResults: this._evaluationResults,
             summaryResults: aggregateFeedbackGen,
+            includeAttachments: this._includeAttachments,
         });
     }
     async *getResults() {
@@ -388,7 +402,7 @@ class _ExperimentManager {
         const examples = await this.getExamples();
         if (maxConcurrency === 0) {
             for (const example of examples) {
-                yield await _forward(target, example, this.experimentName, this._metadata, this.client);
+                yield await _forward(target, example, this.experimentName, this._metadata, this.client, this._includeAttachments);
             }
         }
         else {
@@ -397,7 +411,7 @@ class _ExperimentManager {
             });
             const futures = [];
             for await (const example of examples) {
-                futures.push(caller.call(_forward, target, example, this.experimentName, this._metadata, this.client));
+                futures.push(caller.call(_forward, target, example, this.experimentName, this._metadata, this.client, this._includeAttachments));
             }
             for await (const future of futures) {
                 yield future;
@@ -659,6 +673,7 @@ async function _evaluate(target, fields) {
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
         numRepetitions: fields.numRepetitions ?? 1,
+        includeAttachments: standardFields.includeAttachments,
     }).start();
     if (_isCallable(target)) {
         manager = await manager.withPredictions(target, {
@@ -678,7 +693,7 @@ async function _evaluate(target, fields) {
     await results.processData(manager);
     return results;
 }
-async function _forward(fn, example, experimentName, metadata, client) {
+async function _forward(fn, example, experimentName, metadata, client, includeAttachments) {
     let run = null;
     const _getRun = (r) => {
         run = r;
@@ -708,16 +723,32 @@ async function _forward(fn, example, experimentName, metadata, client) {
                 // no-op
             }
             // Issue with retrieving LangChain callbacks, rely on interop
-            if (langChainCallbacks === undefined) {
+            if (langChainCallbacks === undefined && !includeAttachments) {
                 return await fn.invoke(inputs);
             }
-            else {
+            else if (langChainCallbacks === undefined && includeAttachments) {
+                return await fn.invoke(inputs, {
+                    attachments: example.attachments,
+                });
+            }
+            else if (!includeAttachments) {
                 return await fn.invoke(inputs, { callbacks: langChainCallbacks });
             }
+            else {
+                return await fn.invoke(inputs, {
+                    attachments: example.attachments,
+                    callbacks: langChainCallbacks,
+                });
+            }
         }, options)
         : (0, traceable_js_1.traceable)(fn, options);
     try {
-        await wrappedFn(example.inputs);
+        if (includeAttachments && !("invoke" in fn)) {
+            await wrappedFn(example.inputs, { attachments: example.attachments });
+        }
+        else {
+            await wrappedFn(example.inputs);
+        }
     }
     catch (e) {
         console.error(`Error running target function: ${e}`);
@@ -747,11 +778,13 @@ function _resolveData(data, options) {
     if (typeof data === "string" && isUUID) {
         return options.client.listExamples({
             datasetId: data,
+            includeAttachments: options.includeAttachments,
         });
     }
     if (typeof data === "string") {
         return options.client.listExamples({
             datasetName: data,
+            includeAttachments: options.includeAttachments,
         });
     }
     return data;

package/dist/evaluation/_runner.d.ts

@@ -1,11 +1,15 @@
 import { Client } from "../index.js";
-import { Example, KVMap, Run, TracerSession } from "../schemas.js";
+import { AttachmentInfo, Example, KVMap, Run, TracerSession } from "../schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluator.js";
 import { ComparisonEvaluationResults, ComparativeEvaluator } from "./evaluate_comparative.js";
-type StandardTargetT<TInput = any, TOutput = KVMap> = ((input: TInput, config?: KVMap) => Promise<TOutput>) | ((input: TInput, config?: KVMap) => TOutput) | {
-    invoke: (input: TInput, config?: KVMap) => TOutput;
+export type TargetConfigT = KVMap & {
+    attachments?: Record<string, AttachmentInfo>;
+    callbacks?: any;
+};
+type StandardTargetT<TInput = any, TOutput = KVMap> = ((input: TInput, config?: TargetConfigT) => Promise<TOutput>) | ((input: TInput, config?: TargetConfigT) => TOutput) | {
+    invoke: (input: TInput, config?: TargetConfigT) => TOutput;
 } | {
-    invoke: (input: TInput, config?: KVMap) => Promise<TOutput>;
+    invoke: (input: TInput, config?: TargetConfigT) => Promise<TOutput>;
 };
 type ComparativeTargetT = Array<string> | Array<Promise<ExperimentResults> | ExperimentResults>;
 export type TargetT<TInput = any, TOutput = KVMap> = StandardTargetT<TInput, TOutput> | ComparativeTargetT;
@@ -39,12 +43,14 @@ export type EvaluatorT = DeprecatedRunEvaluator | DeprecatedFunctionEvaluator |
     inputs: Record<string, any>;
     outputs: Record<string, any>;
     referenceOutputs?: Record<string, any>;
+    attachments?: Record<string, any>;
 }) => EvaluationResult | EvaluationResults) | ((args: {
     run: Run;
     example: Example;
     inputs: Record<string, any>;
     outputs: Record<string, any>;
     referenceOutputs?: Record<string, any>;
+    attachments?: Record<string, any>;
 }) => Promise<EvaluationResult | EvaluationResults>);
 interface _ForwardResults {
     run: Run;
@@ -61,6 +67,7 @@ interface _ExperimentManagerArgs {
     examples?: Example[];
     numRepetitions?: number;
     _runsArray?: Run[];
+    includeAttachments?: boolean;
 }
 type BaseEvaluateOptions = {
     /**
@@ -110,6 +117,11 @@ export interface EvaluateOptions extends BaseEvaluateOptions {
      * examples, or a generator of examples.
      */
     data: DataT;
+    /**
+     * Whether to use attachments for the experiment.
+     * @default false
+     */
+    includeAttachments?: boolean;
 }
 export interface ComparativeEvaluateOptions extends BaseEvaluateOptions {
     /**
@@ -153,6 +165,7 @@ export declare class _ExperimentManager {
     _experimentName: string;
     _metadata: KVMap;
     _description?: string;
+    _includeAttachments?: boolean;
     get experimentName(): string;
     getExamples(): Promise<Array<Example>>;
     setExamples(examples: Example[]): void;

package/dist/evaluation/_runner.js

@@ -34,7 +34,10 @@ export class _ExperimentManager {
             if (!this._data) {
                 throw new Error("Data not provided in this experiment.");
             }
-            const unresolvedData = _resolveData(this._data, { client: this.client });
+            const unresolvedData = _resolveData(this._data, {
+                client: this.client,
+                includeAttachments: this._includeAttachments,
+            });
             if (!this._examples) {
                 this._examples = [];
             }
@@ -165,6 +168,12 @@ export class _ExperimentManager {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "_includeAttachments", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.client = args.client ?? new Client();
         if (!args.experiment) {
             this._experimentName = randomName();
@@ -198,6 +207,7 @@ export class _ExperimentManager {
         this._evaluationResults = args.evaluationResults;
         this._summaryResults = args.summaryResults;
         this._numRepetitions = args.numRepetitions;
+        this._includeAttachments = args.includeAttachments;
     }
     _getExperiment() {
         if (!this._experiment) {
@@ -285,6 +295,7 @@ export class _ExperimentManager {
             client: this.client,
             evaluationResults: this._evaluationResults,
             summaryResults: this._summaryResults,
+            includeAttachments: this._includeAttachments,
         });
     }
     async withPredictions(target, options) {
@@ -299,6 +310,7 @@ export class _ExperimentManager {
                     yield pred.run;
                 }
             })(),
+            includeAttachments: this._includeAttachments,
         });
     }
     async withEvaluators(evaluators, options) {
@@ -321,6 +333,7 @@ export class _ExperimentManager {
                 }
             })(),
             summaryResults: this._summaryResults,
+            includeAttachments: this._includeAttachments,
         });
     }
     async withSummaryEvaluators(summaryEvaluators) {
@@ -334,6 +347,7 @@ export class _ExperimentManager {
             _runsArray: this._runsArray,
             evaluationResults: this._evaluationResults,
             summaryResults: aggregateFeedbackGen,
+            includeAttachments: this._includeAttachments,
         });
     }
     async *getResults() {
@@ -384,7 +398,7 @@ export class _ExperimentManager {
         const examples = await this.getExamples();
         if (maxConcurrency === 0) {
             for (const example of examples) {
-                yield await _forward(target, example, this.experimentName, this._metadata, this.client);
+                yield await _forward(target, example, this.experimentName, this._metadata, this.client, this._includeAttachments);
             }
         }
         else {
@@ -393,7 +407,7 @@ export class _ExperimentManager {
             });
             const futures = [];
             for await (const example of examples) {
-                futures.push(caller.call(_forward, target, example, this.experimentName, this._metadata, this.client));
+                futures.push(caller.call(_forward, target, example, this.experimentName, this._metadata, this.client, this._includeAttachments));
             }
             for await (const future of futures) {
                 yield future;
@@ -654,6 +668,7 @@ async function _evaluate(target, fields) {
         experiment: experiment_ ?? fields.experimentPrefix,
         runs: newRuns ?? undefined,
         numRepetitions: fields.numRepetitions ?? 1,
+        includeAttachments: standardFields.includeAttachments,
     }).start();
     if (_isCallable(target)) {
         manager = await manager.withPredictions(target, {
@@ -673,7 +688,7 @@ async function _evaluate(target, fields) {
     await results.processData(manager);
     return results;
 }
-async function _forward(fn, example, experimentName, metadata, client) {
+async function _forward(fn, example, experimentName, metadata, client, includeAttachments) {
     let run = null;
     const _getRun = (r) => {
         run = r;
@@ -703,16 +718,32 @@ async function _forward(fn, example, experimentName, metadata, client) {
                 // no-op
             }
             // Issue with retrieving LangChain callbacks, rely on interop
-            if (langChainCallbacks === undefined) {
+            if (langChainCallbacks === undefined && !includeAttachments) {
                 return await fn.invoke(inputs);
             }
-            else {
+            else if (langChainCallbacks === undefined && includeAttachments) {
+                return await fn.invoke(inputs, {
+                    attachments: example.attachments,
+                });
+            }
+            else if (!includeAttachments) {
                 return await fn.invoke(inputs, { callbacks: langChainCallbacks });
             }
+            else {
+                return await fn.invoke(inputs, {
+                    attachments: example.attachments,
+                    callbacks: langChainCallbacks,
+                });
+            }
         }, options)
         : traceable(fn, options);
     try {
-        await wrappedFn(example.inputs);
+        if (includeAttachments && !("invoke" in fn)) {
+            await wrappedFn(example.inputs, { attachments: example.attachments });
+        }
+        else {
+            await wrappedFn(example.inputs);
+        }
     }
     catch (e) {
         console.error(`Error running target function: ${e}`);
@@ -742,11 +773,13 @@ function _resolveData(data, options) {
     if (typeof data === "string" && isUUID) {
         return options.client.listExamples({
             datasetId: data,
+            includeAttachments: options.includeAttachments,
         });
     }
     if (typeof data === "string") {
         return options.client.listExamples({
             datasetName: data,
+            includeAttachments: options.includeAttachments,
         });
     }
     return data;

package/dist/evaluation/evaluator.cjs

@@ -23,6 +23,7 @@ class DynamicRunEvaluator {
                 inputs: example?.inputs,
                 outputs: run?.outputs,
                 referenceOutputs: example?.outputs,
+                attachments: example?.attachments,
             }, example);
         });
     }

package/dist/evaluation/evaluator.js

@@ -20,6 +20,7 @@ export class DynamicRunEvaluator {
                 inputs: example?.inputs,
                 outputs: run?.outputs,
                 referenceOutputs: example?.outputs,
+                attachments: example?.attachments,
             }, example);
         });
     }

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.2.11";
+exports.__version__ = "0.2.13";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig, type LangSmithTracingClientInterface, } from
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.2.11";
+export declare const __version__ = "0.2.13";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client, } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.2.11";
+export const __version__ = "0.2.13";

package/dist/schemas.d.ts

@@ -32,8 +32,18 @@ export interface BaseExample {
     metadata?: KVMap;
     source_run_id?: string;
 }
+export interface AttachmentInfo {
+    presigned_url: string;
+}
 export type AttachmentData = Uint8Array | ArrayBuffer;
-export type Attachments = Record<string, [string, AttachmentData]>;
+export type AttachmentDescription = {
+    mimeType: string;
+    data: AttachmentData;
+};
+export type Attachments = Record<string, [
+    string,
+    AttachmentData
+] | AttachmentDescription>;
 /**
  * A run can represent either a trace (root run)
  * or a child run (~span).
@@ -177,12 +187,50 @@ export interface ExampleCreate extends BaseExample {
     created_at?: string;
     split?: string | string[];
 }
+export interface ExampleUploadWithAttachments {
+    id?: string;
+    inputs: KVMap;
+    outputs?: KVMap;
+    metadata?: KVMap;
+    split?: string | string[];
+    attachments?: Attachments;
+    created_at?: string;
+}
+export interface ExampleUpdateWithAttachments {
+    id: string;
+    inputs?: KVMap;
+    outputs?: KVMap;
+    metadata?: KVMap;
+    split?: string | string[];
+    attachments?: Attachments;
+    attachments_operations?: KVMap;
+}
+export interface UploadExamplesResponse {
+    count: number;
+    example_ids: string[];
+}
+export interface UpdateExamplesResponse extends UploadExamplesResponse {
+}
 export interface Example extends BaseExample {
+    id: string;
+    created_at: string;
+    modified_at?: string;
+    source_run_id?: string;
+    runs: Run[];
+    attachments?: Record<string, AttachmentInfo>;
+    split?: string | string[];
+}
+interface RawAttachmentInfo {
+    presigned_url: string;
+    s3_url: string;
+}
+export interface RawExample extends BaseExample {
     id: string;
     created_at: string;
     modified_at: string;
     source_run_id?: string;
     runs: Run[];
+    attachment_urls?: Record<string, RawAttachmentInfo>;
 }
 export interface ExampleUpdate {
     dataset_id?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [