langsmith 0.1.57 → 0.1.59

package/dist/client.cjs

@@ -2140,6 +2140,145 @@ class Client {
         const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
+    /**
+     * API for managing annotation queues
+     */
+    /**
+     * List the annotation queues on the LangSmith API.
+     * @param options - The options for listing annotation queues
+     * @param options.queueIds - The IDs of the queues to filter by
+     * @param options.name - The name of the queue to filter by
+     * @param options.nameContains - The substring that the queue name should contain
+     * @param options.limit - The maximum number of queues to return
+     * @returns An iterator of AnnotationQueue objects
+     */
+    async *listAnnotationQueues(options = {}) {
+        const { queueIds, name, nameContains, limit } = options;
+        const params = new URLSearchParams();
+        if (queueIds) {
+            queueIds.forEach((id, i) => {
+                (0, _uuid_js_1.assertUuid)(id, `queueIds[${i}]`);
+                params.append("ids", id);
+            });
+        }
+        if (name)
+            params.append("name", name);
+        if (nameContains)
+            params.append("name_contains", nameContains);
+        params.append("limit", (limit !== undefined ? Math.min(limit, 100) : 100).toString());
+        let count = 0;
+        for await (const queues of this._getPaginated("/annotation-queues", params)) {
+            yield* queues;
+            count++;
+            if (limit !== undefined && count >= limit)
+                break;
+        }
+    }
+    /**
+     * Create an annotation queue on the LangSmith API.
+     * @param options - The options for creating an annotation queue
+     * @param options.name - The name of the annotation queue
+     * @param options.description - The description of the annotation queue
+     * @param options.queueId - The ID of the annotation queue
+     * @returns The created AnnotationQueue object
+     */
+    async createAnnotationQueue(options) {
+        const { name, description, queueId } = options;
+        const body = {
+            name,
+            description,
+            id: queueId || uuid.v4(),
+        };
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/annotation-queues`, {
+            method: "POST",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify(Object.fromEntries(Object.entries(body).filter(([_, v]) => v !== undefined))),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "create annotation queue");
+        const data = await response.json();
+        return data;
+    }
+    /**
+     * Read an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to read
+     * @returns The AnnotationQueue object
+     */
+    async readAnnotationQueue(queueId) {
+        // TODO: Replace when actual endpoint is added
+        const queueIteratorResult = await this.listAnnotationQueues({
+            queueIds: [queueId],
+        }).next();
+        if (queueIteratorResult.done) {
+            throw new Error(`Annotation queue with ID ${queueId} not found`);
+        }
+        return queueIteratorResult.value;
+    }
+    /**
+     * Update an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to update
+     * @param options - The options for updating the annotation queue
+     * @param options.name - The new name for the annotation queue
+     * @param options.description - The new description for the annotation queue
+     */
+    async updateAnnotationQueue(queueId, options) {
+        const { name, description } = options;
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/annotation-queues/${(0, _uuid_js_1.assertUuid)(queueId, "queueId")}`, {
+            method: "PATCH",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify({ name, description }),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "update annotation queue");
+    }
+    /**
+     * Delete an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to delete
+     */
+    async deleteAnnotationQueue(queueId) {
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/annotation-queues/${(0, _uuid_js_1.assertUuid)(queueId, "queueId")}`, {
+            method: "DELETE",
+            headers: { ...this.headers, Accept: "application/json" },
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "delete annotation queue");
+    }
+    /**
+     * Add runs to an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue
+     * @param runIds - The IDs of the runs to be added to the annotation queue
+     */
+    async addRunsToAnnotationQueue(queueId, runIds) {
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}/annotation-queues/${(0, _uuid_js_1.assertUuid)(queueId, "queueId")}/runs`, {
+            method: "POST",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify(runIds.map((id, i) => (0, _uuid_js_1.assertUuid)(id, `runIds[${i}]`).toString())),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "add runs to annotation queue");
+    }
+    /**
+     * Get a run from an annotation queue at the specified index.
+     * @param queueId - The ID of the annotation queue
+     * @param index - The index of the run to retrieve
+     * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+     * @throws {Error} If the run is not found at the given index or for other API-related errors
+     */
+    async getRunFromAnnotationQueue(queueId, index) {
+        const baseUrl = `/annotation-queues/${(0, _uuid_js_1.assertUuid)(queueId, "queueId")}/run`;
+        const response = await this.caller.call((0, fetch_js_1._getFetchImplementation)(), `${this.apiUrl}${baseUrl}/${index}`, {
+            method: "GET",
+            headers: this.headers,
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await (0, error_js_1.raiseForStatus)(response, "get run from annotation queue");
+        return await response.json();
+    }
     async _currentTenantIsOwner(owner) {
         const settings = await this._getSettings();
         return owner == "-" || settings.tenant_handle === owner;
@@ -2215,7 +2354,7 @@ class Client {
         return this._likeOrUnlikePrompt(promptIdentifier, false);
     }
     async *listCommits(promptOwnerAndName) {
-        for await (const commits of this._getPaginated(`/commits/${promptOwnerAndName}/`, {}, (res) => res.commits)) {
+        for await (const commits of this._getPaginated(`/commits/${promptOwnerAndName}/`, new URLSearchParams(), (res) => res.commits)) {
             yield* commits;
         }
     }

package/dist/client.d.ts

@@ -1,5 +1,5 @@
 import { AsyncCallerParams } from "./utils/async_caller.js";
-import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType } from "./schemas.js";
+import { ComparativeExperiment, DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, ExampleUpdateWithId, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, LangSmithSettings, LikePromptResponse, Prompt, PromptCommit, PromptSortField, Run, RunCreate, RunUpdate, ScoreType, ExampleSearch, TimeDelta, TracerSession, TracerSessionResult, ValueType, AnnotationQueue, RunWithAnnotationQueueInfo } from "./schemas.js";
 import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
 export interface ClientConfig {
     apiUrl?: string;
@@ -601,6 +601,73 @@ export declare class Client {
     logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResults, run?: Run, sourceInfo?: {
         [key: string]: any;
     }): Promise<EvaluationResult[]>;
+    /**
+     * API for managing annotation queues
+     */
+    /**
+     * List the annotation queues on the LangSmith API.
+     * @param options - The options for listing annotation queues
+     * @param options.queueIds - The IDs of the queues to filter by
+     * @param options.name - The name of the queue to filter by
+     * @param options.nameContains - The substring that the queue name should contain
+     * @param options.limit - The maximum number of queues to return
+     * @returns An iterator of AnnotationQueue objects
+     */
+    listAnnotationQueues(options?: {
+        queueIds?: string[];
+        name?: string;
+        nameContains?: string;
+        limit?: number;
+    }): AsyncIterableIterator<AnnotationQueue>;
+    /**
+     * Create an annotation queue on the LangSmith API.
+     * @param options - The options for creating an annotation queue
+     * @param options.name - The name of the annotation queue
+     * @param options.description - The description of the annotation queue
+     * @param options.queueId - The ID of the annotation queue
+     * @returns The created AnnotationQueue object
+     */
+    createAnnotationQueue(options: {
+        name: string;
+        description?: string;
+        queueId?: string;
+    }): Promise<AnnotationQueue>;
+    /**
+     * Read an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to read
+     * @returns The AnnotationQueue object
+     */
+    readAnnotationQueue(queueId: string): Promise<AnnotationQueue>;
+    /**
+     * Update an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to update
+     * @param options - The options for updating the annotation queue
+     * @param options.name - The new name for the annotation queue
+     * @param options.description - The new description for the annotation queue
+     */
+    updateAnnotationQueue(queueId: string, options: {
+        name: string;
+        description?: string;
+    }): Promise<void>;
+    /**
+     * Delete an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to delete
+     */
+    deleteAnnotationQueue(queueId: string): Promise<void>;
+    /**
+     * Add runs to an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue
+     * @param runIds - The IDs of the runs to be added to the annotation queue
+     */
+    addRunsToAnnotationQueue(queueId: string, runIds: string[]): Promise<void>;
+    /**
+     * Get a run from an annotation queue at the specified index.
+     * @param queueId - The ID of the annotation queue
+     * @param index - The index of the run to retrieve
+     * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+     * @throws {Error} If the run is not found at the given index or for other API-related errors
+     */
+    getRunFromAnnotationQueue(queueId: string, index: number): Promise<RunWithAnnotationQueueInfo>;
     protected _currentTenantIsOwner(owner: string): Promise<boolean>;
     protected _ownerConflictError(action: string, owner: string): Promise<Error>;
     protected _getLatestCommitHash(promptOwnerAndName: string): Promise<string | undefined>;

package/dist/client.js

@@ -2113,6 +2113,145 @@ export class Client {
         const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
+    /**
+     * API for managing annotation queues
+     */
+    /**
+     * List the annotation queues on the LangSmith API.
+     * @param options - The options for listing annotation queues
+     * @param options.queueIds - The IDs of the queues to filter by
+     * @param options.name - The name of the queue to filter by
+     * @param options.nameContains - The substring that the queue name should contain
+     * @param options.limit - The maximum number of queues to return
+     * @returns An iterator of AnnotationQueue objects
+     */
+    async *listAnnotationQueues(options = {}) {
+        const { queueIds, name, nameContains, limit } = options;
+        const params = new URLSearchParams();
+        if (queueIds) {
+            queueIds.forEach((id, i) => {
+                assertUuid(id, `queueIds[${i}]`);
+                params.append("ids", id);
+            });
+        }
+        if (name)
+            params.append("name", name);
+        if (nameContains)
+            params.append("name_contains", nameContains);
+        params.append("limit", (limit !== undefined ? Math.min(limit, 100) : 100).toString());
+        let count = 0;
+        for await (const queues of this._getPaginated("/annotation-queues", params)) {
+            yield* queues;
+            count++;
+            if (limit !== undefined && count >= limit)
+                break;
+        }
+    }
+    /**
+     * Create an annotation queue on the LangSmith API.
+     * @param options - The options for creating an annotation queue
+     * @param options.name - The name of the annotation queue
+     * @param options.description - The description of the annotation queue
+     * @param options.queueId - The ID of the annotation queue
+     * @returns The created AnnotationQueue object
+     */
+    async createAnnotationQueue(options) {
+        const { name, description, queueId } = options;
+        const body = {
+            name,
+            description,
+            id: queueId || uuid.v4(),
+        };
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues`, {
+            method: "POST",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify(Object.fromEntries(Object.entries(body).filter(([_, v]) => v !== undefined))),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "create annotation queue");
+        const data = await response.json();
+        return data;
+    }
+    /**
+     * Read an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to read
+     * @returns The AnnotationQueue object
+     */
+    async readAnnotationQueue(queueId) {
+        // TODO: Replace when actual endpoint is added
+        const queueIteratorResult = await this.listAnnotationQueues({
+            queueIds: [queueId],
+        }).next();
+        if (queueIteratorResult.done) {
+            throw new Error(`Annotation queue with ID ${queueId} not found`);
+        }
+        return queueIteratorResult.value;
+    }
+    /**
+     * Update an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to update
+     * @param options - The options for updating the annotation queue
+     * @param options.name - The new name for the annotation queue
+     * @param options.description - The new description for the annotation queue
+     */
+    async updateAnnotationQueue(queueId, options) {
+        const { name, description } = options;
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, {
+            method: "PATCH",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify({ name, description }),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "update annotation queue");
+    }
+    /**
+     * Delete an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue to delete
+     */
+    async deleteAnnotationQueue(queueId) {
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, {
+            method: "DELETE",
+            headers: { ...this.headers, Accept: "application/json" },
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "delete annotation queue");
+    }
+    /**
+     * Add runs to an annotation queue with the specified queue ID.
+     * @param queueId - The ID of the annotation queue
+     * @param runIds - The IDs of the runs to be added to the annotation queue
+     */
+    async addRunsToAnnotationQueue(queueId, runIds) {
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`, {
+            method: "POST",
+            headers: { ...this.headers, "Content-Type": "application/json" },
+            body: JSON.stringify(runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString())),
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "add runs to annotation queue");
+    }
+    /**
+     * Get a run from an annotation queue at the specified index.
+     * @param queueId - The ID of the annotation queue
+     * @param index - The index of the run to retrieve
+     * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object
+     * @throws {Error} If the run is not found at the given index or for other API-related errors
+     */
+    async getRunFromAnnotationQueue(queueId, index) {
+        const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`;
+        const response = await this.caller.call(_getFetchImplementation(), `${this.apiUrl}${baseUrl}/${index}`, {
+            method: "GET",
+            headers: this.headers,
+            signal: AbortSignal.timeout(this.timeout_ms),
+            ...this.fetchOptions,
+        });
+        await raiseForStatus(response, "get run from annotation queue");
+        return await response.json();
+    }
     async _currentTenantIsOwner(owner) {
         const settings = await this._getSettings();
         return owner == "-" || settings.tenant_handle === owner;
@@ -2188,7 +2327,7 @@ export class Client {
         return this._likeOrUnlikePrompt(promptIdentifier, false);
     }
     async *listCommits(promptOwnerAndName) {
-        for await (const commits of this._getPaginated(`/commits/${promptOwnerAndName}/`, {}, (res) => res.commits)) {
+        for await (const commits of this._getPaginated(`/commits/${promptOwnerAndName}/`, new URLSearchParams(), (res) => res.commits)) {
             yield* commits;
         }
     }

package/dist/evaluation/_runner.cjs

@@ -421,6 +421,7 @@ class _ExperimentManager {
                             : new Date(example.created_at).toISOString(),
                     },
                     client: fields.client,
+                    tracingEnabled: true,
                 };
                 const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
                 evaluationResults.results.push(...(await fields.client.logEvaluationFeedback(evaluatorResponse, run)));

package/dist/evaluation/_runner.js

@@ -417,6 +417,7 @@ export class _ExperimentManager {
                             : new Date(example.created_at).toISOString(),
                     },
                     client: fields.client,
+                    tracingEnabled: true,
                 };
                 const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
                 evaluationResults.results.push(...(await fields.client.logEvaluationFeedback(evaluatorResponse, run)));

package/dist/index.cjs

@@ -8,4 +8,4 @@ Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () {
 var fetch_js_1 = require("./singletons/fetch.cjs");
 Object.defineProperty(exports, "overrideFetchImplementation", { enumerable: true, get: function () { return fetch_js_1.overrideFetchImplementation; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.57";
+exports.__version__ = "0.1.59";

package/dist/index.d.ts

@@ -2,4 +2,4 @@ export { Client, type ClientConfig } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
-export declare const __version__ = "0.1.57";
+export declare const __version__ = "0.1.59";

package/dist/index.js

@@ -2,4 +2,4 @@ export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.57";
+export const __version__ = "0.1.59";

package/dist/schemas.d.ts

@@ -371,4 +371,24 @@ export interface LangSmithSettings {
     created_at: string;
     tenant_handle?: string;
 }
+export interface AnnotationQueue {
+    /** The unique identifier of the annotation queue. */
+    id: string;
+    /** The name of the annotation queue. */
+    name: string;
+    /** An optional description of the annotation queue. */
+    description?: string;
+    /** The timestamp when the annotation queue was created. */
+    created_at: string;
+    /** The timestamp when the annotation queue was last updated. */
+    updated_at: string;
+    /** The ID of the tenant associated with the annotation queue. */
+    tenant_id: string;
+}
+export interface RunWithAnnotationQueueInfo extends BaseRun {
+    /** The last time this run was reviewed. */
+    last_reviewed_time?: string;
+    /** The time this run was added to the queue. */
+    added_at?: string;
+}
 export {};

package/dist/utils/_uuid.cjs

@@ -25,9 +25,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.assertUuid = void 0;
 const uuid = __importStar(require("uuid"));
-function assertUuid(str) {
+function assertUuid(str, which) {
     if (!uuid.validate(str)) {
-        throw new Error(`Invalid UUID: ${str}`);
+        const msg = which !== undefined
+            ? `Invalid UUID for ${which}: ${str}`
+            : `Invalid UUID: ${str}`;
+        throw new Error(msg);
     }
+    return str;
 }
 exports.assertUuid = assertUuid;

package/dist/utils/_uuid.d.ts

@@ -1 +1 @@
-export declare function assertUuid(str: string): void;
+export declare function assertUuid(str: string, which?: string): string;

package/dist/utils/_uuid.js

@@ -1,6 +1,10 @@
 import * as uuid from "uuid";
-export function assertUuid(str) {
+export function assertUuid(str, which) {
     if (!uuid.validate(str)) {
-        throw new Error(`Invalid UUID: ${str}`);
+        const msg = which !== undefined
+            ? `Invalid UUID for ${which}: ${str}`
+            : `Invalid UUID: ${str}`;
+        throw new Error(msg);
     }
+    return str;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.57",
+  "version": "0.1.59",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [