langsmith 0.0.6 → 0.0.8

package/README.md

@@ -14,7 +14,7 @@ LangSmith helps you and your team develop and evaluate language models and intel
 
 A typical workflow looks like:
 
-1. Set up an account with LangSmith or host your [local server](https://docs.smith.langchain.com/docs/additional-resources/local_installation).
+1. Set up an account with LangSmith.
 2. Log traces.
 3. Debug, Create Datasets, and Evaluate Runs.
 
@@ -24,7 +24,7 @@ We'll walk through these steps in more detail below.
 
 Sign up for [LangSmith](https://smith.langchain.com/) using your GitHub, Discord accounts, or an email address and password. If you sign up with an email, make sure to verify your email address before logging in.
 
-Then, create a unique API key on the [Settings Page](https://smith.langchain.com/settings), which is found in the menu at the top right corner of the page.
+Then, create a unique API key on the [Settings Page](https://smith.langchain.com/settings).
 
 Note: Save the API Key in a secure location. It will not be shown again.
 
@@ -211,7 +211,9 @@ for (const run of runs) {
 
 # Evaluating Runs
 
-You can run evaluations directly using the LangSmith client.
+Check out the [LangSmith Testing & Evaluation dos](https://docs.smith.langchain.com/docs/evaluation/) for up-to-date workflows.
+
+For generating automated feedback on individual runs, you can run evaluations directly using the LangSmith client.
 
 ```ts
 import { StringEvaluator } from "langsmith/evaluation";

package/dist/cli/main.cjs

@@ -182,6 +182,8 @@ class SmithCommand {
         return new SmithCommand({ dockerComposeCommand });
     }
     async start(args) {
+        console.info("BY USING THIS SOFTWARE YOU AGREE TO THE TERMS OF SERVICE AT:");
+        console.info("https://smith.langchain.com/terms-of-service.pdf");
         if (args.dev) {
             (0, env_js_1.setEnvironmentVariable)("_LANGSMITH_IMAGE_PREFIX", "rc-");
         }

package/dist/cli/main.js

@@ -156,6 +156,8 @@ class SmithCommand {
         return new SmithCommand({ dockerComposeCommand });
     }
     async start(args) {
+        console.info("BY USING THIS SOFTWARE YOU AGREE TO THE TERMS OF SERVICE AT:");
+        console.info("https://smith.langchain.com/terms-of-service.pdf");
         if (args.dev) {
             setEnvironmentVariable("_LANGSMITH_IMAGE_PREFIX", "rc-");
         }

package/dist/cli/main.ts

@@ -169,6 +169,10 @@ class SmithCommand {
   }
 
   async start(args: any) {
+    console.info(
+      "BY USING THIS SOFTWARE YOU AGREE TO THE TERMS OF SERVICE AT:"
+    );
+    console.info("https://smith.langchain.com/terms-of-service.pdf");
     if (args.dev) {
       setEnvironmentVariable("_LANGSMITH_IMAGE_PREFIX", "rc-");
     }

package/dist/client.cjs

@@ -41,6 +41,13 @@ const raiseForStatus = async (response, operation) => {
         throw new Error(`Failed to ${operation}: ${response.status} ${response.statusText} ${body}`);
     }
 };
+async function toArray(iterable) {
+    const result = [];
+    for await (const item of iterable) {
+        result.push(item);
+    }
+    return result;
+}
 class Client {
     constructor(config = {}) {
         Object.defineProperty(this, "apiKey", {
@@ -75,9 +82,12 @@ class Client {
         this.caller = new async_caller_js_1.AsyncCaller(config.callerOptions ?? {});
     }
     static getDefaultClientConfig() {
+        const apiKey = (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_API_KEY");
+        const apiUrl = (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_ENDPOINT") ??
+            (apiKey ? "https://api.smith.langchain.com" : "http://localhost:1984");
         return {
-            apiUrl: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_ENDPOINT") ?? "http://localhost:1984",
-            apiKey: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_API_KEY"),
+            apiUrl: apiUrl,
+            apiKey: apiKey,
         };
     }
     validateApiKeyIfHosted() {
@@ -106,6 +116,32 @@ class Client {
         }
         return response.json();
     }
+    async *_getPaginated(path, queryParams = new URLSearchParams()) {
+        let offset = Number(queryParams.get("offset")) || 0;
+        const limit = Number(queryParams.get("limit")) || 100;
+        while (true) {
+            queryParams.set("offset", String(offset));
+            queryParams.set("limit", String(limit));
+            const url = `${this.apiUrl}${path}?${queryParams}`;
+            const response = await this.caller.call(fetch, url, {
+                method: "GET",
+                headers: this.headers,
+                signal: AbortSignal.timeout(this.timeout_ms),
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch ${path}: ${response.status} ${response.statusText}`);
+            }
+            const items = await response.json();
+            if (items.length === 0) {
+                break;
+            }
+            yield items;
+            if (items.length < limit) {
+                break;
+            }
+            offset += items.length;
+        }
+    }
     async createRun(run) {
         const headers = { ...this.headers, "Content-Type": "application/json" };
         const extra = run.extra ?? {};
@@ -149,7 +185,7 @@ class Client {
         return run;
     }
     async _loadChildRuns(run) {
-        const childRuns = await this.listRuns({ id: run.child_run_ids });
+        const childRuns = await toArray(this.listRuns({ id: run.child_run_ids }));
         const treemap = {};
         const runs = {};
         childRuns.sort((a, b) => a.execution_order - b.execution_order);
@@ -172,7 +208,7 @@ class Client {
         }
         return run;
     }
-    async listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }) {
+    async *listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }) {
         const queryParams = new URLSearchParams();
         let projectId_ = projectId;
         if (projectName) {
@@ -228,7 +264,9 @@ class Client {
         if (orderBy !== undefined) {
             orderBy.map((order) => queryParams.append("order_by", order));
         }
-        return this._get("/runs", queryParams);
+        for await (const runs of this._getPaginated("/runs", queryParams)) {
+            yield* runs;
+        }
     }
     async deleteRun(runId) {
         const response = await this.caller.call(fetch, `${this.apiUrl}/runs/${runId}`, {
@@ -287,8 +325,10 @@ class Client {
         }
         return result;
     }
-    async listProjects() {
-        return this._get("/sessions");
+    async *listProjects() {
+        for await (const projects of this._getPaginated("/sessions")) {
+            yield* projects;
+        }
     }
     async deleteProject({ projectId, projectName, }) {
         let projectId_;
@@ -399,17 +439,15 @@ class Client {
         }
         return result;
     }
-    async listDatasets({ limit = 100, offset = 0, } = {}) {
+    async *listDatasets({ limit = 100, offset = 0, } = {}) {
         const path = "/datasets";
         const params = new URLSearchParams({
             limit: limit.toString(),
             offset: offset.toString(),
         });
-        const response = await this._get(path, params);
-        if (!Array.isArray(response)) {
-            throw new Error(`Expected ${path} to return an array, but got ${response}`);
+        for await (const datasets of this._getPaginated(path, params)) {
+            yield* datasets;
         }
-        return response;
     }
     async deleteDataset({ datasetId, datasetName, }) {
         let path = "/datasets";
@@ -472,7 +510,7 @@ class Client {
         const path = `/examples/${exampleId}`;
         return await this._get(path);
     }
-    async listExamples({ datasetId, datasetName, limit, offset, } = {}) {
+    async *listExamples({ datasetId, datasetName, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -487,15 +525,10 @@ class Client {
         else {
             throw new Error("Must provide a datasetName or datasetId");
         }
-        const response = await this._get("/examples", new URLSearchParams({
-            dataset: datasetId_,
-            limit: limit?.toString() ?? "100",
-            offset: offset?.toString() ?? "0",
-        }));
-        if (!Array.isArray(response)) {
-            throw new Error(`Expected /examples to return an array, but got ${response}`);
+        const params = new URLSearchParams({ dataset: datasetId_ });
+        for await (const examples of this._getPaginated("/examples", params)) {
+            yield* examples;
         }
-        return response;
     }
     async deleteExample(exampleId) {
         const path = `/examples/${exampleId}`;
@@ -602,19 +635,14 @@ class Client {
         }
         await response.json();
     }
-    async listFeedback({ runIds, limit, offset, } = {}) {
+    async *listFeedback({ runIds, } = {}) {
         const queryParams = new URLSearchParams();
         if (runIds) {
             queryParams.append("run", runIds.join(","));
         }
-        if (limit !== undefined) {
-            queryParams.append("limit", limit.toString());
+        for await (const feedbacks of this._getPaginated("/feedback", queryParams)) {
+            yield* feedbacks;
         }
-        if (offset !== undefined) {
-            queryParams.append("offset", offset.toString());
-        }
-        const response = await this._get("/feedback", queryParams);
-        return response;
     }
 }
 exports.Client = Client;

package/dist/client.d.ts

@@ -64,13 +64,14 @@ export declare class Client {
     private validateApiKeyIfHosted;
     private get headers();
     private _get;
+    private _getPaginated;
     createRun(run: CreateRunParams): Promise<void>;
     updateRun(runId: string, run: RunUpdate): Promise<void>;
     readRun(runId: string, { loadChildRuns }?: {
         loadChildRuns: boolean;
     }): Promise<Run>;
     private _loadChildRuns;
-    listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }: ListRunsParams): Promise<Run[]>;
+    listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }: ListRunsParams): AsyncIterable<Run>;
     deleteRun(runId: string): Promise<void>;
     createProject({ projectName, projectExtra, upsert, }: {
         projectName: string;
@@ -81,7 +82,7 @@ export declare class Client {
         projectId?: string;
         projectName?: string;
     }): Promise<TracerSessionResult>;
-    listProjects(): Promise<TracerSession[]>;
+    listProjects(): AsyncIterable<TracerSession>;
     deleteProject({ projectId, projectName, }: {
         projectId?: string;
         projectName?: string;
@@ -98,7 +99,7 @@ export declare class Client {
     listDatasets({ limit, offset, }?: {
         limit?: number;
         offset?: number;
-    }): Promise<Dataset[]>;
+    }): AsyncIterable<Dataset>;
     deleteDataset({ datasetId, datasetName, }: {
         datasetId?: string;
         datasetName?: string;
@@ -109,12 +110,10 @@ export declare class Client {
         createdAt?: Date;
     }): Promise<Example>;
     readExample(exampleId: string): Promise<Example>;
-    listExamples({ datasetId, datasetName, limit, offset, }?: {
+    listExamples({ datasetId, datasetName, }?: {
         datasetId?: string;
         datasetName?: string;
-        limit?: number;
-        offset?: number;
-    }): Promise<Example[]>;
+    }): AsyncIterable<Example>;
     deleteExample(exampleId: string): Promise<void>;
     updateExample(exampleId: string, update: ExampleUpdate): Promise<object>;
     evaluateRun(run: Run | string, evaluator: RunEvaluator, { sourceInfo, loadChildRuns, }?: {
@@ -131,10 +130,8 @@ export declare class Client {
     }): Promise<Feedback>;
     readFeedback(feedbackId: string): Promise<Feedback>;
     deleteFeedback(feedbackId: string): Promise<void>;
-    listFeedback({ runIds, limit, offset, }?: {
+    listFeedback({ runIds, }?: {
         runIds?: string[];
-        limit?: number;
-        offset?: number;
-    }): Promise<Feedback[]>;
+    }): AsyncIterable<Feedback>;
 }
 export {};

package/dist/client.js

@@ -15,6 +15,13 @@ const raiseForStatus = async (response, operation) => {
         throw new Error(`Failed to ${operation}: ${response.status} ${response.statusText} ${body}`);
     }
 };
+async function toArray(iterable) {
+    const result = [];
+    for await (const item of iterable) {
+        result.push(item);
+    }
+    return result;
+}
 export class Client {
     constructor(config = {}) {
         Object.defineProperty(this, "apiKey", {
@@ -49,9 +56,12 @@ export class Client {
         this.caller = new AsyncCaller(config.callerOptions ?? {});
     }
     static getDefaultClientConfig() {
+        const apiKey = getEnvironmentVariable("LANGCHAIN_API_KEY");
+        const apiUrl = getEnvironmentVariable("LANGCHAIN_ENDPOINT") ??
+            (apiKey ? "https://api.smith.langchain.com" : "http://localhost:1984");
         return {
-            apiUrl: getEnvironmentVariable("LANGCHAIN_ENDPOINT") ?? "http://localhost:1984",
-            apiKey: getEnvironmentVariable("LANGCHAIN_API_KEY"),
+            apiUrl: apiUrl,
+            apiKey: apiKey,
         };
     }
     validateApiKeyIfHosted() {
@@ -80,6 +90,32 @@ export class Client {
         }
         return response.json();
     }
+    async *_getPaginated(path, queryParams = new URLSearchParams()) {
+        let offset = Number(queryParams.get("offset")) || 0;
+        const limit = Number(queryParams.get("limit")) || 100;
+        while (true) {
+            queryParams.set("offset", String(offset));
+            queryParams.set("limit", String(limit));
+            const url = `${this.apiUrl}${path}?${queryParams}`;
+            const response = await this.caller.call(fetch, url, {
+                method: "GET",
+                headers: this.headers,
+                signal: AbortSignal.timeout(this.timeout_ms),
+            });
+            if (!response.ok) {
+                throw new Error(`Failed to fetch ${path}: ${response.status} ${response.statusText}`);
+            }
+            const items = await response.json();
+            if (items.length === 0) {
+                break;
+            }
+            yield items;
+            if (items.length < limit) {
+                break;
+            }
+            offset += items.length;
+        }
+    }
     async createRun(run) {
         const headers = { ...this.headers, "Content-Type": "application/json" };
         const extra = run.extra ?? {};
@@ -123,7 +159,7 @@ export class Client {
         return run;
     }
     async _loadChildRuns(run) {
-        const childRuns = await this.listRuns({ id: run.child_run_ids });
+        const childRuns = await toArray(this.listRuns({ id: run.child_run_ids }));
         const treemap = {};
         const runs = {};
         childRuns.sort((a, b) => a.execution_order - b.execution_order);
@@ -146,7 +182,7 @@ export class Client {
         }
         return run;
     }
-    async listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }) {
+    async *listRuns({ projectId, projectName, parentRunId, referenceExampleId, datasetId, startTime, endTime, executionOrder, runType, error, id, limit, offset, query, filter, orderBy, }) {
         const queryParams = new URLSearchParams();
         let projectId_ = projectId;
         if (projectName) {
@@ -202,7 +238,9 @@ export class Client {
         if (orderBy !== undefined) {
             orderBy.map((order) => queryParams.append("order_by", order));
         }
-        return this._get("/runs", queryParams);
+        for await (const runs of this._getPaginated("/runs", queryParams)) {
+            yield* runs;
+        }
     }
     async deleteRun(runId) {
         const response = await this.caller.call(fetch, `${this.apiUrl}/runs/${runId}`, {
@@ -261,8 +299,10 @@ export class Client {
         }
         return result;
     }
-    async listProjects() {
-        return this._get("/sessions");
+    async *listProjects() {
+        for await (const projects of this._getPaginated("/sessions")) {
+            yield* projects;
+        }
     }
     async deleteProject({ projectId, projectName, }) {
         let projectId_;
@@ -373,17 +413,15 @@ export class Client {
         }
         return result;
     }
-    async listDatasets({ limit = 100, offset = 0, } = {}) {
+    async *listDatasets({ limit = 100, offset = 0, } = {}) {
         const path = "/datasets";
         const params = new URLSearchParams({
             limit: limit.toString(),
             offset: offset.toString(),
         });
-        const response = await this._get(path, params);
-        if (!Array.isArray(response)) {
-            throw new Error(`Expected ${path} to return an array, but got ${response}`);
+        for await (const datasets of this._getPaginated(path, params)) {
+            yield* datasets;
         }
-        return response;
     }
     async deleteDataset({ datasetId, datasetName, }) {
         let path = "/datasets";
@@ -446,7 +484,7 @@ export class Client {
         const path = `/examples/${exampleId}`;
         return await this._get(path);
     }
-    async listExamples({ datasetId, datasetName, limit, offset, } = {}) {
+    async *listExamples({ datasetId, datasetName, } = {}) {
         let datasetId_;
         if (datasetId !== undefined && datasetName !== undefined) {
             throw new Error("Must provide either datasetName or datasetId, not both");
@@ -461,15 +499,10 @@ export class Client {
         else {
             throw new Error("Must provide a datasetName or datasetId");
         }
-        const response = await this._get("/examples", new URLSearchParams({
-            dataset: datasetId_,
-            limit: limit?.toString() ?? "100",
-            offset: offset?.toString() ?? "0",
-        }));
-        if (!Array.isArray(response)) {
-            throw new Error(`Expected /examples to return an array, but got ${response}`);
+        const params = new URLSearchParams({ dataset: datasetId_ });
+        for await (const examples of this._getPaginated("/examples", params)) {
+            yield* examples;
         }
-        return response;
     }
     async deleteExample(exampleId) {
         const path = `/examples/${exampleId}`;
@@ -576,18 +609,13 @@ export class Client {
         }
         await response.json();
     }
-    async listFeedback({ runIds, limit, offset, } = {}) {
+    async *listFeedback({ runIds, } = {}) {
         const queryParams = new URLSearchParams();
         if (runIds) {
             queryParams.append("run", runIds.join(","));
         }
-        if (limit !== undefined) {
-            queryParams.append("limit", limit.toString());
+        for await (const feedbacks of this._getPaginated("/feedback", queryParams)) {
+            yield* feedbacks;
         }
-        if (offset !== undefined) {
-            queryParams.append("offset", offset.toString());
-        }
-        const response = await this._get("/feedback", queryParams);
-        return response;
     }
 }

package/dist/run_trees.cjs

@@ -153,7 +153,6 @@ class RunTree {
             api_key: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_API_KEY"),
             caller_options: {},
             start_time: Date.now(),
-            end_time: Date.now(),
             serialized: {},
             inputs: {},
             extra: {},

package/dist/run_trees.d.ts

@@ -29,7 +29,7 @@ export declare class RunTree implements BaseRun {
     execution_order: number;
     child_execution_order: number;
     start_time: number;
-    end_time: number;
+    end_time?: number;
     extra: KVMap;
     error?: string;
     serialized: object;

package/dist/run_trees.js

@@ -127,7 +127,6 @@ export class RunTree {
             api_key: getEnvironmentVariable("LANGCHAIN_API_KEY"),
             caller_options: {},
             start_time: Date.now(),
-            end_time: Date.now(),
             serialized: {},
             inputs: {},
             extra: {},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "files": [
     "dist/",