langsmith 0.1.21 → 0.1.22

package/README.md

@@ -51,7 +51,7 @@ yarn add langchain
 Tracing can be activated by setting the following environment variables or by manually specifying the LangChainTracer.
 
 ```typescript
-process.env["LANGCHAIN_TRACING_V2"] = "true";
+process.env["LANGSMITH_TRACING"] = "true";
 process.env["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com";
 process.env["LANGCHAIN_API_KEY"] = "<YOUR-LANGSMITH-API-KEY>";
 // process.env["LANGCHAIN_PROJECT"] = "My Project Name"; // Optional: "default" is used if not set

package/dist/client.cjs

@@ -29,6 +29,7 @@ const async_caller_js_1 = require("./utils/async_caller.cjs");
 const messages_js_1 = require("./utils/messages.cjs");
 const env_js_1 = require("./utils/env.cjs");
 const index_js_1 = require("./index.cjs");
+const _uuid_js_1 = require("./utils/_uuid.cjs");
 async function mergeRuntimeEnvIntoRunCreates(runs) {
     const runtimeEnv = await (0, env_js_1.getRuntimeEnvironment)();
     const envVars = (0, env_js_1.getLangChainEnvVarsMetadata)();
@@ -93,11 +94,6 @@ function trimQuotes(str) {
         .replace(/^"(.*)"$/, "$1")
         .replace(/^'(.*)'$/, "$1");
 }
-function assertUuid(str) {
-    if (!uuid.validate(str)) {
-        throw new Error(`Invalid UUID: ${str}`);
-    }
-}
 const handle429 = async (response) => {
     if (response?.status === 429) {
         const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000;
@@ -667,7 +663,7 @@ class Client {
         await raiseForStatus(response, "batch create run");
     }
     async updateRun(runId, run) {
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         if (run.inputs) {
             run.inputs = this.processInputs(run.inputs);
         }
@@ -704,7 +700,7 @@ class Client {
         await raiseForStatus(response, "update run");
     }
     async readRun(runId, { loadChildRuns } = { loadChildRuns: false }) {
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         let run = await this._get(`/runs/${runId}`);
         if (loadChildRuns && run.child_run_ids) {
             run = await this._loadChildRuns(run);
@@ -920,7 +916,7 @@ class Client {
             run_id: runId,
             share_token: shareId || uuid.v4(),
         };
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/runs/${runId}/share`, {
             method: "PUT",
             headers: this.headers,
@@ -935,7 +931,7 @@ class Client {
         return `${this.getHostUrl()}/public/${result["share_token"]}/r`;
     }
     async unshareRun(runId) {
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/runs/${runId}/share`, {
             method: "DELETE",
             headers: this.headers,
@@ -945,7 +941,7 @@ class Client {
         await raiseForStatus(response, "unshare run");
     }
     async readRunSharedLink(runId) {
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/runs/${runId}/share`, {
             method: "GET",
             headers: this.headers,
@@ -967,7 +963,7 @@ class Client {
                 queryParams.append("id", runId);
             }
         }
-        assertUuid(shareToken);
+        (0, _uuid_js_1.assertUuid)(shareToken);
         const response = await this.caller.call(fetch, `${this.apiUrl}/public/${shareToken}/runs${queryParams}`, {
             method: "GET",
             headers: this.headers,
@@ -985,7 +981,7 @@ class Client {
             const dataset = await this.readDataset({ datasetName });
             datasetId = dataset.id;
         }
-        assertUuid(datasetId);
+        (0, _uuid_js_1.assertUuid)(datasetId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/datasets/${datasetId}/share`, {
             method: "GET",
             headers: this.headers,
@@ -1007,7 +1003,7 @@ class Client {
         const data = {
             dataset_id: datasetId,
         };
-        assertUuid(datasetId);
+        (0, _uuid_js_1.assertUuid)(datasetId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/datasets/${datasetId}/share`, {
             method: "PUT",
             headers: this.headers,
@@ -1020,7 +1016,7 @@ class Client {
         return shareSchema;
     }
     async unshareDataset(datasetId) {
-        assertUuid(datasetId);
+        (0, _uuid_js_1.assertUuid)(datasetId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/datasets/${datasetId}/share`, {
             method: "DELETE",
             headers: this.headers,
@@ -1030,7 +1026,7 @@ class Client {
         await raiseForStatus(response, "unshare dataset");
     }
     async readSharedDataset(shareToken) {
-        assertUuid(shareToken);
+        (0, _uuid_js_1.assertUuid)(shareToken);
         const response = await this.caller.call(fetch, `${this.apiUrl}/public/${shareToken}/datasets`, {
             method: "GET",
             headers: this.headers,
@@ -1101,7 +1097,7 @@ class Client {
             throw new Error("Must provide either projectName or projectId, not both");
         }
         else if (projectId !== undefined) {
-            assertUuid(projectId);
+            (0, _uuid_js_1.assertUuid)(projectId);
             path += `/${projectId}`;
         }
         else if (projectName !== undefined) {
@@ -1141,7 +1137,7 @@ class Client {
             throw new Error("Must provide either projectName or projectId, not both");
         }
         else if (projectId !== undefined) {
-            assertUuid(projectId);
+            (0, _uuid_js_1.assertUuid)(projectId);
             path += `/${projectId}`;
         }
         else if (projectName !== undefined) {
@@ -1220,7 +1216,7 @@ class Client {
         else {
             projectId_ = projectId;
         }
-        assertUuid(projectId_);
+        (0, _uuid_js_1.assertUuid)(projectId_);
         const response = await this.caller.call(fetch, `${this.apiUrl}/sessions/${projectId_}`, {
             method: "DELETE",
             headers: this.headers,
@@ -1298,7 +1294,7 @@ class Client {
             throw new Error("Must provide either datasetName or datasetId, not both");
         }
         else if (datasetId !== undefined) {
-            assertUuid(datasetId);
+            (0, _uuid_js_1.assertUuid)(datasetId);
             path += `/${datasetId}`;
         }
         else if (datasetName !== undefined) {
@@ -1392,7 +1388,7 @@ class Client {
             datasetId_ = dataset.id;
         }
         if (datasetId_ !== undefined) {
-            assertUuid(datasetId_);
+            (0, _uuid_js_1.assertUuid)(datasetId_);
             path += `/${datasetId_}`;
         }
         else {
@@ -1495,7 +1491,7 @@ class Client {
         return this.createExample({ input: finalInput }, { output: finalOutput }, options);
     }
     async readExample(exampleId) {
-        assertUuid(exampleId);
+        (0, _uuid_js_1.assertUuid)(exampleId);
         const path = `/examples/${exampleId}`;
         return await this._get(path);
     }
@@ -1535,7 +1531,7 @@ class Client {
         }
     }
     async deleteExample(exampleId) {
-        assertUuid(exampleId);
+        (0, _uuid_js_1.assertUuid)(exampleId);
         const path = `/examples/${exampleId}`;
         const response = await this.caller.call(fetch, this.apiUrl + path, {
             method: "DELETE",
@@ -1549,7 +1545,7 @@ class Client {
         await response.json();
     }
     async updateExample(exampleId, update) {
-        assertUuid(exampleId);
+        (0, _uuid_js_1.assertUuid)(exampleId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/examples/${exampleId}`, {
             method: "PATCH",
             headers: { ...this.headers, "Content-Type": "application/json" },
@@ -1594,7 +1590,13 @@ class Client {
             sourceRunId: feedbackResult?.sourceRunId,
         });
     }
-    async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, }) {
+    async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, projectId, }) {
+        if (!runId && !projectId) {
+            throw new Error("One of runId or projectId must be provided");
+        }
+        if (runId && projectId) {
+            throw new Error("Only one of runId or projectId can be provided");
+        }
         const feedback_source = {
             type: feedbackSourceType ?? "api",
             metadata: sourceInfo ?? {},
@@ -1606,7 +1608,7 @@ class Client {
         }
         if (feedback_source?.metadata !== undefined &&
             feedback_source.metadata["__run"]?.run_id !== undefined) {
-            assertUuid(feedback_source.metadata["__run"].run_id);
+            (0, _uuid_js_1.assertUuid)(feedback_source.metadata["__run"].run_id);
         }
         const feedback = {
             id: feedbackId ?? uuid.v4(),
@@ -1618,6 +1620,7 @@ class Client {
             comment,
             feedback_source: feedback_source,
             feedbackConfig,
+            session_id: projectId,
         };
         const url = `${this.apiUrl}/feedback`;
         const response = await this.caller.call(fetch, url, {
@@ -1644,7 +1647,7 @@ class Client {
         if (comment !== undefined && comment !== null) {
             feedbackUpdate["comment"] = comment;
         }
-        assertUuid(feedbackId);
+        (0, _uuid_js_1.assertUuid)(feedbackId);
         const response = await this.caller.call(fetch, `${this.apiUrl}/feedback/${feedbackId}`, {
             method: "PATCH",
             headers: { ...this.headers, "Content-Type": "application/json" },
@@ -1655,13 +1658,13 @@ class Client {
         await raiseForStatus(response, "update feedback");
     }
     async readFeedback(feedbackId) {
-        assertUuid(feedbackId);
+        (0, _uuid_js_1.assertUuid)(feedbackId);
         const path = `/feedback/${feedbackId}`;
         const response = await this._get(path);
         return response;
     }
     async deleteFeedback(feedbackId) {
-        assertUuid(feedbackId);
+        (0, _uuid_js_1.assertUuid)(feedbackId);
         const path = `/feedback/${feedbackId}`;
         const response = await this.caller.call(fetch, this.apiUrl + path, {
             method: "DELETE",
@@ -1743,11 +1746,48 @@ class Client {
      * @returns An async iterable of FeedbackIngestToken objects.
      */
     async *listPresignedFeedbackTokens(runId) {
-        assertUuid(runId);
+        (0, _uuid_js_1.assertUuid)(runId);
         const params = new URLSearchParams({ run_id: runId });
         for await (const tokens of this._getPaginated("/feedback/tokens", params)) {
             yield* tokens;
         }
     }
+    _selectEvalResults(results) {
+        let results_;
+        if ("results" in results) {
+            results_ = results.results;
+        }
+        else {
+            results_ = [results];
+        }
+        return results_;
+    }
+    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const results = this._selectEvalResults(evaluatorResponse);
+        for (const res of results) {
+            let sourceInfo_ = sourceInfo || {};
+            if (res.evaluatorInfo) {
+                sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ };
+            }
+            let runId_ = null;
+            if (res.targetRunId) {
+                runId_ = res.targetRunId;
+            }
+            else if (run) {
+                runId_ = run.id;
+            }
+            await this.createFeedback(runId_, res.key, {
+                score: res.score,
+                value: res.value,
+                comment: res.comment,
+                correction: res.correction,
+                sourceInfo: sourceInfo_,
+                sourceRunId: res.sourceRunId,
+                feedbackConfig: res.feedbackConfig,
+                feedbackSourceType: "model",
+            });
+        }
+        return results;
+    }
 }
 exports.Client = Client;

package/dist/client.d.ts

@@ -1,6 +1,6 @@
 import { AsyncCallerParams } from "./utils/async_caller.js";
 import { DataType, Dataset, DatasetDiffInfo, DatasetShareSchema, Example, ExampleUpdate, Feedback, FeedbackConfig, FeedbackIngestToken, KVMap, LangChainBaseMessage, Run, RunCreate, RunUpdate, ScoreType, TimeDelta, TracerSession, TracerSessionResult, ValueType } from "./schemas.js";
-import { RunEvaluator } from "./evaluation/evaluator.js";
+import { EvaluationResult, EvaluationResults, RunEvaluator } from "./evaluation/evaluator.js";
 interface ClientConfig {
     apiUrl?: string;
     apiKey?: string;
@@ -102,7 +102,6 @@ interface ListRunsParams {
     treeFilter?: string;
     /**
      * The values to include in the response.
-     *
      */
     select?: string[];
 }
@@ -405,7 +404,7 @@ export declare class Client {
         loadChildRuns: boolean;
         referenceExample?: Example;
     }): Promise<Feedback>;
-    createFeedback(runId: string, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, }: {
+    createFeedback(runId: string | null, key: string, { score, value, correction, comment, sourceInfo, feedbackSourceType, sourceRunId, feedbackId, feedbackConfig, projectId, }: {
         score?: ScoreType;
         value?: ValueType;
         correction?: object;
@@ -416,6 +415,7 @@ export declare class Client {
         sourceRunId?: string;
         feedbackId?: string;
         eager?: boolean;
+        projectId?: string;
     }): Promise<Feedback>;
     updateFeedback(feedbackId: string, { score, value, correction, comment, }: {
         score?: number | boolean | null;
@@ -455,5 +455,9 @@ export declare class Client {
      * @returns An async iterable of FeedbackIngestToken objects.
      */
     listPresignedFeedbackTokens(runId: string): AsyncIterable<FeedbackIngestToken>;
+    _selectEvalResults(results: EvaluationResult | EvaluationResults): Array<EvaluationResult>;
+    logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResults, run?: Run, sourceInfo?: {
+        [key: string]: any;
+    }): Promise<EvaluationResult[]>;
 }
 export {};

package/dist/client.js

@@ -3,6 +3,7 @@ import { AsyncCaller } from "./utils/async_caller.js";
 import { convertLangChainMessageToExample, isLangChainMessage, } from "./utils/messages.js";
 import { getEnvironmentVariable, getLangChainEnvVarsMetadata, getRuntimeEnvironment, } from "./utils/env.js";
 import { __version__ } from "./index.js";
+import { assertUuid } from "./utils/_uuid.js";
 async function mergeRuntimeEnvIntoRunCreates(runs) {
     const runtimeEnv = await getRuntimeEnvironment();
     const envVars = getLangChainEnvVarsMetadata();
@@ -67,11 +68,6 @@ function trimQuotes(str) {
         .replace(/^"(.*)"$/, "$1")
         .replace(/^'(.*)'$/, "$1");
 }
-function assertUuid(str) {
-    if (!uuid.validate(str)) {
-        throw new Error(`Invalid UUID: ${str}`);
-    }
-}
 const handle429 = async (response) => {
     if (response?.status === 429) {
         const retryAfter = parseInt(response.headers.get("retry-after") ?? "30", 10) * 1000;
@@ -1567,7 +1563,13 @@ export class Client {
             sourceRunId: feedbackResult?.sourceRunId,
         });
     }
-    async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, }) {
+    async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, projectId, }) {
+        if (!runId && !projectId) {
+            throw new Error("One of runId or projectId must be provided");
+        }
+        if (runId && projectId) {
+            throw new Error("Only one of runId or projectId can be provided");
+        }
         const feedback_source = {
             type: feedbackSourceType ?? "api",
             metadata: sourceInfo ?? {},
@@ -1591,6 +1593,7 @@ export class Client {
             comment,
             feedback_source: feedback_source,
             feedbackConfig,
+            session_id: projectId,
         };
         const url = `${this.apiUrl}/feedback`;
         const response = await this.caller.call(fetch, url, {
@@ -1722,4 +1725,41 @@ export class Client {
             yield* tokens;
         }
     }
+    _selectEvalResults(results) {
+        let results_;
+        if ("results" in results) {
+            results_ = results.results;
+        }
+        else {
+            results_ = [results];
+        }
+        return results_;
+    }
+    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const results = this._selectEvalResults(evaluatorResponse);
+        for (const res of results) {
+            let sourceInfo_ = sourceInfo || {};
+            if (res.evaluatorInfo) {
+                sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ };
+            }
+            let runId_ = null;
+            if (res.targetRunId) {
+                runId_ = res.targetRunId;
+            }
+            else if (run) {
+                runId_ = run.id;
+            }
+            await this.createFeedback(runId_, res.key, {
+                score: res.score,
+                value: res.value,
+                comment: res.comment,
+                correction: res.correction,
+                sourceInfo: sourceInfo_,
+                sourceRunId: res.sourceRunId,
+                feedbackConfig: res.feedbackConfig,
+                feedbackSourceType: "model",
+            });
+        }
+        return results;
+    }
 }