langsmith 0.1.31 → 0.1.32

package/dist/client.cjs

@@ -30,6 +30,7 @@ const messages_js_1 = require("./utils/messages.cjs");
 const env_js_1 = require("./utils/env.cjs");
 const index_js_1 = require("./index.cjs");
 const _uuid_js_1 = require("./utils/_uuid.cjs");
+const warn_js_1 = require("./utils/warn.cjs");
 async function mergeRuntimeEnvIntoRunCreates(runs) {
     const runtimeEnv = await (0, env_js_1.getRuntimeEnvironment)();
     const envVars = (0, env_js_1.getLangChainEnvVarsMetadata)();
@@ -1616,7 +1617,11 @@ class Client {
         const result = await response.json();
         return result;
     }
+    /**
+     * @deprecated This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead.
+     */
     async evaluateRun(run, evaluator, { sourceInfo, loadChildRuns, referenceExample, } = { loadChildRuns: false }) {
+        (0, warn_js_1.warnOnce)("This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead.");
         let run_;
         if (typeof run === "string") {
             run_ = await this.readRun(run, { loadChildRuns });
@@ -1632,20 +1637,8 @@ class Client {
             referenceExample = await this.readExample(run_.reference_example_id);
         }
         const feedbackResult = await evaluator.evaluateRun(run_, referenceExample);
-        let sourceInfo_ = sourceInfo ?? {};
-        if (feedbackResult.evaluatorInfo) {
-            sourceInfo_ = { ...sourceInfo_, ...feedbackResult.evaluatorInfo };
-        }
-        const runId = feedbackResult.targetRunId ?? run_.id;
-        return await this.createFeedback(runId, feedbackResult.key, {
-            score: feedbackResult?.score,
-            value: feedbackResult?.value,
-            comment: feedbackResult?.comment,
-            correction: feedbackResult?.correction,
-            sourceInfo: sourceInfo_,
-            feedbackSourceType: "model",
-            sourceRunId: feedbackResult?.sourceRunId,
-        });
+        const [_, feedbacks] = await this._logEvaluationFeedback(feedbackResult, run_, sourceInfo);
+        return feedbacks[0];
     }
     async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, projectId, comparativeExperimentId, }) {
         if (!runId && !projectId) {
@@ -1852,9 +1845,10 @@ class Client {
         }
         return results_;
     }
-    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
-        const results = this._selectEvalResults(evaluatorResponse);
-        for (const res of results) {
+    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const evalResults = this._selectEvalResults(evaluatorResponse);
+        const feedbacks = [];
+        for (const res of evalResults) {
             let sourceInfo_ = sourceInfo || {};
             if (res.evaluatorInfo) {
                 sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ };
@@ -1866,7 +1860,7 @@ class Client {
             else if (run) {
                 runId_ = run.id;
             }
-            await this.createFeedback(runId_, res.key, {
+            feedbacks.push(await this.createFeedback(runId_, res.key, {
                 score: res.score,
                 value: res.value,
                 comment: res.comment,
@@ -1875,8 +1869,12 @@ class Client {
                 sourceRunId: res.sourceRunId,
                 feedbackConfig: res.feedbackConfig,
                 feedbackSourceType: "model",
-            });
+            }));
         }
+        return [evalResults, feedbacks];
+    }
+    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
 }

package/dist/client.d.ts

@@ -415,6 +415,9 @@ export declare class Client {
     }): AsyncIterable<Example>;
     deleteExample(exampleId: string): Promise<void>;
     updateExample(exampleId: string, update: ExampleUpdate): Promise<object>;
+    /**
+     * @deprecated This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead.
+     */
     evaluateRun(run: Run | string, evaluator: RunEvaluator, { sourceInfo, loadChildRuns, referenceExample, }?: {
         sourceInfo?: KVMap;
         loadChildRuns: boolean;
@@ -482,6 +485,9 @@ export declare class Client {
      */
     listPresignedFeedbackTokens(runId: string): AsyncIterable<FeedbackIngestToken>;
     _selectEvalResults(results: EvaluationResult | EvaluationResults): Array<EvaluationResult>;
+    _logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResults, run?: Run, sourceInfo?: {
+        [key: string]: any;
+    }): Promise<[results: EvaluationResult[], feedbacks: Feedback[]]>;
     logEvaluationFeedback(evaluatorResponse: EvaluationResult | EvaluationResults, run?: Run, sourceInfo?: {
         [key: string]: any;
     }): Promise<EvaluationResult[]>;

package/dist/client.js

@@ -4,6 +4,7 @@ import { convertLangChainMessageToExample, isLangChainMessage, } from "./utils/m
 import { getEnvironmentVariable, getLangChainEnvVarsMetadata, getRuntimeEnvironment, } from "./utils/env.js";
 import { __version__ } from "./index.js";
 import { assertUuid } from "./utils/_uuid.js";
+import { warnOnce } from "./utils/warn.js";
 async function mergeRuntimeEnvIntoRunCreates(runs) {
     const runtimeEnv = await getRuntimeEnvironment();
     const envVars = getLangChainEnvVarsMetadata();
@@ -1589,7 +1590,11 @@ export class Client {
         const result = await response.json();
         return result;
     }
+    /**
+     * @deprecated This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead.
+     */
     async evaluateRun(run, evaluator, { sourceInfo, loadChildRuns, referenceExample, } = { loadChildRuns: false }) {
+        warnOnce("This method is deprecated and will be removed in future LangSmith versions, use `evaluate` from `langsmith/evaluation` instead.");
         let run_;
         if (typeof run === "string") {
             run_ = await this.readRun(run, { loadChildRuns });
@@ -1605,20 +1610,8 @@ export class Client {
             referenceExample = await this.readExample(run_.reference_example_id);
         }
         const feedbackResult = await evaluator.evaluateRun(run_, referenceExample);
-        let sourceInfo_ = sourceInfo ?? {};
-        if (feedbackResult.evaluatorInfo) {
-            sourceInfo_ = { ...sourceInfo_, ...feedbackResult.evaluatorInfo };
-        }
-        const runId = feedbackResult.targetRunId ?? run_.id;
-        return await this.createFeedback(runId, feedbackResult.key, {
-            score: feedbackResult?.score,
-            value: feedbackResult?.value,
-            comment: feedbackResult?.comment,
-            correction: feedbackResult?.correction,
-            sourceInfo: sourceInfo_,
-            feedbackSourceType: "model",
-            sourceRunId: feedbackResult?.sourceRunId,
-        });
+        const [_, feedbacks] = await this._logEvaluationFeedback(feedbackResult, run_, sourceInfo);
+        return feedbacks[0];
     }
     async createFeedback(runId, key, { score, value, correction, comment, sourceInfo, feedbackSourceType = "api", sourceRunId, feedbackId, feedbackConfig, projectId, comparativeExperimentId, }) {
         if (!runId && !projectId) {
@@ -1825,9 +1818,10 @@ export class Client {
         }
         return results_;
     }
-    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
-        const results = this._selectEvalResults(evaluatorResponse);
-        for (const res of results) {
+    async _logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const evalResults = this._selectEvalResults(evaluatorResponse);
+        const feedbacks = [];
+        for (const res of evalResults) {
             let sourceInfo_ = sourceInfo || {};
             if (res.evaluatorInfo) {
                 sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ };
@@ -1839,7 +1833,7 @@ export class Client {
             else if (run) {
                 runId_ = run.id;
             }
-            await this.createFeedback(runId_, res.key, {
+            feedbacks.push(await this.createFeedback(runId_, res.key, {
                 score: res.score,
                 value: res.value,
                 comment: res.comment,
@@ -1848,8 +1842,12 @@ export class Client {
                 sourceRunId: res.sourceRunId,
                 feedbackConfig: res.feedbackConfig,
                 feedbackSourceType: "model",
-            });
+            }));
         }
+        return [evalResults, feedbacks];
+    }
+    async logEvaluationFeedback(evaluatorResponse, run, sourceInfo) {
+        const [results] = await this._logEvaluationFeedback(evaluatorResponse, run, sourceInfo);
         return results;
     }
 }

package/dist/evaluation/_runner.d.ts

@@ -8,7 +8,7 @@ type TargetT<TInput = any, TOutput = KVMap> = ((input: TInput, config?: KVMap) =
 };
 type DataT = string | AsyncIterable<Example> | Example[];
 type SummaryEvaluatorT = ((runs: Array<Run>, examples: Array<Example>) => Promise<EvaluationResult | EvaluationResults>) | ((runs: Array<Run>, examples: Array<Example>) => EvaluationResult | EvaluationResults);
-type EvaluatorT = RunEvaluator | ((run: Run, example?: Example) => EvaluationResult) | ((run: Run, example?: Example) => Promise<EvaluationResult>);
+type EvaluatorT = RunEvaluator | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults) | ((run: Run, example?: Example) => Promise<EvaluationResult | EvaluationResults>);
 interface _ForwardResults {
     run: Run;
     example: Example;

package/dist/evaluation/evaluator.cjs

@@ -19,9 +19,18 @@ class DynamicRunEvaluator {
             return evaluator(run, example);
         });
     }
+    isEvaluationResults(x) {
+        return (typeof x === "object" &&
+            x != null &&
+            "results" in x &&
+            Array.isArray(x.results) &&
+            x.results.length > 0);
+    }
     coerceEvaluationResults(results, sourceRunId) {
-        if ("results" in results) {
-            throw new Error("EvaluationResults not supported yet.");
+        if (this.isEvaluationResults(results)) {
+            return {
+                results: results.results.map((r) => this.coerceEvaluationResult(r, sourceRunId, false)),
+            };
         }
         return this.coerceEvaluationResult(results, sourceRunId, true);
     }

package/dist/evaluation/evaluator.d.ts

@@ -70,7 +70,7 @@ export type EvaluationResults = {
     results: Array<EvaluationResult>;
 };
 export interface RunEvaluator {
-    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult | EvaluationResults>;
 }
 export type RunEvaluatorLike = ((run: Run, example?: Example) => Promise<EvaluationResult | EvaluationResults>) | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults);
 /**
@@ -79,6 +79,7 @@ export type RunEvaluatorLike = ((run: Run, example?: Example) => Promise<Evaluat
 export declare class DynamicRunEvaluator<Func extends (...args: any[]) => any> implements RunEvaluator {
     func: Func;
     constructor(evaluator: Func);
+    private isEvaluationResults;
     private coerceEvaluationResults;
     private coerceEvaluationResult;
     /**
@@ -87,6 +88,6 @@ export declare class DynamicRunEvaluator<Func extends (...args: any[]) => any> i
      * @param example The optional example to use for evaluation.
      * @returns A promise that extracts to the evaluation result.
      */
-    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult | EvaluationResults>;
 }
 export declare function runEvaluator(func: RunEvaluatorLike): RunEvaluator;

package/dist/evaluation/evaluator.js

@@ -16,9 +16,18 @@ export class DynamicRunEvaluator {
             return evaluator(run, example);
         });
     }
+    isEvaluationResults(x) {
+        return (typeof x === "object" &&
+            x != null &&
+            "results" in x &&
+            Array.isArray(x.results) &&
+            x.results.length > 0);
+    }
     coerceEvaluationResults(results, sourceRunId) {
-        if ("results" in results) {
-            throw new Error("EvaluationResults not supported yet.");
+        if (this.isEvaluationResults(results)) {
+            return {
+                results: results.results.map((r) => this.coerceEvaluationResult(r, sourceRunId, false)),
+            };
         }
         return this.coerceEvaluationResult(results, sourceRunId, true);
     }

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.31";
+exports.__version__ = "0.1.32";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.31";
+export declare const __version__ = "0.1.32";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.31";
+export const __version__ = "0.1.32";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.31",
+  "version": "0.1.32",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [