langsmith 0.1.20 → 0.1.22

package/dist/evaluation/evaluator.cjs

@@ -1,2 +1,88 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEvaluator = exports.DynamicRunEvaluator = void 0;
+const uuid_1 = require("uuid");
+const traceable_js_1 = require("../traceable.cjs");
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const wrappedFunc = (input) => {
+            const runAndExample = input.langSmithRunAndExample;
+            return evaluator(...Object.values(runAndExample));
+        };
+        this.func = wrappedFunc;
+    }
+    coerceEvaluationResults(results, sourceRunId) {
+        if ("results" in results) {
+            throw new Error("EvaluationResults not supported yet.");
+        }
+        return this.coerceEvaluationResult(results, sourceRunId, true);
+    }
+    coerceEvaluationResult(result, sourceRunId, allowNoKey = false) {
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (!("key" in result)) {
+            if (allowNoKey) {
+                result["key"] = this.func.name;
+            }
+        }
+        return {
+            sourceRunId,
+            ...result,
+        };
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    async evaluateRun(run, example, options) {
+        const sourceRunId = (0, uuid_1.v4)();
+        const metadata = {
+            targetRunId: run.id,
+        };
+        if ("session_id" in run) {
+            metadata["experiment"] = run.session_id;
+        }
+        const wrappedTraceableFunc = (0, traceable_js_1.wrapFunctionAndEnsureTraceable)(this.func, options || {}, "evaluator");
+        // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
+        // inputs. This key is extracted in the wrapped function, with `run` and
+        // `example` passed to evaluator function as arguments.
+        const langSmithRunAndExample = {
+            run,
+            example,
+        };
+        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
+            metadata,
+        }));
+        // Check the one required property of EvaluationResult since 'instanceof' is not possible
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (typeof result !== "object") {
+            throw new Error("Evaluator function must return an object.");
+        }
+        return this.coerceEvaluationResults(result, sourceRunId);
+    }
+}
+exports.DynamicRunEvaluator = DynamicRunEvaluator;
+function runEvaluator(func) {
+    return new DynamicRunEvaluator(func);
+}
+exports.runEvaluator = runEvaluator;

package/dist/evaluation/evaluator.d.ts

@@ -1,4 +1,5 @@
-import { Example, Run, ScoreType, ValueType } from "../schemas.js";
+import { Example, FeedbackConfig, Run, ScoreType, ValueType } from "../schemas.js";
+import { RunTreeConfig } from "../run_trees.js";
 /**
  * Represents a categorical class.
  */
@@ -12,31 +13,6 @@ export type Category = {
      */
     label: string;
 };
-/**
- * Configuration for feedback.
- */
-export type FeedbackConfig = {
-    /**
-     * The type of feedback.
-     * - "continuous": Feedback with a continuous numeric.
-     * - "categorical": Feedback with a categorical value (classes)
-     * - "freeform": Feedback with a freeform text value (notes).
-     */
-    type: "continuous" | "categorical" | "freeform";
-    /**
-     * The minimum value for continuous feedback.
-     */
-    min?: number;
-    /**
-     * The maximum value for continuous feedback.
-     */
-    max?: number;
-    /**
-     * The categories for categorical feedback.
-     * Each category can be a string or an object with additional properties.
-     */
-    categories?: (Category | Record<string, unknown>)[];
-};
 /**
  * Represents the result of an evaluation.
  */
@@ -83,6 +59,34 @@ export type EvaluationResult = {
      */
     feedbackConfig?: FeedbackConfig;
 };
+/**
+ * Batch evaluation results, if your evaluator wishes
+ * to return multiple scores.
+ */
+export type EvaluationResults = {
+    /**
+     * The evaluation results.
+     */
+    results: Array<EvaluationResult>;
+};
 export interface RunEvaluator {
-    evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
+}
+export type RunEvaluatorLike = ((run: Run, example?: Example) => Promise<EvaluationResult | EvaluationResults>) | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults);
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export declare class DynamicRunEvaluator<Func extends (...args: any[]) => any> implements RunEvaluator {
+    func: Func;
+    constructor(evaluator: Func);
+    private coerceEvaluationResults;
+    private coerceEvaluationResult;
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
 }
+export declare function runEvaluator(func: RunEvaluatorLike): RunEvaluator;

package/dist/evaluation/evaluator.js

@@ -1 +1,83 @@
-export {};
+import { v4 as uuidv4 } from "uuid";
+import { wrapFunctionAndEnsureTraceable } from "../traceable.js";
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const wrappedFunc = (input) => {
+            const runAndExample = input.langSmithRunAndExample;
+            return evaluator(...Object.values(runAndExample));
+        };
+        this.func = wrappedFunc;
+    }
+    coerceEvaluationResults(results, sourceRunId) {
+        if ("results" in results) {
+            throw new Error("EvaluationResults not supported yet.");
+        }
+        return this.coerceEvaluationResult(results, sourceRunId, true);
+    }
+    coerceEvaluationResult(result, sourceRunId, allowNoKey = false) {
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (!("key" in result)) {
+            if (allowNoKey) {
+                result["key"] = this.func.name;
+            }
+        }
+        return {
+            sourceRunId,
+            ...result,
+        };
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    async evaluateRun(run, example, options) {
+        const sourceRunId = uuidv4();
+        const metadata = {
+            targetRunId: run.id,
+        };
+        if ("session_id" in run) {
+            metadata["experiment"] = run.session_id;
+        }
+        const wrappedTraceableFunc = wrapFunctionAndEnsureTraceable(this.func, options || {}, "evaluator");
+        // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
+        // inputs. This key is extracted in the wrapped function, with `run` and
+        // `example` passed to evaluator function as arguments.
+        const langSmithRunAndExample = {
+            run,
+            example,
+        };
+        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
+            metadata,
+        }));
+        // Check the one required property of EvaluationResult since 'instanceof' is not possible
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (typeof result !== "object") {
+            throw new Error("Evaluator function must return an object.");
+        }
+        return this.coerceEvaluationResults(result, sourceRunId);
+    }
+}
+export function runEvaluator(func) {
+    return new DynamicRunEvaluator(func);
+}

package/dist/evaluation/index.cjs

@@ -1,5 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StringEvaluator = void 0;
+exports.evaluate = exports.StringEvaluator = void 0;
 var string_evaluator_js_1 = require("./string_evaluator.cjs");
 Object.defineProperty(exports, "StringEvaluator", { enumerable: true, get: function () { return string_evaluator_js_1.StringEvaluator; } });
+var _runner_js_1 = require("./_runner.cjs");
+Object.defineProperty(exports, "evaluate", { enumerable: true, get: function () { return _runner_js_1.evaluate; } });

package/dist/evaluation/index.d.ts

@@ -1,2 +1,3 @@
 export { RunEvaluator, EvaluationResult } from "./evaluator.js";
 export { StringEvaluator, GradingFunctionParams, GradingFunctionResult, } from "./string_evaluator.js";
+export { evaluate, type EvaluateOptions } from "./_runner.js";

package/dist/evaluation/index.js

@@ -1 +1,2 @@
 export { StringEvaluator, } from "./string_evaluator.js";
+export { evaluate } from "./_runner.js";

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.20";
+exports.__version__ = "0.1.22";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.20";
+export declare const __version__ = "0.1.22";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.20";
+export const __version__ = "0.1.22";

package/dist/run_trees.cjs

@@ -232,7 +232,7 @@ class RunTree {
             extra: {},
         };
     }
-    async createChild(config) {
+    createChild(config) {
         const child = new RunTree({
             ...config,
             parent_run: this,
@@ -243,9 +243,9 @@ class RunTree {
         return child;
     }
     async end(outputs, error, endTime = Date.now()) {
-        this.outputs = outputs;
-        this.error = error;
-        this.end_time = endTime;
+        this.outputs = this.outputs ?? outputs;
+        this.error = this.error ?? error;
+        this.end_time = this.end_time ?? endTime;
     }
     async _convertToCreate(run, excludeChildRuns = true) {
         const runExtra = run.extra ?? {};

package/dist/run_trees.d.ts

@@ -20,6 +20,7 @@ export interface RunTreeConfig {
     outputs?: KVMap;
     reference_example_id?: string;
     client?: Client;
+    on_end?: (runTree: RunTree) => void;
 }
 export interface RunnableConfigLike {
     /**
@@ -65,7 +66,7 @@ export declare class RunTree implements BaseRun {
         metadata?: KVMap;
     }): RunTree;
     private static getDefaultConfig;
-    createChild(config: RunTreeConfig): Promise<RunTree>;
+    createChild(config: RunTreeConfig): RunTree;
     end(outputs?: KVMap, error?: string, endTime?: number): Promise<void>;
     private _convertToCreate;
     postRun(excludeChildRuns?: boolean): Promise<void>;

package/dist/run_trees.js

@@ -205,7 +205,7 @@ export class RunTree {
             extra: {},
         };
     }
-    async createChild(config) {
+    createChild(config) {
         const child = new RunTree({
             ...config,
             parent_run: this,
@@ -216,9 +216,9 @@ export class RunTree {
         return child;
     }
     async end(outputs, error, endTime = Date.now()) {
-        this.outputs = outputs;
-        this.error = error;
-        this.end_time = endTime;
+        this.outputs = this.outputs ?? outputs;
+        this.error = this.error ?? error;
+        this.end_time = this.end_time ?? endTime;
     }
     async _convertToCreate(run, excludeChildRuns = true) {
         const runExtra = run.extra ?? {};

package/dist/schemas.d.ts

@@ -5,6 +5,9 @@ export interface TracerSession {
     end_time?: number;
     description?: string;
     name?: string;
+    /** Extra metadata for the project. */
+    extra?: KVMap;
+    reference_dataset_id?: string;
 }
 export interface TracerSessionResult extends TracerSession {
     run_count?: number;
@@ -15,7 +18,6 @@ export interface TracerSessionResult extends TracerSession {
     completion_tokens?: number;
     last_run_start_time?: number;
     feedback_stats?: Record<string, unknown>;
-    reference_dataset_id?: string;
     run_facets?: KVMap[];
 }
 export type KVMap = Record<string, any>;
@@ -78,6 +80,14 @@ export interface BaseRun {
      */
     dotted_order?: string;
 }
+type S3URL = {
+    ROOT: {
+        /** A pre-signed URL */
+        presigned_url: string;
+        /** The S3 path to the object in storage */
+        s3_url: string;
+    };
+};
 /**
  * Describes properties of a run when loaded from the database.
  * Extends the BaseRun interface.
@@ -111,6 +121,10 @@ export interface Run extends BaseRun {
     parent_run_ids?: string[];
     /** Whether the run is included in a dataset. */
     in_dataset?: boolean;
+    /** The output S3 URLs */
+    outputs_s3_urls?: S3URL;
+    /** The input S3 URLs */
+    inputs_s3_urls?: S3URL;
 }
 export interface RunCreate extends BaseRun {
     revision_id?: string;
@@ -235,6 +249,9 @@ export interface FeedbackCategory {
 export interface FeedbackConfig {
     /**
      * The type of feedback.
+     * - "continuous": Feedback with a continuous numeric.
+     * - "categorical": Feedback with a categorical value (classes)
+     * - "freeform": Feedback with a freeform text value (notes).
      */
     type: "continuous" | "categorical" | "freeform";
     /**
@@ -246,6 +263,9 @@ export interface FeedbackConfig {
      */
     max?: number | null;
     /**
+     * The categories for categorical feedback.
+     * Each category can be a string or an object with additional properties.
+     *
      * If feedback is categorical, this defines the valid categories the server will accept.
      * Not applicable to continuous or freeform feedback types.
      */
@@ -256,3 +276,4 @@ export interface DatasetDiffInfo {
     examples_added: string[];
     examples_removed: string[];
 }
+export {};