langsmith 0.1.21 → 0.1.22

package/dist/evaluation/evaluator.cjs

@@ -1,2 +1,88 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEvaluator = exports.DynamicRunEvaluator = void 0;
+const uuid_1 = require("uuid");
+const traceable_js_1 = require("../traceable.cjs");
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const wrappedFunc = (input) => {
+            const runAndExample = input.langSmithRunAndExample;
+            return evaluator(...Object.values(runAndExample));
+        };
+        this.func = wrappedFunc;
+    }
+    coerceEvaluationResults(results, sourceRunId) {
+        if ("results" in results) {
+            throw new Error("EvaluationResults not supported yet.");
+        }
+        return this.coerceEvaluationResult(results, sourceRunId, true);
+    }
+    coerceEvaluationResult(result, sourceRunId, allowNoKey = false) {
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (!("key" in result)) {
+            if (allowNoKey) {
+                result["key"] = this.func.name;
+            }
+        }
+        return {
+            sourceRunId,
+            ...result,
+        };
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    async evaluateRun(run, example, options) {
+        const sourceRunId = (0, uuid_1.v4)();
+        const metadata = {
+            targetRunId: run.id,
+        };
+        if ("session_id" in run) {
+            metadata["experiment"] = run.session_id;
+        }
+        const wrappedTraceableFunc = (0, traceable_js_1.wrapFunctionAndEnsureTraceable)(this.func, options || {}, "evaluator");
+        // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
+        // inputs. This key is extracted in the wrapped function, with `run` and
+        // `example` passed to evaluator function as arguments.
+        const langSmithRunAndExample = {
+            run,
+            example,
+        };
+        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
+            metadata,
+        }));
+        // Check the one required property of EvaluationResult since 'instanceof' is not possible
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (typeof result !== "object") {
+            throw new Error("Evaluator function must return an object.");
+        }
+        return this.coerceEvaluationResults(result, sourceRunId);
+    }
+}
+exports.DynamicRunEvaluator = DynamicRunEvaluator;
+function runEvaluator(func) {
+    return new DynamicRunEvaluator(func);
+}
+exports.runEvaluator = runEvaluator;

package/dist/evaluation/evaluator.d.ts

@@ -1,4 +1,5 @@
-import { Example, Run, ScoreType, ValueType } from "../schemas.js";
+import { Example, FeedbackConfig, Run, ScoreType, ValueType } from "../schemas.js";
+import { RunTreeConfig } from "../run_trees.js";
 /**
  * Represents a categorical class.
  */
@@ -12,31 +13,6 @@ export type Category = {
      */
     label: string;
 };
-/**
- * Configuration for feedback.
- */
-export type FeedbackConfig = {
-    /**
-     * The type of feedback.
-     * - "continuous": Feedback with a continuous numeric.
-     * - "categorical": Feedback with a categorical value (classes)
-     * - "freeform": Feedback with a freeform text value (notes).
-     */
-    type: "continuous" | "categorical" | "freeform";
-    /**
-     * The minimum value for continuous feedback.
-     */
-    min?: number;
-    /**
-     * The maximum value for continuous feedback.
-     */
-    max?: number;
-    /**
-     * The categories for categorical feedback.
-     * Each category can be a string or an object with additional properties.
-     */
-    categories?: (Category | Record<string, unknown>)[];
-};
 /**
  * Represents the result of an evaluation.
  */
@@ -83,6 +59,34 @@ export type EvaluationResult = {
      */
     feedbackConfig?: FeedbackConfig;
 };
+/**
+ * Batch evaluation results, if your evaluator wishes
+ * to return multiple scores.
+ */
+export type EvaluationResults = {
+    /**
+     * The evaluation results.
+     */
+    results: Array<EvaluationResult>;
+};
 export interface RunEvaluator {
-    evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
+}
+export type RunEvaluatorLike = ((run: Run, example?: Example) => Promise<EvaluationResult | EvaluationResults>) | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults);
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export declare class DynamicRunEvaluator<Func extends (...args: any[]) => any> implements RunEvaluator {
+    func: Func;
+    constructor(evaluator: Func);
+    private coerceEvaluationResults;
+    private coerceEvaluationResult;
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    evaluateRun(run: Run, example?: Example, options?: Partial<RunTreeConfig>): Promise<EvaluationResult>;
 }
+export declare function runEvaluator(func: RunEvaluatorLike): RunEvaluator;

package/dist/evaluation/evaluator.js

@@ -1 +1,83 @@
-export {};
+import { v4 as uuidv4 } from "uuid";
+import { wrapFunctionAndEnsureTraceable } from "../traceable.js";
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+export class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const wrappedFunc = (input) => {
+            const runAndExample = input.langSmithRunAndExample;
+            return evaluator(...Object.values(runAndExample));
+        };
+        this.func = wrappedFunc;
+    }
+    coerceEvaluationResults(results, sourceRunId) {
+        if ("results" in results) {
+            throw new Error("EvaluationResults not supported yet.");
+        }
+        return this.coerceEvaluationResult(results, sourceRunId, true);
+    }
+    coerceEvaluationResult(result, sourceRunId, allowNoKey = false) {
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (!("key" in result)) {
+            if (allowNoKey) {
+                result["key"] = this.func.name;
+            }
+        }
+        return {
+            sourceRunId,
+            ...result,
+        };
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that extracts to the evaluation result.
+     */
+    async evaluateRun(run, example, options) {
+        const sourceRunId = uuidv4();
+        const metadata = {
+            targetRunId: run.id,
+        };
+        if ("session_id" in run) {
+            metadata["experiment"] = run.session_id;
+        }
+        const wrappedTraceableFunc = wrapFunctionAndEnsureTraceable(this.func, options || {}, "evaluator");
+        // Pass data via `langSmithRunAndExample` key to avoid conflicts with other
+        // inputs. This key is extracted in the wrapped function, with `run` and
+        // `example` passed to evaluator function as arguments.
+        const langSmithRunAndExample = {
+            run,
+            example,
+        };
+        const result = (await wrappedTraceableFunc({ langSmithRunAndExample }, {
+            metadata,
+        }));
+        // Check the one required property of EvaluationResult since 'instanceof' is not possible
+        if ("key" in result) {
+            if (!result.sourceRunId) {
+                result.sourceRunId = sourceRunId;
+            }
+            return result;
+        }
+        if (typeof result !== "object") {
+            throw new Error("Evaluator function must return an object.");
+        }
+        return this.coerceEvaluationResults(result, sourceRunId);
+    }
+}
+export function runEvaluator(func) {
+    return new DynamicRunEvaluator(func);
+}

package/dist/evaluation/index.cjs

@@ -1,5 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.StringEvaluator = void 0;
+exports.evaluate = exports.StringEvaluator = void 0;
 var string_evaluator_js_1 = require("./string_evaluator.cjs");
 Object.defineProperty(exports, "StringEvaluator", { enumerable: true, get: function () { return string_evaluator_js_1.StringEvaluator; } });
+var _runner_js_1 = require("./_runner.cjs");
+Object.defineProperty(exports, "evaluate", { enumerable: true, get: function () { return _runner_js_1.evaluate; } });

package/dist/evaluation/index.d.ts

@@ -1,2 +1,3 @@
 export { RunEvaluator, EvaluationResult } from "./evaluator.js";
 export { StringEvaluator, GradingFunctionParams, GradingFunctionResult, } from "./string_evaluator.js";
+export { evaluate, type EvaluateOptions } from "./_runner.js";

package/dist/evaluation/index.js

@@ -1 +1,2 @@
 export { StringEvaluator, } from "./string_evaluator.js";
+export { evaluate } from "./_runner.js";

package/dist/index.cjs

@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
 var run_trees_js_1 = require("./run_trees.cjs");
 Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
 // Update using yarn bump-version
-exports.__version__ = "0.1.21";
+exports.__version__ = "0.1.22";

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
 export { RunTree, type RunTreeConfig } from "./run_trees.js";
-export declare const __version__ = "0.1.21";
+export declare const __version__ = "0.1.22";

package/dist/index.js

@@ -1,4 +1,4 @@
 export { Client } from "./client.js";
 export { RunTree } from "./run_trees.js";
 // Update using yarn bump-version
-export const __version__ = "0.1.21";
+export const __version__ = "0.1.22";

package/dist/run_trees.d.ts

@@ -20,6 +20,7 @@ export interface RunTreeConfig {
     outputs?: KVMap;
     reference_example_id?: string;
     client?: Client;
+    on_end?: (runTree: RunTree) => void;
 }
 export interface RunnableConfigLike {
     /**

package/dist/schemas.d.ts

@@ -5,6 +5,9 @@ export interface TracerSession {
     end_time?: number;
     description?: string;
     name?: string;
+    /** Extra metadata for the project. */
+    extra?: KVMap;
+    reference_dataset_id?: string;
 }
 export interface TracerSessionResult extends TracerSession {
     run_count?: number;
@@ -15,7 +18,6 @@ export interface TracerSessionResult extends TracerSession {
     completion_tokens?: number;
     last_run_start_time?: number;
     feedback_stats?: Record<string, unknown>;
-    reference_dataset_id?: string;
     run_facets?: KVMap[];
 }
 export type KVMap = Record<string, any>;
@@ -78,6 +80,14 @@ export interface BaseRun {
      */
     dotted_order?: string;
 }
+type S3URL = {
+    ROOT: {
+        /** A pre-signed URL */
+        presigned_url: string;
+        /** The S3 path to the object in storage */
+        s3_url: string;
+    };
+};
 /**
  * Describes properties of a run when loaded from the database.
  * Extends the BaseRun interface.
@@ -111,6 +121,10 @@ export interface Run extends BaseRun {
     parent_run_ids?: string[];
     /** Whether the run is included in a dataset. */
     in_dataset?: boolean;
+    /** The output S3 URLs */
+    outputs_s3_urls?: S3URL;
+    /** The input S3 URLs */
+    inputs_s3_urls?: S3URL;
 }
 export interface RunCreate extends BaseRun {
     revision_id?: string;
@@ -235,6 +249,9 @@ export interface FeedbackCategory {
 export interface FeedbackConfig {
     /**
      * The type of feedback.
+     * - "continuous": Feedback with a continuous numeric.
+     * - "categorical": Feedback with a categorical value (classes)
+     * - "freeform": Feedback with a freeform text value (notes).
      */
     type: "continuous" | "categorical" | "freeform";
     /**
@@ -246,6 +263,9 @@ export interface FeedbackConfig {
      */
     max?: number | null;
     /**
+     * The categories for categorical feedback.
+     * Each category can be a string or an object with additional properties.
+     *
      * If feedback is categorical, this defines the valid categories the server will accept.
      * Not applicable to continuous or freeform feedback types.
      */
@@ -256,3 +276,4 @@ export interface DatasetDiffInfo {
     examples_added: string[];
     examples_removed: string[];
 }
+export {};

package/dist/traceable.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isTraceableFunction = exports.getCurrentRunTree = exports.traceable = void 0;
+exports.wrapFunctionAndEnsureTraceable = exports.isTraceableFunction = exports.getCurrentRunTree = exports.traceable = void 0;
 const async_hooks_1 = require("async_hooks");
 const run_trees_js_1 = require("./run_trees.cjs");
 const env_js_1 = require("./utils/env.cjs");
@@ -15,10 +15,21 @@ const isAsyncIterable = (x) => x != null &&
     typeof x === "object" &&
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     typeof x[Symbol.asyncIterator] === "function";
-const getTracingRunTree = (runTree) => {
-    const tracingEnabled = (0, env_js_1.getEnvironmentVariable)("LANGSMITH_TRACING_V2") === "true" ||
-        (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_TRACING_V2") === "true";
-    if (!tracingEnabled) {
+const tracingIsEnabled = (tracingEnabled) => {
+    if (tracingEnabled !== undefined) {
+        return tracingEnabled;
+    }
+    const envVars = [
+        "LANGSMITH_TRACING_V2",
+        "LANGCHAIN_TRACING_V2",
+        "LANGSMITH_TRACING",
+        "LANGCHAIN_TRACING",
+    ];
+    return Boolean(envVars.find((envVar) => (0, env_js_1.getEnvironmentVariable)(envVar) === "true"));
+};
+const getTracingRunTree = (runTree, tracingEnabled) => {
+    const tracingEnabled_ = tracingIsEnabled(tracingEnabled);
+    if (!tracingEnabled_) {
         return undefined;
     }
     return runTree;
@@ -39,7 +50,7 @@ const getTracingRunTree = (runTree) => {
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function traceable(wrappedFunc, config) {
-    const { aggregator, argsConfigPath, ...runTreeConfig } = config ?? {};
+    const { aggregator, argsConfigPath, tracingEnabled, ...runTreeConfig } = config ?? {};
     const traceableFunc = (...args) => {
         let currentRunTree;
         let rawInputs;
@@ -105,7 +116,7 @@ function traceable(wrappedFunc, config) {
             currentRunTree = new run_trees_js_1.RunTree(ensuredConfig);
             rawInputs = args;
         }
-        currentRunTree = getTracingRunTree(currentRunTree);
+        currentRunTree = getTracingRunTree(currentRunTree, tracingEnabled);
         let inputs;
         const firstInput = rawInputs[0];
         if (firstInput == null) {
@@ -173,6 +184,15 @@ function traceable(wrappedFunc, config) {
                         else {
                             await currentRunTree?.end({ outputs: finalOutputs });
                         }
+                        const onEnd = config?.on_end;
+                        if (onEnd) {
+                            if (!currentRunTree) {
+                                console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                            }
+                            else {
+                                onEnd(currentRunTree);
+                            }
+                        }
                         await postRunPromise;
                         await currentRunTree?.patchRun();
                     }
@@ -225,6 +245,15 @@ function traceable(wrappedFunc, config) {
                                 else {
                                     await currentRunTree?.end({ outputs: finalOutputs });
                                 }
+                                const onEnd = config?.on_end;
+                                if (onEnd) {
+                                    if (!currentRunTree) {
+                                        console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                    }
+                                    else {
+                                        onEnd(currentRunTree);
+                                    }
+                                }
                                 await postRunPromise;
                                 await currentRunTree?.patchRun();
                             }
@@ -234,6 +263,15 @@ function traceable(wrappedFunc, config) {
                     else {
                         try {
                             await currentRunTree?.end(isKVMap(rawOutput) ? rawOutput : { outputs: rawOutput });
+                            const onEnd = config?.on_end;
+                            if (onEnd) {
+                                if (!currentRunTree) {
+                                    console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                }
+                                else {
+                                    onEnd(currentRunTree);
+                                }
+                            }
                             await postRunPromise;
                             await currentRunTree?.patchRun();
                         }
@@ -246,6 +284,15 @@ function traceable(wrappedFunc, config) {
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 async (error) => {
                     await currentRunTree?.end(undefined, String(error));
+                    const onEnd = config?.on_end;
+                    if (onEnd) {
+                        if (!currentRunTree) {
+                            console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                        }
+                        else {
+                            onEnd(currentRunTree);
+                        }
+                    }
                     await postRunPromise;
                     await currentRunTree?.patchRun();
                     throw error;
@@ -302,3 +349,13 @@ function isKVMap(x) {
         // eslint-disable-next-line no-instanceof/no-instanceof
         !(x instanceof Date));
 }
+function wrapFunctionAndEnsureTraceable(target, options, name = "target") {
+    if (typeof target === "function") {
+        return traceable(target, {
+            ...options,
+            name,
+        });
+    }
+    throw new Error("Target must be runnable function");
+}
+exports.wrapFunctionAndEnsureTraceable = wrapFunctionAndEnsureTraceable;

package/dist/traceable.d.ts

@@ -48,6 +48,7 @@ export type TraceableFunction<Func extends (...args: any[]) => any> = Func exten
 export declare function traceable<Func extends (...args: any[]) => any>(wrappedFunc: Func, config?: Partial<RunTreeConfig> & {
     aggregator?: (args: any[]) => any;
     argsConfigPath?: [number] | [number, string];
+    tracingEnabled?: boolean;
 }): TraceableFunction<Func>;
 /**
  * Return the current run tree from within a traceable-wrapped function.
@@ -57,4 +58,5 @@ export declare function traceable<Func extends (...args: any[]) => any>(wrappedF
  */
 export declare function getCurrentRunTree(): RunTree;
 export declare function isTraceableFunction(x: unknown): x is TraceableFunction<any>;
+export declare function wrapFunctionAndEnsureTraceable<Func extends (...args: any[]) => any>(target: Func, options: Partial<RunTreeConfig>, name?: string): TraceableFunction<Func>;
 export {};

package/dist/traceable.js

@@ -12,10 +12,21 @@ const isAsyncIterable = (x) => x != null &&
     typeof x === "object" &&
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     typeof x[Symbol.asyncIterator] === "function";
-const getTracingRunTree = (runTree) => {
-    const tracingEnabled = getEnvironmentVariable("LANGSMITH_TRACING_V2") === "true" ||
-        getEnvironmentVariable("LANGCHAIN_TRACING_V2") === "true";
-    if (!tracingEnabled) {
+const tracingIsEnabled = (tracingEnabled) => {
+    if (tracingEnabled !== undefined) {
+        return tracingEnabled;
+    }
+    const envVars = [
+        "LANGSMITH_TRACING_V2",
+        "LANGCHAIN_TRACING_V2",
+        "LANGSMITH_TRACING",
+        "LANGCHAIN_TRACING",
+    ];
+    return Boolean(envVars.find((envVar) => getEnvironmentVariable(envVar) === "true"));
+};
+const getTracingRunTree = (runTree, tracingEnabled) => {
+    const tracingEnabled_ = tracingIsEnabled(tracingEnabled);
+    if (!tracingEnabled_) {
         return undefined;
     }
     return runTree;
@@ -36,7 +47,7 @@ const getTracingRunTree = (runTree) => {
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function traceable(wrappedFunc, config) {
-    const { aggregator, argsConfigPath, ...runTreeConfig } = config ?? {};
+    const { aggregator, argsConfigPath, tracingEnabled, ...runTreeConfig } = config ?? {};
     const traceableFunc = (...args) => {
         let currentRunTree;
         let rawInputs;
@@ -102,7 +113,7 @@ export function traceable(wrappedFunc, config) {
             currentRunTree = new RunTree(ensuredConfig);
             rawInputs = args;
         }
-        currentRunTree = getTracingRunTree(currentRunTree);
+        currentRunTree = getTracingRunTree(currentRunTree, tracingEnabled);
         let inputs;
         const firstInput = rawInputs[0];
         if (firstInput == null) {
@@ -170,6 +181,15 @@ export function traceable(wrappedFunc, config) {
                         else {
                             await currentRunTree?.end({ outputs: finalOutputs });
                         }
+                        const onEnd = config?.on_end;
+                        if (onEnd) {
+                            if (!currentRunTree) {
+                                console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                            }
+                            else {
+                                onEnd(currentRunTree);
+                            }
+                        }
                         await postRunPromise;
                         await currentRunTree?.patchRun();
                     }
@@ -222,6 +242,15 @@ export function traceable(wrappedFunc, config) {
                                 else {
                                     await currentRunTree?.end({ outputs: finalOutputs });
                                 }
+                                const onEnd = config?.on_end;
+                                if (onEnd) {
+                                    if (!currentRunTree) {
+                                        console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                    }
+                                    else {
+                                        onEnd(currentRunTree);
+                                    }
+                                }
                                 await postRunPromise;
                                 await currentRunTree?.patchRun();
                             }
@@ -231,6 +260,15 @@ export function traceable(wrappedFunc, config) {
                     else {
                         try {
                             await currentRunTree?.end(isKVMap(rawOutput) ? rawOutput : { outputs: rawOutput });
+                            const onEnd = config?.on_end;
+                            if (onEnd) {
+                                if (!currentRunTree) {
+                                    console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                                }
+                                else {
+                                    onEnd(currentRunTree);
+                                }
+                            }
                             await postRunPromise;
                             await currentRunTree?.patchRun();
                         }
@@ -243,6 +281,15 @@ export function traceable(wrappedFunc, config) {
                 // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 async (error) => {
                     await currentRunTree?.end(undefined, String(error));
+                    const onEnd = config?.on_end;
+                    if (onEnd) {
+                        if (!currentRunTree) {
+                            console.warn("Can not call 'on_end' if currentRunTree is undefined");
+                        }
+                        else {
+                            onEnd(currentRunTree);
+                        }
+                    }
                     await postRunPromise;
                     await currentRunTree?.patchRun();
                     throw error;
@@ -296,3 +343,12 @@ function isKVMap(x) {
         // eslint-disable-next-line no-instanceof/no-instanceof
         !(x instanceof Date));
 }
+export function wrapFunctionAndEnsureTraceable(target, options, name = "target") {
+    if (typeof target === "function") {
+        return traceable(target, {
+            ...options,
+            name,
+        });
+    }
+    throw new Error("Target must be runnable function");
+}