langsmith 0.0.1-0

package/dist/evaluation/evaluator.d.ts

@@ -0,0 +1,12 @@
+import { Example, KVMap, Run, ScoreType, ValueType } from "../schemas.js";
+export interface EvaluationResult {
+    key: string;
+    score?: ScoreType;
+    value?: ValueType;
+    comment?: string;
+    correction?: string | object;
+    evaluatorInfo?: KVMap;
+}
+export interface RunEvaluator {
+    evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;
+}

package/dist/evaluation/evaluator.js

@@ -0,0 +1 @@
+export {};

package/dist/evaluation/index.cjs

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StringEvaluator = void 0;
+var string_evaluator_js_1 = require("./string_evaluator.cjs");
+Object.defineProperty(exports, "StringEvaluator", { enumerable: true, get: function () { return string_evaluator_js_1.StringEvaluator; } });

package/dist/evaluation/index.d.ts

@@ -0,0 +1,2 @@
+export { RunEvaluator, EvaluationResult } from "./evaluator.js";
+export { StringEvaluator, GradingFunctionParams, GradingFunctionResult, } from "./string_evaluator.js";

package/dist/evaluation/index.js

@@ -0,0 +1 @@
+export { StringEvaluator, } from "./string_evaluator.js";

package/dist/evaluation/string_evaluator.cjs

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StringEvaluator = void 0;
+class StringEvaluator {
+    constructor(params) {
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inputKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "predictionKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "answerKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "gradingFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluationName = params.evaluationName;
+        this.inputKey = params.inputKey ?? "input";
+        this.predictionKey = params.predictionKey ?? "output";
+        this.answerKey =
+            params.answerKey !== undefined ? params.answerKey : "output";
+        this.gradingFunction = params.gradingFunction;
+    }
+    async evaluateRun(run, example) {
+        if (!run.outputs) {
+            throw new Error("Run outputs cannot be undefined.");
+        }
+        const functionInputs = {
+            input: run.inputs[this.inputKey],
+            prediction: run.outputs[this.predictionKey],
+            answer: this.answerKey ? example?.outputs?.[this.answerKey] : null,
+        };
+        const gradingResults = await this.gradingFunction(functionInputs);
+        const key = gradingResults.key || this.evaluationName;
+        if (!key) {
+            throw new Error("Evaluation name cannot be undefined.");
+        }
+        return {
+            key,
+            score: gradingResults.score,
+            value: gradingResults.value,
+            comment: gradingResults.comment,
+            correction: gradingResults.correction,
+        };
+    }
+}
+exports.StringEvaluator = StringEvaluator;

package/dist/evaluation/string_evaluator.d.ts

@@ -0,0 +1,30 @@
+import { Example, Run, ScoreType, ValueType } from "../schemas.js";
+import { EvaluationResult, RunEvaluator } from "./evaluator.js";
+export interface GradingFunctionResult {
+    key?: string;
+    score?: ScoreType;
+    value?: ValueType;
+    comment?: string;
+    correction?: string | object;
+}
+export interface GradingFunctionParams {
+    input: string;
+    prediction: string;
+    answer?: string;
+}
+export interface StringEvaluatorParams {
+    evaluationName?: string;
+    inputKey?: string;
+    predictionKey?: string;
+    answerKey?: string;
+    gradingFunction: (params: GradingFunctionParams) => Promise<GradingFunctionResult>;
+}
+export declare class StringEvaluator implements RunEvaluator {
+    protected evaluationName?: string;
+    protected inputKey: string;
+    protected predictionKey: string;
+    protected answerKey?: string;
+    protected gradingFunction: (params: GradingFunctionParams) => Promise<GradingFunctionResult>;
+    constructor(params: StringEvaluatorParams);
+    evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;
+}

package/dist/evaluation/string_evaluator.js

@@ -0,0 +1,62 @@
+export class StringEvaluator {
+    constructor(params) {
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inputKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "predictionKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "answerKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "gradingFunction", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluationName = params.evaluationName;
+        this.inputKey = params.inputKey ?? "input";
+        this.predictionKey = params.predictionKey ?? "output";
+        this.answerKey =
+            params.answerKey !== undefined ? params.answerKey : "output";
+        this.gradingFunction = params.gradingFunction;
+    }
+    async evaluateRun(run, example) {
+        if (!run.outputs) {
+            throw new Error("Run outputs cannot be undefined.");
+        }
+        const functionInputs = {
+            input: run.inputs[this.inputKey],
+            prediction: run.outputs[this.predictionKey],
+            answer: this.answerKey ? example?.outputs?.[this.answerKey] : null,
+        };
+        const gradingResults = await this.gradingFunction(functionInputs);
+        const key = gradingResults.key || this.evaluationName;
+        if (!key) {
+            throw new Error("Evaluation name cannot be undefined.");
+        }
+        return {
+            key,
+            score: gradingResults.score,
+            value: gradingResults.value,
+            comment: gradingResults.comment,
+            correction: gradingResults.correction,
+        };
+    }
+}

package/dist/index.cjs

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunTree = exports.Client = void 0;
+var client_js_1 = require("./client.cjs");
+Object.defineProperty(exports, "Client", { enumerable: true, get: function () { return client_js_1.Client; } });
+var run_trees_js_1 = require("./run_trees.cjs");
+Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { Client } from "./client.js";
+export { Dataset, Example, TracerSession, Run, Feedback } from "./schemas.js";
+export { RunTree } from "./run_trees.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { Client } from "./client.js";
+export { RunTree } from "./run_trees.js";

package/dist/run_trees.cjs

@@ -0,0 +1,232 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunTree = void 0;
+const uuid = __importStar(require("uuid"));
+const env_js_1 = require("./utils/env.cjs");
+const client_js_1 = require("./client.cjs");
+class RunTree {
+    constructor(config) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "run_type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "project_name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parent_run", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "child_runs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "execution_order", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "child_execution_order", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "start_time", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "end_time", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extra", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "error", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "serialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "outputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "reference_example_id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const defaultConfig = RunTree.getDefaultConfig();
+        Object.assign(this, { ...defaultConfig, ...config });
+    }
+    static getDefaultConfig() {
+        return {
+            id: uuid.v4(),
+            project_name: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_PROJECT") ??
+                (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_SESSION") ?? // TODO: Deprecate
+                "default",
+            child_runs: [],
+            execution_order: 1,
+            child_execution_order: 1,
+            api_url: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_ENDPOINT") ?? "http://localhost:1984",
+            api_key: (0, env_js_1.getEnvironmentVariable)("LANGCHAIN_API_KEY"),
+            caller_options: {},
+            start_time: Date.now(),
+            end_time: Date.now(),
+            serialized: {},
+            inputs: {},
+            extra: {},
+            client: new client_js_1.Client({}),
+        };
+    }
+    async createChild(config) {
+        const child = new RunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+            execution_order: this.child_execution_order + 1,
+            child_execution_order: this.child_execution_order + 1,
+        });
+        this.child_runs.push(child);
+        return child;
+    }
+    async end(outputs, error, endTime = Date.now()) {
+        this.outputs = outputs;
+        this.error = error;
+        this.end_time = endTime;
+        if (this.parent_run) {
+            this.parent_run.child_execution_order = Math.max(this.parent_run.child_execution_order, this.child_execution_order);
+        }
+    }
+    async _convertToCreate(run, excludeChildRuns = true) {
+        const runExtra = run.extra ?? {};
+        if (!runExtra.runtime) {
+            runExtra.runtime = {};
+        }
+        const runtimeEnv = await (0, env_js_1.getRuntimeEnvironment)();
+        for (const [k, v] of Object.entries(runtimeEnv)) {
+            if (!runExtra.runtime[k]) {
+                runExtra.runtime[k] = v;
+            }
+        }
+        let child_runs;
+        let parent_run_id;
+        if (!excludeChildRuns) {
+            child_runs = await Promise.all(run.child_runs.map((child_run) => this._convertToCreate(child_run, excludeChildRuns)));
+            parent_run_id = undefined;
+        }
+        else {
+            parent_run_id = run.parent_run?.id;
+            child_runs = [];
+        }
+        const persistedRun = {
+            id: run.id,
+            name: run.name,
+            start_time: run.start_time,
+            end_time: run.end_time,
+            run_type: run.run_type,
+            reference_example_id: run.reference_example_id,
+            extra: runExtra,
+            execution_order: run.execution_order,
+            serialized: run.serialized,
+            error: run.error,
+            inputs: run.inputs,
+            outputs: run.outputs,
+            session_name: run.project_name,
+            child_runs: child_runs,
+            parent_run_id: parent_run_id,
+        };
+        return persistedRun;
+    }
+    async postRun(excludeChildRuns = true) {
+        const runCreate = await this._convertToCreate(this, excludeChildRuns);
+        await this.client.createRun(runCreate);
+    }
+    async patchRun() {
+        const runUpdate = {
+            end_time: this.end_time,
+            error: this.error,
+            outputs: this.outputs,
+            parent_run_id: this.parent_run?.id,
+            reference_example_id: this.reference_example_id,
+        };
+        await this.client.updateRun(this.id, runUpdate);
+    }
+}
+exports.RunTree = RunTree;

package/dist/run_trees.d.ts

@@ -0,0 +1,47 @@
+import { BaseRun, KVMap, RunType } from "./schemas.js";
+import { Client } from "./client.js";
+export interface RunTreeConfig {
+    name: string;
+    run_type: RunType;
+    id?: string;
+    project_name?: string;
+    execution_order?: number;
+    child_execution_order?: number;
+    parent_run?: RunTree;
+    child_runs?: RunTree[];
+    start_time?: number;
+    end_time?: number;
+    extra?: KVMap;
+    error?: string;
+    serialized?: object;
+    inputs?: KVMap;
+    outputs?: KVMap;
+    reference_example_id?: string;
+    client?: Client;
+}
+export declare class RunTree implements BaseRun {
+    id: string;
+    name: RunTreeConfig["name"];
+    run_type: RunTreeConfig["run_type"];
+    project_name: string;
+    parent_run?: RunTree;
+    child_runs: RunTree[];
+    execution_order: number;
+    child_execution_order: number;
+    start_time: number;
+    end_time: number;
+    extra: KVMap;
+    error?: string;
+    serialized: object;
+    inputs: KVMap;
+    outputs?: KVMap;
+    reference_example_id?: string;
+    client: Client;
+    constructor(config: RunTreeConfig);
+    private static getDefaultConfig;
+    createChild(config: RunTreeConfig): Promise<RunTree>;
+    end(outputs?: KVMap, error?: string, endTime?: number): Promise<void>;
+    private _convertToCreate;
+    postRun(excludeChildRuns?: boolean): Promise<void>;
+    patchRun(): Promise<void>;
+}

package/dist/run_trees.js

@@ -0,0 +1,205 @@
+import * as uuid from "uuid";
+import { getEnvironmentVariable, getRuntimeEnvironment } from "./utils/env.js";
+import { Client } from "./client.js";
+export class RunTree {
+    constructor(config) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "run_type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "project_name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parent_run", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "child_runs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "execution_order", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "child_execution_order", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "start_time", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "end_time", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extra", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "error", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "serialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "outputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "reference_example_id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const defaultConfig = RunTree.getDefaultConfig();
+        Object.assign(this, { ...defaultConfig, ...config });
+    }
+    static getDefaultConfig() {
+        return {
+            id: uuid.v4(),
+            project_name: getEnvironmentVariable("LANGCHAIN_PROJECT") ??
+                getEnvironmentVariable("LANGCHAIN_SESSION") ?? // TODO: Deprecate
+                "default",
+            child_runs: [],
+            execution_order: 1,
+            child_execution_order: 1,
+            api_url: getEnvironmentVariable("LANGCHAIN_ENDPOINT") ?? "http://localhost:1984",
+            api_key: getEnvironmentVariable("LANGCHAIN_API_KEY"),
+            caller_options: {},
+            start_time: Date.now(),
+            end_time: Date.now(),
+            serialized: {},
+            inputs: {},
+            extra: {},
+            client: new Client({}),
+        };
+    }
+    async createChild(config) {
+        const child = new RunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+            execution_order: this.child_execution_order + 1,
+            child_execution_order: this.child_execution_order + 1,
+        });
+        this.child_runs.push(child);
+        return child;
+    }
+    async end(outputs, error, endTime = Date.now()) {
+        this.outputs = outputs;
+        this.error = error;
+        this.end_time = endTime;
+        if (this.parent_run) {
+            this.parent_run.child_execution_order = Math.max(this.parent_run.child_execution_order, this.child_execution_order);
+        }
+    }
+    async _convertToCreate(run, excludeChildRuns = true) {
+        const runExtra = run.extra ?? {};
+        if (!runExtra.runtime) {
+            runExtra.runtime = {};
+        }
+        const runtimeEnv = await getRuntimeEnvironment();
+        for (const [k, v] of Object.entries(runtimeEnv)) {
+            if (!runExtra.runtime[k]) {
+                runExtra.runtime[k] = v;
+            }
+        }
+        let child_runs;
+        let parent_run_id;
+        if (!excludeChildRuns) {
+            child_runs = await Promise.all(run.child_runs.map((child_run) => this._convertToCreate(child_run, excludeChildRuns)));
+            parent_run_id = undefined;
+        }
+        else {
+            parent_run_id = run.parent_run?.id;
+            child_runs = [];
+        }
+        const persistedRun = {
+            id: run.id,
+            name: run.name,
+            start_time: run.start_time,
+            end_time: run.end_time,
+            run_type: run.run_type,
+            reference_example_id: run.reference_example_id,
+            extra: runExtra,
+            execution_order: run.execution_order,
+            serialized: run.serialized,
+            error: run.error,
+            inputs: run.inputs,
+            outputs: run.outputs,
+            session_name: run.project_name,
+            child_runs: child_runs,
+            parent_run_id: parent_run_id,
+        };
+        return persistedRun;
+    }
+    async postRun(excludeChildRuns = true) {
+        const runCreate = await this._convertToCreate(this, excludeChildRuns);
+        await this.client.createRun(runCreate);
+    }
+    async patchRun() {
+        const runUpdate = {
+            end_time: this.end_time,
+            error: this.error,
+            outputs: this.outputs,
+            parent_run_id: this.parent_run?.id,
+            reference_example_id: this.reference_example_id,
+        };
+        await this.client.updateRun(this.id, runUpdate);
+    }
+}