langchain 0.1.5 → 0.1.6

package/dist/smith/progress.cjs

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProgressBar = void 0;
+class ProgressBar {
+    constructor(props) {
+        Object.defineProperty(this, "total", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "current", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "barLength", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "format", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { total, format, barLength } = props;
+        this.total = total;
+        this.current = 0;
+        this.barLength = barLength ?? 40;
+        this.format = format || "{bar} {percentage}% | {value}/{total}";
+    }
+    initialize() {
+        this.update({ current: 0 });
+    }
+    update({ current, formatArgs, }) {
+        this.current = current;
+        const ratio = this.current / this.total;
+        const filledBarLength = Math.round(ratio * this.barLength);
+        const emptyBarLength = this.barLength - filledBarLength;
+        const filledBar = "▓".repeat(filledBarLength);
+        const emptyBar = "░".repeat(emptyBarLength);
+        const percentage = (ratio * 100).toFixed(2);
+        let formattedString = this.format
+            .replace("{bar}", `${filledBar}${emptyBar}`)
+            .replace("{percentage}", percentage)
+            .replace("{value}", this.current.toString())
+            .replace("{total}", this.total.toString());
+        if (formatArgs) {
+            for (const key in formatArgs) {
+                if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {
+                    formattedString = formattedString.replace(`{${key}}`, formatArgs[key].toString());
+                }
+            }
+        }
+        console.log(formattedString);
+    }
+    increment({ formatArgs, } = {}) {
+        this.update({ current: this.current + 1, formatArgs });
+    }
+    complete({ formatArgs } = {}) {
+        this.update({ current: this.total, formatArgs });
+        console.log("\nCompleted");
+    }
+}
+exports.ProgressBar = ProgressBar;

package/dist/smith/progress.d.ts

@@ -0,0 +1,22 @@
+export declare class ProgressBar {
+    total: number;
+    current: number;
+    barLength: number;
+    format: string;
+    constructor(props: {
+        total: number;
+        format?: string;
+        barLength?: number;
+    });
+    initialize(): void;
+    update({ current, formatArgs, }: {
+        current: number;
+        formatArgs?: Record<string, string>;
+    }): void;
+    increment({ formatArgs, }?: {
+        formatArgs?: Record<string, string>;
+    }): void;
+    complete({ formatArgs }?: {
+        formatArgs?: Record<string, string>;
+    }): void;
+}

package/dist/smith/progress.js

@@ -0,0 +1,65 @@
+export class ProgressBar {
+    constructor(props) {
+        Object.defineProperty(this, "total", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "current", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "barLength", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "format", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { total, format, barLength } = props;
+        this.total = total;
+        this.current = 0;
+        this.barLength = barLength ?? 40;
+        this.format = format || "{bar} {percentage}% | {value}/{total}";
+    }
+    initialize() {
+        this.update({ current: 0 });
+    }
+    update({ current, formatArgs, }) {
+        this.current = current;
+        const ratio = this.current / this.total;
+        const filledBarLength = Math.round(ratio * this.barLength);
+        const emptyBarLength = this.barLength - filledBarLength;
+        const filledBar = "▓".repeat(filledBarLength);
+        const emptyBar = "░".repeat(emptyBarLength);
+        const percentage = (ratio * 100).toFixed(2);
+        let formattedString = this.format
+            .replace("{bar}", `${filledBar}${emptyBar}`)
+            .replace("{percentage}", percentage)
+            .replace("{value}", this.current.toString())
+            .replace("{total}", this.total.toString());
+        if (formatArgs) {
+            for (const key in formatArgs) {
+                if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {
+                    formattedString = formattedString.replace(`{${key}}`, formatArgs[key].toString());
+                }
+            }
+        }
+        console.log(formattedString);
+    }
+    increment({ formatArgs, } = {}) {
+        this.update({ current: this.current + 1, formatArgs });
+    }
+    complete({ formatArgs } = {}) {
+        this.update({ current: this.total, formatArgs });
+        console.log("\nCompleted");
+    }
+}

package/dist/smith/runner_utils.cjs

@@ -0,0 +1,353 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runOnDataset = void 0;
+const messages_1 = require("@langchain/core/messages");
+const runnables_1 = require("@langchain/core/runnables");
+const run_collector_1 = require("@langchain/core/tracers/run_collector");
+const tracer_langchain_1 = require("@langchain/core/tracers/tracer_langchain");
+const langsmith_1 = require("langsmith");
+const loader_js_1 = require("../evaluation/loader.cjs");
+const name_generation_js_1 = require("./name_generation.cjs");
+const progress_js_1 = require("./progress.cjs");
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "evaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluator = new runnables_1.RunnableLambda({ func: evaluator });
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that resolves to the evaluation result.
+     */
+    async evaluateRun(run, example) {
+        return await this.evaluator.invoke({ run, example });
+    }
+}
+function isLLMStringEvaluator(evaluator) {
+    return evaluator && typeof evaluator.evaluateStrings === "function";
+}
+/**
+ * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])
+ * and composes with a prepareData function so the user can prepare the trace and
+ * dataset data for the evaluator.
+ */
+class PreparedRunEvaluator {
+    constructor(evaluator, evaluationName, formatEvaluatorInputs) {
+        Object.defineProperty(this, "evaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "formatEvaluatorInputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isStringEvaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluator = evaluator;
+        this.isStringEvaluator = typeof evaluator?.evaluateStrings === "function";
+        this.evaluationName = evaluationName;
+        this.formatEvaluatorInputs = formatEvaluatorInputs;
+    }
+    static async fromEvalConfig(config) {
+        const evaluatorType = typeof config === "string" ? config : config.evaluatorType;
+        const evalConfig = typeof config === "string" ? {} : config;
+        const evaluator = await (0, loader_js_1.loadEvaluator)(evaluatorType, evalConfig);
+        const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
+        }
+        if (!isLLMStringEvaluator(evaluator)) {
+            throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
+                "Please use a string evaluator, or implement your " +
+                "evaluation logic as a customEvaluator.");
+        }
+        return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that resolves to the evaluation result.
+     */
+    async evaluateRun(run, example) {
+        const { prediction, input, reference } = this.formatEvaluatorInputs({
+            rawInput: run.inputs,
+            rawPrediction: run.outputs,
+            rawReferenceOutput: example?.outputs,
+            run,
+        });
+        if (this.isStringEvaluator) {
+            const evalResult = await this.evaluator.evaluateStrings({
+                prediction: prediction,
+                reference: reference,
+                input: input,
+            });
+            return {
+                key: this.evaluationName,
+                comment: evalResult?.reasoning,
+                ...evalResult,
+            };
+        }
+        throw new Error("Evaluator not yet supported. " +
+            "Please use a string evaluator, or implement your " +
+            "evaluation logic as a customEvaluator.");
+    }
+}
+class LoadedEvalConfig {
+    constructor(evaluators) {
+        Object.defineProperty(this, "evaluators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: evaluators
+        });
+    }
+    static async fromRunEvalConfig(config) {
+        // Custom evaluators are applied "as-is"
+        const customEvaluators = config?.customEvaluators?.map((evaluator) => {
+            if (typeof evaluator === "function") {
+                return new DynamicRunEvaluator(evaluator);
+            }
+            else {
+                return evaluator;
+            }
+        });
+        const offTheShelfEvaluators = await Promise.all(config?.evaluators?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
+        return new LoadedEvalConfig((customEvaluators ?? []).concat(offTheShelfEvaluators ?? []));
+    }
+}
+/**
+ * Internals expect a constructor () -> Runnable. This function wraps/coerces
+ * the provided LangChain object, custom function, or factory function into
+ * a constructor of a runnable.
+ * @param modelOrFactory The model or factory to create a wrapped model from.
+ * @returns A function that returns the wrapped model.
+ * @throws Error if the modelOrFactory is invalid.
+ */
+const createWrappedModel = async (modelOrFactory) => {
+    if (runnables_1.Runnable.isRunnable(modelOrFactory)) {
+        return () => modelOrFactory;
+    }
+    if (typeof modelOrFactory === "function") {
+        try {
+            // If it works with no arguments, assume it's a factory
+            let res = modelOrFactory();
+            if (res &&
+                typeof res.then === "function") {
+                res = await res;
+            }
+            return modelOrFactory;
+        }
+        catch (err) {
+            // Otherwise, it's a custom UDF, and we'll wrap
+            // in a lambda
+            const wrappedModel = new runnables_1.RunnableLambda({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
+    }
+    throw new Error("Invalid modelOrFactory");
+};
+const loadExamples = async ({ datasetName, client, projectName, }) => {
+    const exampleIterator = client.listExamples({ datasetName });
+    const configs = [];
+    const runCollectors = [];
+    const examples = [];
+    for await (const example of exampleIterator) {
+        const runCollector = new run_collector_1.RunCollectorCallbackHandler({
+            exampleId: example.id,
+        });
+        configs.push({
+            callbacks: [
+                new tracer_langchain_1.LangChainTracer({ exampleId: example.id, projectName }),
+                runCollector,
+            ],
+        });
+        examples.push(example);
+        runCollectors.push(runCollector);
+    }
+    return {
+        configs,
+        examples,
+        runCollectors,
+    };
+};
+const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
+    // TODO: Parallelize and/or put in callbacks to speed up evals.
+    const { evaluators } = evaluation;
+    const progress = new progress_js_1.ProgressBar({
+        total: examples.length,
+        format: "Running Evaluators: {bar} {percentage}% | {value}/{total}\n",
+    });
+    const results = {};
+    for (let i = 0; i < runs.length; i += 1) {
+        const run = runs[i];
+        const example = examples[i];
+        const evaluatorResults = await Promise.all(evaluators.map((evaluator) => client.evaluateRun(run, evaluator, {
+            referenceExample: example,
+            loadChildRuns: false,
+        })));
+        progress.increment();
+        results[example.id] = {
+            execution_time: run?.end_time && run.start_time
+                ? run.end_time - run.start_time
+                : undefined,
+            feedback: evaluatorResults,
+            run_id: run.id,
+        };
+    }
+    return results;
+};
+const getExamplesInputs = (examples, chainOrFactory, dataType) => {
+    if (dataType === "chat") {
+        // For some batty reason, we store the chat dataset differently.
+        // { type: "system", data: { content: inputs.input } },
+        // But we need to create AIMesage, SystemMessage, etc.
+        return examples.map(({ inputs }) => (0, messages_1.mapStoredMessagesToChatMessages)(inputs.input));
+    }
+    // If it's a language model and ALL example inputs have a single value,
+    // then we can be friendly and flatten the inputs to a list of strings.
+    const isLanguageModel = typeof chainOrFactory === "object" &&
+        typeof chainOrFactory._llmType === "function";
+    if (isLanguageModel &&
+        examples.every(({ inputs }) => Object.keys(inputs).length === 1)) {
+        return examples.map(({ inputs }) => Object.values(inputs)[0]);
+    }
+    return examples.map(({ inputs }) => inputs);
+};
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluation` (RunEvalConfig): Configuration for the evaluation, including
+ *     standard and custom evaluators.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangChain service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const evaluationConfig = new RunEvalConfig({
+ *     evaluators: [/* ...evaluators... *\//],
+ *     customEvaluators: [/* ...custom evaluators... *\//],
+ *   });
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluationConfig,
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
+const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }) => {
+    const wrappedModel = await createWrappedModel(chainOrFactory);
+    const testClient = client ?? new langsmith_1.Client();
+    const testProjectName = projectName ?? (0, name_generation_js_1.randomName)();
+    const dataset = await testClient.readDataset({ datasetName });
+    const datasetId = dataset.id;
+    const testConcurrency = maxConcurrency ?? 5;
+    const { configs, examples, runCollectors } = await loadExamples({
+        datasetName,
+        client: testClient,
+        projectName: testProjectName,
+        maxConcurrency: testConcurrency,
+    });
+    await testClient.createProject({
+        projectName: testProjectName,
+        referenceDatasetId: datasetId,
+        projectExtra: { metadata: { ...projectMetadata } },
+    });
+    const wrappedRunnable = new runnables_1.RunnableLambda({
+        func: wrappedModel,
+    }).withConfig({ runName: "evaluationRun" });
+    const runInputs = getExamplesInputs(examples, chainOrFactory, dataset.data_type);
+    const progress = new progress_js_1.ProgressBar({
+        total: runInputs.length,
+        format: "Predicting: {bar} {percentage}% | {value}/{total}",
+    });
+    // TODO: Collect the runs as well.
+    await wrappedRunnable
+        .withListeners({
+        onEnd: () => progress.increment(),
+    })
+        // TODO: Insert evaluation inline for immediate feedback.
+        .batch(runInputs, configs, {
+        maxConcurrency,
+        returnExceptions: true,
+    });
+    progress.complete();
+    const runs = [];
+    for (let i = 0; i < examples.length; i += 1) {
+        runs.push(runCollectors[i].tracedRuns[0]);
+    }
+    let evalResults = {};
+    if (evaluationConfig) {
+        const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);
+        evalResults = await applyEvaluators({
+            evaluation: loadedEvalConfig,
+            runs,
+            examples,
+            client: testClient,
+        });
+    }
+    const results = {
+        projectName: testProjectName,
+        results: evalResults ?? {},
+    };
+    return results;
+};
+exports.runOnDataset = runOnDataset;

package/dist/smith/runner_utils.d.ts

@@ -0,0 +1,77 @@
+import { Runnable } from "@langchain/core/runnables";
+import { Client, Feedback } from "langsmith";
+import { RunEvalConfig } from "./config.js";
+export type ChainOrFactory = Runnable | (() => Runnable) | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
+export type RunOnDatasetParams = {
+    evaluationConfig?: RunEvalConfig;
+    projectMetadata?: Record<string, unknown>;
+    projectName?: string;
+    client?: Client;
+    maxConcurrency?: number;
+};
+export type EvalResults = {
+    projectName: string;
+    results: {
+        [key: string]: {
+            execution_time?: number;
+            run_id: string;
+            feedback: Feedback[];
+        };
+    };
+};
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluation` (RunEvalConfig): Configuration for the evaluation, including
+ *     standard and custom evaluators.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangChain service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const evaluationConfig = new RunEvalConfig({
+ *     evaluators: [/* ...evaluators... *\//],
+ *     customEvaluators: [/* ...custom evaluators... *\//],
+ *   });
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluationConfig,
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
+export declare const runOnDataset: (chainOrFactory: ChainOrFactory, datasetName: string, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }: RunOnDatasetParams) => Promise<EvalResults>;