langchain 0.0.147 → 0.0.149

package/dist/evaluation/agents/trajectory.js

@@ -0,0 +1,184 @@
+import { BaseLLMOutputParser } from "../../schema/output_parser.js";
+import { AgentTrajectoryEvaluator, } from "../base.js";
+import { RUN_KEY, } from "../../schema/index.js";
+import { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from "./prompt.js";
+/**
+ * A parser for the output of the TrajectoryEvalChain.
+ */
+export class TrajectoryOutputParser extends BaseLLMOutputParser {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "evaluation", "agents"]
+        });
+    }
+    static lc_name() {
+        return "TrajectoryOutputParser";
+    }
+    parseResult(generations, _callbacks) {
+        const { text } = generations[0];
+        if (!text.includes("Score:")) {
+            throw new Error(`Could not find score in model eval output: ${text}`);
+        }
+        let [reasoning, scoreStr] = text.split("Score:", 2);
+        reasoning = reasoning.trim();
+        scoreStr = scoreStr.trim();
+        // Use regex to extract the score.
+        // This will get the number in the string, even if it is a float or more than 10.
+        // E.g. "Score: 1" will return 1, "Score: 3.5" will return 3.5, and
+        // "Score: 10" will return 10.
+        // The score should be an integer digit in the range 1-5.
+        const scoreMatch = scoreStr.match(/(\d+(\.\d+)?)/);
+        if (scoreMatch === null || scoreMatch[1].includes(".")) {
+            throw new Error(`Score is not an integer digit in the range 1-5: ${text}`);
+        }
+        const score = +scoreMatch[1];
+        if (score < 1 || score > 5) {
+            throw new Error(`Score is not a digit in the range 1-5: ${text}`);
+        }
+        const normalizedScore = (score - 1) / 4;
+        return Promise.resolve({
+            reasoning,
+            score: normalizedScore,
+        });
+    }
+}
+/**
+ * A chain for evaluating ReAct style agents.
+ *
+ * This chain is used to evaluate ReAct style agents by reasoning about
+ * the sequence of actions taken and their outcomes.
+ */
+export class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "criterionName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.criterionName
+        });
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "outputParser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new TrajectoryOutputParser()
+        });
+    }
+    static lc_name() {
+        return "TrajectoryEvalChain";
+    }
+    static resolveTrajectoryPrompt(prompt, agentTools) {
+        let _prompt;
+        if (prompt) {
+            _prompt = prompt;
+        }
+        else if (agentTools) {
+            _prompt = EVAL_CHAT_PROMPT;
+        }
+        else {
+            _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;
+        }
+        return _prompt;
+    }
+    /**
+     * Get the description of the agent tools.
+     *
+     * @returns The description of the agent tools.
+     */
+    static toolsDescription(agentTools) {
+        return agentTools
+            .map((tool, i) => `Tool ${i + 1}: ${tool.name}\n Description: ${tool.description}`)
+            .join("\n\n");
+    }
+    /**
+     * Create a new TrajectoryEvalChain.
+     * @param llm
+     * @param agentTools - The tools used by the agent.
+     * @param chainOptions - The options for the chain.
+     */
+    static async fromLLM(llm, agentTools, chainOptions) {
+        let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);
+        if (agentTools) {
+            const toolDescriptions = this.toolsDescription(agentTools);
+            prompt = await prompt.partial({ toolDescriptions });
+        }
+        const options = chainOptions;
+        if (options) {
+            // remove prompt from chainOptions
+            delete options.prompt;
+        }
+        return new this({
+            llm,
+            prompt,
+            ...options,
+        });
+    }
+    _prepareOutput(result) {
+        const parsed = result[this.outputKey];
+        if (RUN_KEY in result && result[RUN_KEY]) {
+            parsed[RUN_KEY] = result[RUN_KEY];
+        }
+        return parsed;
+    }
+    /**
+     * Get the agent trajectory as a formatted string.
+     *
+     * @param steps - The agent trajectory.
+     * @returns The formatted agent trajectory.
+     */
+    getAgentTrajectory(steps) {
+        return steps
+            .map((step, i) => {
+            const { action, observation } = step;
+            return (`Step ${i + 1}:\n` +
+                `Tool used: ${action.tool}\n` +
+                `Tool input: ${action.toolInput}\n` +
+                `Tool output: ${observation}`);
+        })
+            .join("\n\n");
+    }
+    formatReference(reference) {
+        if (!reference) {
+            return "";
+        }
+        return `
+The following is the expected answer. Use this to measure correctness:
+[GROUND_TRUTH]
+${reference}
+[END_GROUND_TRUTH]
+        `;
+    }
+    async _evaluateAgentTrajectory(args, callOptions, config) {
+        const { input, prediction, reference, agentTrajectory } = args;
+        const inputs = {
+            question: input,
+            agentTrajectory: this.getAgentTrajectory(agentTrajectory),
+            answer: prediction,
+            reference: this.formatReference(reference),
+        };
+        const result = await this.call({ ...inputs, ...callOptions }, config);
+        return this._prepareOutput(result);
+    }
+}

package/dist/evaluation/base.cjs

@@ -0,0 +1,274 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AgentTrajectoryEvaluator = exports.LLMPairwiseStringEvaluator = exports.PairwiseStringEvaluator = exports.StringEvaluator = exports.LLMStringEvaluator = exports.EvalChain = exports.LLMEvalChain = exports.eqSet = void 0;
+const index_js_1 = require("../chains/index.cjs");
+/**
+ * Compare two sets for equality
+ *
+ * @param xs
+ * @param ys
+ */
+const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+exports.eqSet = eqSet;
+/**
+ * Base llm chain class for evaluators.
+ */
+class LLMEvalChain extends index_js_1.LLMChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "skipInputWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring input in ${this.constructor.name}, as it is not expected.`
+        });
+        Object.defineProperty(this, "skipReferenceWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring reference in ${this.constructor.name}, as it is not expected.`
+        });
+    }
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference, input) {
+        if (this.requiresInput && input == null) {
+            throw new Error(`${this.constructor.name} requires an input string.`);
+        }
+        else if (input != null && !this.requiresInput) {
+            console.warn(this.skipInputWarning);
+        }
+        if (this.requiresReference && reference == null) {
+            throw new Error(`${this.constructor.name} requires a reference string.`);
+        }
+        else if (reference != null && !this.requiresReference) {
+            console.warn(this.skipReferenceWarning);
+        }
+    }
+}
+exports.LLMEvalChain = LLMEvalChain;
+/**
+ * Base chain class for evaluators.
+ */
+class EvalChain extends index_js_1.BaseChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "skipInputWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring input in ${this.constructor.name}, as it is not expected.`
+        });
+        Object.defineProperty(this, "skipReferenceWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring reference in ${this.constructor.name}, as it is not expected.`
+        });
+    }
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference, input) {
+        if (this.requiresInput && input == null) {
+            throw new Error(`${this.constructor.name} requires an input string.`);
+        }
+        else if (input != null && !this.requiresInput) {
+            console.warn(this.skipInputWarning);
+        }
+        if (this.requiresReference && reference == null) {
+            throw new Error(`${this.constructor.name} requires a reference string.`);
+        }
+        else if (reference != null && !this.requiresReference) {
+            console.warn(this.skipReferenceWarning);
+        }
+    }
+}
+exports.EvalChain = EvalChain;
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+class LLMStringEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateStrings(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStrings(args, callOptions, config);
+    }
+}
+exports.LLMStringEvaluator = LLMStringEvaluator;
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+class StringEvaluator extends EvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param config
+     */
+    evaluateStrings(args, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStrings(args, config);
+    }
+}
+exports.StringEvaluator = StringEvaluator;
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+class PairwiseStringEvaluator extends EvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args, config) {
+        return this._evaluateStringPairs(args, config);
+    }
+}
+exports.PairwiseStringEvaluator = PairwiseStringEvaluator;
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+class LLMPairwiseStringEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param callOptions
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStringPairs(args, callOptions, config);
+    }
+}
+exports.LLMPairwiseStringEvaluator = LLMPairwiseStringEvaluator;
+/**
+ * Interface for evaluating agent trajectories.
+ */
+class AgentTrajectoryEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate a trajectory.
+     * @return The evaluation result.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateAgentTrajectory(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateAgentTrajectory(args, callOptions, config);
+    }
+}
+exports.AgentTrajectoryEvaluator = AgentTrajectoryEvaluator;

package/dist/evaluation/base.d.ts

@@ -0,0 +1,232 @@
+import { BaseChain, LLMChain, LLMChainInput } from "../chains/index.js";
+import { AgentStep, ChainValues } from "../schema/index.js";
+import { BaseLanguageModel } from "../base_language/index.js";
+import { Callbacks } from "../callbacks/index.js";
+import { BaseCallbackConfig } from "../callbacks/manager.js";
+/**
+ * Base input for evaluators.
+ */
+export interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModel = BaseLanguageModel> extends LLMChainInput<T, L> {
+}
+/**
+ * Compare two sets for equality
+ *
+ * @param xs
+ * @param ys
+ */
+export declare const eqSet: (xs: Set<string>, ys: Set<string>) => boolean;
+/**
+ * The type of the output of an evaluation evaluator.
+ */
+export type EvalOutputType = Record<string, string | number | boolean>;
+/**
+ * Base llm chain class for evaluators.
+ */
+export declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModel = BaseLanguageModel> extends LLMChain<T, L> {
+    requiresInput?: boolean;
+    requiresReference?: boolean;
+    skipInputWarning?: string;
+    skipReferenceWarning?: string;
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * Base chain class for evaluators.
+ */
+export declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {
+    requiresInput?: boolean;
+    requiresReference?: boolean;
+    skipInputWarning?: string;
+    skipReferenceWarning?: string;
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * @field prediction The output string from the  model.
+ * @field reference The expected output / reference string.
+ * @field input The input string.
+ */
+export interface StringEvaluatorArgs {
+    prediction: string;
+    reference?: string;
+    input?: string;
+}
+/**
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ */
+export interface PairwiseStringEvaluatorArgs {
+    prediction: string;
+    predictionB: string;
+}
+/**
+ * @field The input string.
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ * @field reference The expected output / reference string.
+ */
+export interface LLMPairwiseStringEvaluatorArgs {
+    input: string;
+    prediction: string;
+    predictionB: string;
+    reference?: string;
+}
+/**
+ * Args for AgentTrajectoryEvaluator
+ * @field input The input to the agent.
+ * @field prediction The final predicted response.
+ * @field reference The reference answer.
+ * @field agentTrajectory  The intermediate steps forming the agent trajectory.
+ */
+export interface LLMTrajectoryEvaluatorArgs {
+    input: string;
+    prediction: string;
+    reference?: string;
+    agentTrajectory: AgentStep[];
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+export declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModel = BaseLanguageModel> extends LLMEvalChain<T, L> {
+    /**
+     * The name of the evaluation.
+     */
+    evaluationName?: string;
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    abstract _evaluateStrings(args: StringEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateStrings(args: StringEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+export declare abstract class StringEvaluator extends EvalChain {
+    /**
+     * The name of the evaluation.
+     */
+    evaluationName?: string;
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param config
+     */
+    abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param config
+     */
+    evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+export declare abstract class PairwiseStringEvaluator extends EvalChain {
+    /**
+     * The name of the evaluation.
+     */
+    evaluationName?: string;
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+export declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {
+    /**
+     * The name of the evaluation.
+     */
+    evaluationName?: string;
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param callOptions
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param callOptions
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Interface for evaluating agent trajectories.
+ */
+export declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {
+    requiresInput: boolean;
+    /**
+     * The name of the evaluation.
+     */
+    evaluationName?: string;
+    /**
+     * Evaluate a trajectory.
+     * @return The evaluation result.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    /**
+     * Evaluate a trajectory.
+     * @return The evaluation result.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}