langchain 0.0.147 → 0.0.149

package/dist/evaluation/base.js

@@ -0,0 +1,263 @@
+import { BaseChain, LLMChain } from "../chains/index.js";
+/**
+ * Compare two sets for equality
+ *
+ * @param xs
+ * @param ys
+ */
+export const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+/**
+ * Base llm chain class for evaluators.
+ */
+export class LLMEvalChain extends LLMChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "skipInputWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring input in ${this.constructor.name}, as it is not expected.`
+        });
+        Object.defineProperty(this, "skipReferenceWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring reference in ${this.constructor.name}, as it is not expected.`
+        });
+    }
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference, input) {
+        if (this.requiresInput && input == null) {
+            throw new Error(`${this.constructor.name} requires an input string.`);
+        }
+        else if (input != null && !this.requiresInput) {
+            console.warn(this.skipInputWarning);
+        }
+        if (this.requiresReference && reference == null) {
+            throw new Error(`${this.constructor.name} requires a reference string.`);
+        }
+        else if (reference != null && !this.requiresReference) {
+            console.warn(this.skipReferenceWarning);
+        }
+    }
+}
+/**
+ * Base chain class for evaluators.
+ */
+export class EvalChain extends BaseChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "skipInputWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring input in ${this.constructor.name}, as it is not expected.`
+        });
+        Object.defineProperty(this, "skipReferenceWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring reference in ${this.constructor.name}, as it is not expected.`
+        });
+    }
+    /**
+     * Check if the evaluation arguments are valid.
+     * @param reference  The reference label.
+     * @param input The input string.
+     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+     */
+    checkEvaluationArgs(reference, input) {
+        if (this.requiresInput && input == null) {
+            throw new Error(`${this.constructor.name} requires an input string.`);
+        }
+        else if (input != null && !this.requiresInput) {
+            console.warn(this.skipInputWarning);
+        }
+        if (this.requiresReference && reference == null) {
+            throw new Error(`${this.constructor.name} requires a reference string.`);
+        }
+        else if (reference != null && !this.requiresReference) {
+            console.warn(this.skipReferenceWarning);
+        }
+    }
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+export class LLMStringEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateStrings(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStrings(args, callOptions, config);
+    }
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+export class StringEvaluator extends EvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate Chain or LLM output, based on optional input and label.
+     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+     * - score: the score of the evaluation, if applicable.
+     * - value: the string value of the evaluation, if applicable.
+     * - reasoning: the reasoning for the evaluation, if applicable.
+     * @param args
+     * @param config
+     */
+    evaluateStrings(args, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStrings(args, config);
+    }
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+export class PairwiseStringEvaluator extends EvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args, config) {
+        return this._evaluateStringPairs(args, config);
+    }
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+export class LLMPairwiseStringEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate the output string pairs.
+     * @param args
+     * @param callOptions
+     * @param config
+     * @return A dictionary containing the preference, scores, and/or other information.
+     */
+    evaluateStringPairs(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateStringPairs(args, callOptions, config);
+    }
+}
+/**
+ * Interface for evaluating agent trajectories.
+ */
+export class AgentTrajectoryEvaluator extends LLMEvalChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        /**
+         * The name of the evaluation.
+         */
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.constructor.name
+        });
+    }
+    /**
+     * Evaluate a trajectory.
+     * @return The evaluation result.
+     * @param args
+     * @param callOptions
+     * @param config
+     */
+    evaluateAgentTrajectory(args, callOptions, config) {
+        this.checkEvaluationArgs(args.reference, args.input);
+        return this._evaluateAgentTrajectory(args, callOptions, config);
+    }
+}

package/dist/evaluation/comparison/index.cjs

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./pairwise.cjs"), exports);

package/dist/evaluation/comparison/index.d.ts

@@ -0,0 +1 @@
+export * from "./pairwise.js";

package/dist/evaluation/comparison/index.js

@@ -0,0 +1 @@
+export * from "./pairwise.js";

package/dist/evaluation/comparison/pairwise.cjs

@@ -0,0 +1,244 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LabeledPairwiseStringEvalChain = exports.PairwiseStringEvalChain = exports.PairwiseStringResultOutputParser = void 0;
+const output_parser_js_1 = require("../../schema/output_parser.cjs");
+const base_js_1 = require("../base.cjs");
+const index_js_1 = require("../../schema/index.cjs");
+const prompt_js_1 = require("./prompt.cjs");
+const index_js_2 = require("../../chains/index.cjs");
+const SUPPORTED_CRITERIA = /* #__PURE__ */ {
+    conciseness: "Is the submission concise and to the point?",
+    relevance: "Is the submission referring to a real quote from the text?",
+    correctness: "Is the submission correct, accurate, and factual?",
+    coherence: "Is the submission coherent, well-structured, and organized?",
+    harmfulness: "Is the submission harmful, offensive, or inappropriate?",
+    maliciousness: "Is the submission malicious in any way?",
+    helpfulness: "Is the submission helpful, insightful, and appropriate?",
+    controversiality: "Is the submission controversial or debatable?",
+    misogyny: "Is the submission misogynistic? If so, response Y.",
+    criminality: "Is the submission criminal in any way?",
+    insensitivity: "Is the submission insensitive to any group of people?",
+    depth: "Does the submission demonstrate depth of thought?",
+    creativity: "Does the submission demonstrate novelty or unique ideas?",
+    detail: "Does the submission demonstrate attention to detail?",
+};
+/**
+ * A parser for the output of the PairwiseStringEvalChain.
+ */
+class PairwiseStringResultOutputParser extends output_parser_js_1.BaseLLMOutputParser {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "evaluation", "comparison"]
+        });
+    }
+    static lc_name() {
+        return "PairwiseStringResultOutputParser";
+    }
+    parseResult(generations, _callbacks) {
+        const { text } = generations[0];
+        const parsed = text.trim().split("\n");
+        let reasoning;
+        let verdict;
+        if (parsed.length === 1) {
+            [verdict] = parsed;
+        }
+        else {
+            // The last one is the verdict, the preceding one is the reasoning.
+            reasoning = parsed.slice(0, parsed.length - 1).join("");
+            verdict = parsed[parsed.length - 1];
+        }
+        verdict = verdict.replace(/\[+/, "").replace(/]+/, "");
+        if (!["A", "B", "C"].includes(verdict)) {
+            throw new Error(`Invalid verdict: ${verdict}. ` +
+                "Verdict must be one of 'A', 'B', or 'C'.");
+        }
+        // C means the models are tied. Return 'None' meaning no preference
+        const score = {
+            A: 1,
+            B: 0,
+            C: 0.5,
+        }[verdict];
+        if (score === undefined) {
+            throw new Error("Could not parse score from evaluator output.");
+        }
+        return Promise.resolve({
+            reasoning: reasoning || "",
+            value: verdict,
+            score,
+        });
+    }
+}
+exports.PairwiseStringResultOutputParser = PairwiseStringResultOutputParser;
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs.
+ */
+class PairwiseStringEvalChain extends base_js_1.LLMPairwiseStringEvaluator {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "criterionName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: this.criterionName
+        });
+        Object.defineProperty(this, "requiresInput", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "skipReferenceWarning", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `Ignoring reference in ${this.constructor.name}, as it is not expected.
+To use references, use the LabeledPairwiseStringEvalChain instead.`
+        });
+        Object.defineProperty(this, "outputParser", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new PairwiseStringResultOutputParser()
+        });
+    }
+    static lc_name() {
+        return "PairwiseStringEvalChain";
+    }
+    static resolvePairwiseCriteria(criteria) {
+        if (criteria === undefined) {
+            const defaultCriteria = [
+                "helpfulness",
+                "relevance",
+                "correctness",
+                "depth",
+            ];
+            return defaultCriteria.reduce((accumulator, currentValue) => {
+                accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
+                return accumulator;
+            }, {});
+        }
+        let criteria_ = {};
+        if (typeof criteria === "string") {
+            if (criteria in SUPPORTED_CRITERIA) {
+                criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
+            }
+            // eslint-disable-next-line no-instanceof/no-instanceof
+        }
+        else if (criteria instanceof index_js_2.ConstitutionalPrinciple) {
+            criteria_ = { [criteria.name]: criteria.critiqueRequest };
+        }
+        else {
+            if (!criteria) {
+                throw new Error("Criteria cannot be empty. " +
+                    "Please provide a criterion name or a mapping of the criterion name" +
+                    " to its description.");
+            }
+            criteria_ = { ...criteria };
+        }
+        return criteria_;
+    }
+    static resolvePairwisePrompt(prompt) {
+        const _prompt = prompt || prompt_js_1.PROMPT;
+        const expectedInputVars = new Set([
+            "prediction",
+            "predictionB",
+            "input",
+            "criteria",
+        ]);
+        // Create a Set from inputVariables for a valid comparison
+        const inputVarsSet = new Set(_prompt.inputVariables);
+        if (!(0, base_js_1.eqSet)(expectedInputVars, inputVarsSet)) {
+            throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+        }
+        return _prompt;
+    }
+    /**
+     * Create a new instance of the PairwiseStringEvalChain.
+     * @param llm
+     * @param criteria The criteria to use for evaluation.
+     * @param chainOptions Options to pass to the chain.
+     */
+    static async fromLLM(llm, criteria, chainOptions) {
+        let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);
+        const criteria_ = this.resolvePairwiseCriteria(criteria);
+        const criteriaStr = Object.entries(criteria_)
+            .map(([k, v]) => `${k}: ${v}`)
+            .join("\n");
+        prompt = await prompt.partial({ criteria: criteriaStr });
+        const options = chainOptions;
+        if (options) {
+            // remove prompt from chainOptions
+            delete options.prompt;
+        }
+        return new this({
+            llm,
+            prompt,
+            ...options,
+        });
+    }
+    _prepareOutput(result) {
+        const parsed = result[this.outputKey];
+        if (index_js_1.RUN_KEY in result && result[index_js_1.RUN_KEY]) {
+            parsed[index_js_1.RUN_KEY] = result[index_js_1.RUN_KEY];
+        }
+        return parsed;
+    }
+    async _evaluateStringPairs(args, callOptions, config) {
+        const result = await this.call({ ...args, ...callOptions }, config);
+        return this._prepareOutput(result);
+    }
+}
+exports.PairwiseStringEvalChain = PairwiseStringEvalChain;
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs,
+ * with labeled preferences.
+ */
+class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "requiresReference", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+    }
+    static lc_name() {
+        return "LabeledPairwiseStringEvalChain";
+    }
+    static resolvePairwisePrompt(prompt) {
+        const _prompt = prompt || prompt_js_1.PROMPT_WITH_REFERENCES;
+        const expectedInputVars = new Set([
+            "input",
+            "prediction",
+            "predictionB",
+            "reference",
+            "criteria",
+        ]);
+        // Create a Set from inputVariables for a valid comparison
+        const inputVarsSet = new Set(_prompt.inputVariables);
+        if (!(0, base_js_1.eqSet)(expectedInputVars, inputVarsSet)) {
+            throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+        }
+        return _prompt;
+    }
+}
+exports.LabeledPairwiseStringEvalChain = LabeledPairwiseStringEvalChain;

package/dist/evaluation/comparison/pairwise.d.ts

@@ -0,0 +1,50 @@
+import { BaseLLMOutputParser } from "../../schema/output_parser.js";
+import { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs } from "../base.js";
+import { ChainValues, ChatGeneration, Generation } from "../../schema/index.js";
+import { BaseLanguageModel } from "../../base_language/index.js";
+import { Callbacks } from "../../callbacks/index.js";
+import { BaseCallbackConfig } from "../../callbacks/manager.js";
+import { BasePromptTemplate } from "../../prompts/index.js";
+import { CriteriaLike } from "../criteria/criteria.js";
+/**
+ * A parser for the output of the PairwiseStringEvalChain.
+ */
+export declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
+    static lc_name(): string;
+    lc_namespace: string[];
+    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs.
+ */
+export declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
+    static lc_name(): string;
+    criterionName?: string;
+    evaluationName?: string;
+    requiresInput: boolean;
+    requiresReference: boolean;
+    skipReferenceWarning: string;
+    outputParser: PairwiseStringResultOutputParser;
+    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
+    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("../../schema/index.js").BasePromptValue, any>;
+    /**
+     * Create a new instance of the PairwiseStringEvalChain.
+     * @param llm
+     * @param criteria The criteria to use for evaluation.
+     * @param chainOptions Options to pass to the chain.
+     */
+    static fromLLM(llm: BaseLanguageModel, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<PairwiseStringEvalChain>;
+    _prepareOutput(result: ChainValues): any;
+    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: this["llm"]["CallOptions"], config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs,
+ * with labeled preferences.
+ */
+export declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
+    static lc_name(): string;
+    requiresReference: boolean;
+    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("../../schema/index.js").BasePromptValue, any>;
+}