langchain 1.0.0-alpha.6 → 1.0.0-alpha.8

package/dist/evaluation/base.js

@@ -1,164 +0,0 @@
-import { BaseChain } from "../chains/base.js";
-import { LLMChain } from "../chains/llm_chain.js";
-import "../chains/index.js";
-
-//#region src/evaluation/base.ts
-/**
-* Compare two sets for equality
-*
-* @param xs
-* @param ys
-*/
-const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
-/**
-* Base llm chain class for evaluators.
-*/
-var LLMEvalChain = class extends LLMChain {
-	requiresInput = false;
-	requiresReference = false;
-	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
-	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
-	/**
-	* Check if the evaluation arguments are valid.
-	* @param reference  The reference label.
-	* @param input The input string.
-	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
-	*/
-	checkEvaluationArgs(reference, input) {
-		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
-		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
-		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
-		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
-	}
-};
-/**
-* Base chain class for evaluators.
-*/
-var EvalChain = class extends BaseChain {
-	requiresInput = false;
-	requiresReference = false;
-	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
-	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
-	/**
-	* Check if the evaluation arguments are valid.
-	* @param reference  The reference label.
-	* @param input The input string.
-	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
-	*/
-	checkEvaluationArgs(reference, input) {
-		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
-		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
-		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
-		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
-	}
-};
-/**
-* Grade, tag, or otherwise evaluate predictions relative to their inputs
-* and/or reference labels
-*/
-var LLMStringEvaluator = class extends LLMEvalChain {
-	/**
-	* The name of the evaluation.
-	*/
-	evaluationName = this.constructor.name;
-	/**
-	* Evaluate Chain or LLM output, based on optional input and label.
-	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
-	* - score: the score of the evaluation, if applicable.
-	* - value: the string value of the evaluation, if applicable.
-	* - reasoning: the reasoning for the evaluation, if applicable.
-	* @param args
-	* @param callOptions
-	* @param config
-	*/
-	evaluateStrings(args, config) {
-		this.checkEvaluationArgs(args.reference, args.input);
-		return this._evaluateStrings(args, config);
-	}
-};
-/**
-* Grade, tag, or otherwise evaluate predictions relative to their inputs
-* and/or reference labels
-*/
-var StringEvaluator = class extends EvalChain {
-	/**
-	* The name of the evaluation.
-	*/
-	evaluationName = this.constructor.name;
-	/**
-	* Evaluate Chain or LLM output, based on optional input and label.
-	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
-	* - score: the score of the evaluation, if applicable.
-	* - value: the string value of the evaluation, if applicable.
-	* - reasoning: the reasoning for the evaluation, if applicable.
-	* @param args
-	* @param config
-	*/
-	evaluateStrings(args, config) {
-		this.checkEvaluationArgs(args.reference, args.input);
-		return this._evaluateStrings(args, config);
-	}
-};
-/**
-* Compare the output of two models (or two outputs of the same model).
-*/
-var PairwiseStringEvaluator = class extends EvalChain {
-	/**
-	* The name of the evaluation.
-	*/
-	evaluationName = this.constructor.name;
-	/**
-	* Evaluate the output string pairs.
-	* @param args
-	* @param config
-	* @return A dictionary containing the preference, scores, and/or other information.
-	*/
-	evaluateStringPairs(args, config) {
-		return this._evaluateStringPairs(args, config);
-	}
-};
-/**
-* Compare the output of two models (or two outputs of the same model).
-*/
-var LLMPairwiseStringEvaluator = class extends LLMEvalChain {
-	/**
-	* The name of the evaluation.
-	*/
-	evaluationName = this.constructor.name;
-	/**
-	* Evaluate the output string pairs.
-	* @param args
-	* @param callOptions
-	* @param config
-	* @return A dictionary containing the preference, scores, and/or other information.
-	*/
-	evaluateStringPairs(args, callOptions, config) {
-		this.checkEvaluationArgs(args.reference, args.input);
-		return this._evaluateStringPairs(args, callOptions, config);
-	}
-};
-/**
-* Interface for evaluating agent trajectories.
-*/
-var AgentTrajectoryEvaluator = class extends LLMEvalChain {
-	requiresInput = true;
-	/**
-	* The name of the evaluation.
-	*/
-	evaluationName = this.constructor.name;
-	/**
-	* Evaluate a trajectory.
-	* @return The evaluation result.
-	* @param args
-	* @param callOptions
-	* @param config
-	*/
-	evaluateAgentTrajectory(args, callOptions, config) {
-		this.checkEvaluationArgs(args.reference, args.input);
-		return this._evaluateAgentTrajectory(args, callOptions, config);
-	}
-};
-
-//#endregion
-export { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator, eqSet };
-//# sourceMappingURL=base.js.map

package/dist/evaluation/base.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"base.js","names":["xs: Set<string>","ys: Set<string>","reference?: string","input?: string","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","args: StringEvaluatorArgs","args: PairwiseStringEvaluatorArgs","args: LLMPairwiseStringEvaluatorArgs","callOptions?: ExtractLLMCallOptions<this[\"llm\"]>","args: LLMTrajectoryEvaluatorArgs"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,QAAQ,CAACA,IAAiBC,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBC,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;AAKD,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBD,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;;AAqDD,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACEC,MACAC,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;;AAMD,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACEC,MACAD,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;AAKD,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACEE,MACAF,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;CAC/C;AACF;;;;AAKD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACEG,MACAC,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;CAC5D;AACF;;;;AAKD,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACEK,MACAD,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO;CAChE;AACF"}

package/dist/evaluation/comparison/index.cjs

@@ -1 +0,0 @@
-const require_pairwise = require('./pairwise.cjs');

package/dist/evaluation/comparison/index.js

@@ -1 +0,0 @@
-import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";

package/dist/evaluation/comparison/pairwise.cjs

@@ -1,178 +0,0 @@
-const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
-const require_constitutional_principle = require('../../chains/constitutional_ai/constitutional_principle.cjs');
-require('../../chains/index.cjs');
-const require_base = require('../base.cjs');
-const require_prompt = require('./prompt.cjs');
-const __langchain_core_output_parsers = require_rolldown_runtime.__toESM(require("@langchain/core/output_parsers"));
-const __langchain_core_outputs = require_rolldown_runtime.__toESM(require("@langchain/core/outputs"));
-
-//#region src/evaluation/comparison/pairwise.ts
-const SUPPORTED_CRITERIA = {
-	conciseness: "Is the submission concise and to the point?",
-	relevance: "Is the submission referring to a real quote from the text?",
-	correctness: "Is the submission correct, accurate, and factual?",
-	coherence: "Is the submission coherent, well-structured, and organized?",
-	harmfulness: "Is the submission harmful, offensive, or inappropriate?",
-	maliciousness: "Is the submission malicious in any way?",
-	helpfulness: "Is the submission helpful, insightful, and appropriate?",
-	controversiality: "Is the submission controversial or debatable?",
-	misogyny: "Is the submission misogynistic? If so, response Y.",
-	criminality: "Is the submission criminal in any way?",
-	insensitivity: "Is the submission insensitive to any group of people?",
-	depth: "Does the submission demonstrate depth of thought?",
-	creativity: "Does the submission demonstrate novelty or unique ideas?",
-	detail: "Does the submission demonstrate attention to detail?"
-};
-/**
-* A parser for the output of the PairwiseStringEvalChain.
-*/
-var PairwiseStringResultOutputParser = class extends __langchain_core_output_parsers.BaseLLMOutputParser {
-	static lc_name() {
-		return "PairwiseStringResultOutputParser";
-	}
-	lc_namespace = [
-		"langchain",
-		"evaluation",
-		"comparison"
-	];
-	parseResult(generations, _callbacks) {
-		const { text } = generations[0];
-		const parsed = text.trim().split("\n");
-		let reasoning;
-		let verdict;
-		if (parsed.length === 1) [verdict] = parsed;
-		else {
-			reasoning = parsed.slice(0, parsed.length - 1).join("");
-			verdict = parsed[parsed.length - 1];
-		}
-		verdict = verdict.replace(/\[+/, "").replace(/]+/, "");
-		if (![
-			"A",
-			"B",
-			"C"
-		].includes(verdict)) throw new Error(`Invalid verdict: ${verdict}. Verdict must be one of 'A', 'B', or 'C'.`);
-		const score = {
-			A: 1,
-			B: 0,
-			C: .5
-		}[verdict];
-		if (score === void 0) throw new Error("Could not parse score from evaluator output.");
-		return Promise.resolve({
-			reasoning: reasoning || "",
-			value: verdict,
-			score
-		});
-	}
-};
-/**
-* A chain for comparing two outputs, such as the outputs
-* of two models, prompts, or outputs of a single model on similar inputs.
-*/
-var PairwiseStringEvalChain = class extends require_base.LLMPairwiseStringEvaluator {
-	static lc_name() {
-		return "PairwiseStringEvalChain";
-	}
-	criterionName;
-	evaluationName = this.criterionName;
-	requiresInput = true;
-	requiresReference = false;
-	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.
-To use references, use the LabeledPairwiseStringEvalChain instead.`;
-	outputParser = new PairwiseStringResultOutputParser();
-	static resolvePairwiseCriteria(criteria) {
-		if (criteria === void 0) {
-			const defaultCriteria = [
-				"helpfulness",
-				"relevance",
-				"correctness",
-				"depth"
-			];
-			return defaultCriteria.reduce((accumulator, currentValue) => {
-				accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
-				return accumulator;
-			}, {});
-		}
-		let criteria_ = {};
-		if (typeof criteria === "string") {
-			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
-		} else if (criteria instanceof require_constitutional_principle.ConstitutionalPrinciple) criteria_ = { [criteria.name]: criteria.critiqueRequest };
-		else {
-			if (!criteria) throw new Error("Criteria cannot be empty. Please provide a criterion name or a mapping of the criterion name to its description.");
-			criteria_ = { ...criteria };
-		}
-		return criteria_;
-	}
-	static resolvePairwisePrompt(prompt) {
-		const _prompt = prompt || require_prompt.PROMPT;
-		const expectedInputVars = new Set([
-			"prediction",
-			"predictionB",
-			"input",
-			"criteria"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-	/**
-	* Create a new instance of the PairwiseStringEvalChain.
-	* @param llm
-	* @param criteria The criteria to use for evaluation.
-	* @param chainOptions Options to pass to the chain.
-	*/
-	static async fromLLM(llm, criteria, chainOptions) {
-		let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);
-		const criteria_ = this.resolvePairwiseCriteria(criteria);
-		const criteriaStr = Object.entries(criteria_).map(([k, v]) => `${k}: ${v}`).join("\n");
-		prompt = await prompt.partial({ criteria: criteriaStr });
-		const options = chainOptions;
-		if (options) delete options.prompt;
-		return new this({
-			llm,
-			prompt,
-			...options
-		});
-	}
-	_prepareOutput(result) {
-		const parsed = result[this.outputKey];
-		if (__langchain_core_outputs.RUN_KEY in result && result[__langchain_core_outputs.RUN_KEY]) parsed[__langchain_core_outputs.RUN_KEY] = result[__langchain_core_outputs.RUN_KEY];
-		return parsed;
-	}
-	async _evaluateStringPairs(args, callOptions, config) {
-		const result = await this.call({
-			...args,
-			...callOptions
-		}, config);
-		return this._prepareOutput(result);
-	}
-};
-/**
-* A chain for comparing two outputs, such as the outputs
-* of two models, prompts, or outputs of a single model on similar inputs,
-* with labeled preferences.
-*/
-var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
-	static lc_name() {
-		return "LabeledPairwiseStringEvalChain";
-	}
-	requiresReference = true;
-	static resolvePairwisePrompt(prompt) {
-		const _prompt = prompt || require_prompt.PROMPT_WITH_REFERENCES;
-		const expectedInputVars = new Set([
-			"input",
-			"prediction",
-			"predictionB",
-			"reference",
-			"criteria"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-};
-
-//#endregion
-exports.LabeledPairwiseStringEvalChain = LabeledPairwiseStringEvalChain;
-exports.PairwiseStringEvalChain = PairwiseStringEvalChain;
-exports.PairwiseStringResultOutputParser = PairwiseStringResultOutputParser;
-//# sourceMappingURL=pairwise.cjs.map

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"pairwise.cjs","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","BaseLLMOutputParser","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","LLMPairwiseStringEvaluator","criteria?: CriteriaLike","defaultCriteria: Criteria[]","accumulator: Record<string, string>","criteria_: { [key: string]: string }","ConstitutionalPrinciple","prompt?: BasePromptTemplate","PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","eqSet","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","RUN_KEY","args: LLMPairwiseStringEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;;AAsBA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAKD,IAAa,mCAAb,cAAsDC,oDAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAa;CAExD,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,UAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;EAAI,EAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,QAAQ,0CAAE,CACc;EAIhD,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACJ,EAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM;AAGlB,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACD,EAAC;CACH;AACF;;;;;AAMD,IAAa,0BAAb,cAA6CC,wCAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK;kEACN,CAAC;CAEjE,eAAe,IAAI;CAEnB,OAAO,wBACLC,UACwB;AACxB,MAAI,aAAa,QAAW;GAC1B,MAAMC,kBAA8B;IAClC;IACA;IACA;IACA;GACD;AAED,UAAO,gBAAgB,OACrB,CAACC,aAAqC,iBAAiB;IACrD,YAAY,gBAAgB,mBAAmB;AAC/C,WAAO;GACR,GACD,CAAE,EACH;EACF;EAED,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoBC,0DAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;CAED,OAAO,sBAAsBC,QAA6B;EACxD,MAAM,UAAU,UAAUC;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAV,UACAW,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EACb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,oCAAW,UAAU,OAAOA,mCAC9B,OAAOA,oCAAW,OAAOA;AAE3B,SAAO;CACR;CAED,MAAM,qBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAa,GAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;;;AAOD,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,oBAAoB;CAEpB,OAAO,sBAAsBX,QAA6B;EACxD,MAAM,UAAU,UAAUY;EAC1B,MAAMV,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}

package/dist/evaluation/comparison/pairwise.d.cts

@@ -1,56 +0,0 @@
-import { BasePromptValueInterface } from "../../langchain-core/dist/prompt_values.cjs";
-import { EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs } from "../base.cjs";
-import { CriteriaLike } from "../criteria/criteria.cjs";
-import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-import { ChainValues } from "@langchain/core/utils/types";
-import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-import { BasePromptTemplate } from "@langchain/core/prompts";
-import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
-import { ChatGeneration, Generation } from "@langchain/core/outputs";
-
-//#region src/evaluation/comparison/pairwise.d.ts
-/**
- * A parser for the output of the PairwiseStringEvalChain.
- */
-declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
-  static lc_name(): string;
-  lc_namespace: string[];
-  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
-}
-/**
- * A chain for comparing two outputs, such as the outputs
- * of two models, prompts, or outputs of a single model on similar inputs.
- */
-declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
-  static lc_name(): string;
-  criterionName?: string;
-  evaluationName?: string;
-  requiresInput: boolean;
-  requiresReference: boolean;
-  skipReferenceWarning: string;
-  outputParser: PairwiseStringResultOutputParser;
-  static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-  /**
-   * Create a new instance of the PairwiseStringEvalChain.
-   * @param llm
-   * @param criteria The criteria to use for evaluation.
-   * @param chainOptions Options to pass to the chain.
-   */
-  static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<PairwiseStringEvalChain>;
-  _prepareOutput(result: ChainValues): any;
-  _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
-}
-/**
- * A chain for comparing two outputs, such as the outputs
- * of two models, prompts, or outputs of a single model on similar inputs,
- * with labeled preferences.
- */
-declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
-  static lc_name(): string;
-  requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-}
-//#endregion
-export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
-//# sourceMappingURL=pairwise.d.cts.map

package/dist/evaluation/comparison/pairwise.d.cts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pairwise.d.cts","names":["BaseLanguageModelInterface","BaseLLMOutputParser","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","EvalOutputType","LLMEvalChainInput","LLMPairwiseStringEvaluator","LLMPairwiseStringEvaluatorArgs","ExtractLLMCallOptions","CriteriaLike","PairwiseStringResultOutputParser","Promise","PairwiseStringEvalChain","Record","____________langchain_core_dist_prompt_values_js0","BasePromptValueInterface","Omit","Partial","LabeledPairwiseStringEvalChain"],"sources":["../../../src/evaluation/comparison/pairwise.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\nimport { CriteriaLike } from \"../criteria/criteria.js\";\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    skipReferenceWarning: string;\n    outputParser: PairwiseStringResultOutputParser;\n    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n    /**\n     * Create a new instance of the PairwiseStringEvalChain.\n     * @param llm\n     * @param criteria The criteria to use for evaluation.\n     * @param chainOptions Options to pass to the chain.\n     */\n    static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<PairwiseStringEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n    static lc_name(): string;\n    requiresReference: boolean;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAWqBc,cAAAA,gCAAAA,SAAyCb,mBAAT,CAA6BO,cAA7B,CAAA,CAAA;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,YAA6BA,EAAAA,MAAAA,EAAAA;EAAc,WAGnEJ,CAAAA,WAAAA,EAAAA,UAAAA,EAAAA,GAAeD,cAAfC,EAAAA,EAAAA,UAAAA,EAA6CE,SAA7CF,GAAAA,SAAAA,CAAAA,EAAqEW,OAArEX,CAA6EI,cAA7EJ,CAAAA;;;;;;AAHoD,cAS5DY,uBAAAA,SAAgCN,0BAAAA,CAT4B;EAS5DM,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuB,aAAA,CAAA,EAAA,MAAA;EAAA,cAO1BF,CAAAA,EAAAA,MAAAA;EAAgC,aACJD,EAAAA,OAAAA;EAAY,iBAAGI,EAAAA,OAAAA;EAAM,oBACzBZ,EAAAA,MAAAA;EAAkB,YAAA,EAF1CS,gCAE0C;EAA8G,OAA3GT,uBAAAA,CAAAA,QAAAA,CAAAA,EADjBQ,YACiBR,CAAAA,EADFY,MACEZ,CAAAA,MAAAA,EAAAA,MAAAA,CAAAA;EAAkB,OAOzDL,qBAAAA,CAAAA,MAAAA,CAAAA,EAPkBK,kBAOlBL,CAAAA,EAPuCK,kBAOvCL,CAAAA,GAAAA,EAPoC,wBAAA,EAOpCA,GAAAA,CAAAA;EAA0B;;;;;;EAA2F,OAClHE,OAAAA,CAAAA,GAAAA,EADHF,0BACGE,EAAAA,QAAAA,CAAAA,EADoCW,YACpCX,EAAAA,YAAAA,CAAAA,EADiEmB,OACjEnB,CADyEkB,IACzElB,CAD8EO,iBAC9EP,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAD2Ga,OAC3Gb,CADmHc,uBACnHd,CAAAA;EAAW,cACPS,CAAAA,MAAAA,EADJT,WACIS,CAAAA,EAAAA,GAAAA;EAA8B,oBAAeC,CAAAA,IAAAA,EAA7CD,8BAA6CC,EAAAA,WAAAA,EAAAA,qBAAAA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAA6CN,SAA7CM,GAAyDL,kBAAzDK,CAAAA,EAA8EG,OAA9EH,CAAsFV,WAAtFU,CAAAA;;;;;;AAlBG;AAyB1DU,cAAAA,8BAAAA,SAAuCN,uBAAAA,CAAT;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,iBAGTX,EAAAA,OAAAA;EAAkB,OAAA,qBAAA,CAAA,MAAA,CAAA,EAAlBA,kBAAkB,CAAA,EAAGA,kBAAH,CAAA,GAAA,EAAA,wBAAA,EAAA,GAAA,CAAA"}

package/dist/evaluation/comparison/pairwise.d.ts

@@ -1,56 +0,0 @@
-import { BasePromptValueInterface } from "../../libs/langchain-core/dist/prompt_values.js";
-import { EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs } from "../base.js";
-import { CriteriaLike } from "../criteria/criteria.js";
-import { ChainValues } from "@langchain/core/utils/types";
-import { BasePromptTemplate } from "@langchain/core/prompts";
-import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-import { ChatGeneration, Generation } from "@langchain/core/outputs";
-import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
-import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-
-//#region src/evaluation/comparison/pairwise.d.ts
-/**
- * A parser for the output of the PairwiseStringEvalChain.
- */
-declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
-  static lc_name(): string;
-  lc_namespace: string[];
-  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
-}
-/**
- * A chain for comparing two outputs, such as the outputs
- * of two models, prompts, or outputs of a single model on similar inputs.
- */
-declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
-  static lc_name(): string;
-  criterionName?: string;
-  evaluationName?: string;
-  requiresInput: boolean;
-  requiresReference: boolean;
-  skipReferenceWarning: string;
-  outputParser: PairwiseStringResultOutputParser;
-  static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-  /**
-   * Create a new instance of the PairwiseStringEvalChain.
-   * @param llm
-   * @param criteria The criteria to use for evaluation.
-   * @param chainOptions Options to pass to the chain.
-   */
-  static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<PairwiseStringEvalChain>;
-  _prepareOutput(result: ChainValues): any;
-  _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
-}
-/**
- * A chain for comparing two outputs, such as the outputs
- * of two models, prompts, or outputs of a single model on similar inputs,
- * with labeled preferences.
- */
-declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
-  static lc_name(): string;
-  requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-}
-//#endregion
-export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
-//# sourceMappingURL=pairwise.d.ts.map

package/dist/evaluation/comparison/pairwise.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pairwise.d.ts","names":["BaseLanguageModelInterface","BaseLLMOutputParser","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","EvalOutputType","LLMEvalChainInput","LLMPairwiseStringEvaluator","LLMPairwiseStringEvaluatorArgs","ExtractLLMCallOptions","CriteriaLike","PairwiseStringResultOutputParser","Promise","PairwiseStringEvalChain","Record","____________langchain_core_dist_prompt_values_js0","BasePromptValueInterface","Omit","Partial","LabeledPairwiseStringEvalChain"],"sources":["../../../src/evaluation/comparison/pairwise.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\nimport { CriteriaLike } from \"../criteria/criteria.js\";\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    skipReferenceWarning: string;\n    outputParser: PairwiseStringResultOutputParser;\n    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n    /**\n     * Create a new instance of the PairwiseStringEvalChain.\n     * @param llm\n     * @param criteria The criteria to use for evaluation.\n     * @param chainOptions Options to pass to the chain.\n     */\n    static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<PairwiseStringEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n    static lc_name(): string;\n    requiresReference: boolean;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAWqBc,cAAAA,gCAAAA,SAAyCb,mBAAT,CAA6BO,cAA7B,CAAA,CAAA;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,YAA6BA,EAAAA,MAAAA,EAAAA;EAAc,WAGnEJ,CAAAA,WAAAA,EAAAA,UAAAA,EAAAA,GAAeD,cAAfC,EAAAA,EAAAA,UAAAA,EAA6CE,SAA7CF,GAAAA,SAAAA,CAAAA,EAAqEW,OAArEX,CAA6EI,cAA7EJ,CAAAA;;;;;;AAHoD,cAS5DY,uBAAAA,SAAgCN,0BAAAA,CAT4B;EAS5DM,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuB,aAAA,CAAA,EAAA,MAAA;EAAA,cAO1BF,CAAAA,EAAAA,MAAAA;EAAgC,aACJD,EAAAA,OAAAA;EAAY,iBAAGI,EAAAA,OAAAA;EAAM,oBACzBZ,EAAAA,MAAAA;EAAkB,YAAA,EAF1CS,gCAE0C;EAA8G,OAA3GT,uBAAAA,CAAAA,QAAAA,CAAAA,EADjBQ,YACiBR,CAAAA,EADFY,MACEZ,CAAAA,MAAAA,EAAAA,MAAAA,CAAAA;EAAkB,OAOzDL,qBAAAA,CAAAA,MAAAA,CAAAA,EAPkBK,kBAOlBL,CAAAA,EAPuCK,kBAOvCL,CAAAA,GAAAA,EAPoC,wBAAA,EAOpCA,GAAAA,CAAAA;EAA0B;;;;;;EAA2F,OAClHE,OAAAA,CAAAA,GAAAA,EADHF,0BACGE,EAAAA,QAAAA,CAAAA,EADoCW,YACpCX,EAAAA,YAAAA,CAAAA,EADiEmB,OACjEnB,CADyEkB,IACzElB,CAD8EO,iBAC9EP,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAD2Ga,OAC3Gb,CADmHc,uBACnHd,CAAAA;EAAW,cACPS,CAAAA,MAAAA,EADJT,WACIS,CAAAA,EAAAA,GAAAA;EAA8B,oBAAeC,CAAAA,IAAAA,EAA7CD,8BAA6CC,EAAAA,WAAAA,EAAAA,qBAAAA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAA6CN,SAA7CM,GAAyDL,kBAAzDK,CAAAA,EAA8EG,OAA9EH,CAAsFV,WAAtFU,CAAAA;;;;;;AAlBG;AAyB1DU,cAAAA,8BAAAA,SAAuCN,uBAAAA,CAAT;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,iBAGTX,EAAAA,OAAAA;EAAkB,OAAA,qBAAA,CAAA,MAAA,CAAA,EAAlBA,kBAAkB,CAAA,EAAGA,kBAAH,CAAA,GAAA,EAAA,wBAAA,EAAA,GAAA,CAAA"}