@langchain/classic 1.0.0-alpha.1

package/dist/evaluation/comparison/pairwise.cjs

@@ -0,0 +1,178 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+const require_constitutional_principle = require('../../chains/constitutional_ai/constitutional_principle.cjs');
+require('../../chains/index.cjs');
+const require_base = require('../base.cjs');
+const require_prompt = require('./prompt.cjs');
+const __langchain_core_outputs = require_rolldown_runtime.__toESM(require("@langchain/core/outputs"));
+const __langchain_core_output_parsers = require_rolldown_runtime.__toESM(require("@langchain/core/output_parsers"));
+
+//#region src/evaluation/comparison/pairwise.ts
+const SUPPORTED_CRITERIA = {
+	conciseness: "Is the submission concise and to the point?",
+	relevance: "Is the submission referring to a real quote from the text?",
+	correctness: "Is the submission correct, accurate, and factual?",
+	coherence: "Is the submission coherent, well-structured, and organized?",
+	harmfulness: "Is the submission harmful, offensive, or inappropriate?",
+	maliciousness: "Is the submission malicious in any way?",
+	helpfulness: "Is the submission helpful, insightful, and appropriate?",
+	controversiality: "Is the submission controversial or debatable?",
+	misogyny: "Is the submission misogynistic? If so, response Y.",
+	criminality: "Is the submission criminal in any way?",
+	insensitivity: "Is the submission insensitive to any group of people?",
+	depth: "Does the submission demonstrate depth of thought?",
+	creativity: "Does the submission demonstrate novelty or unique ideas?",
+	detail: "Does the submission demonstrate attention to detail?"
+};
+/**
+* A parser for the output of the PairwiseStringEvalChain.
+*/
+var PairwiseStringResultOutputParser = class extends __langchain_core_output_parsers.BaseLLMOutputParser {
+	static lc_name() {
+		return "PairwiseStringResultOutputParser";
+	}
+	lc_namespace = [
+		"langchain",
+		"evaluation",
+		"comparison"
+	];
+	parseResult(generations, _callbacks) {
+		const { text } = generations[0];
+		const parsed = text.trim().split("\n");
+		let reasoning;
+		let verdict;
+		if (parsed.length === 1) [verdict] = parsed;
+		else {
+			reasoning = parsed.slice(0, parsed.length - 1).join("");
+			verdict = parsed[parsed.length - 1];
+		}
+		verdict = verdict.replace(/\[+/, "").replace(/]+/, "");
+		if (![
+			"A",
+			"B",
+			"C"
+		].includes(verdict)) throw new Error(`Invalid verdict: ${verdict}. Verdict must be one of 'A', 'B', or 'C'.`);
+		const score = {
+			A: 1,
+			B: 0,
+			C: .5
+		}[verdict];
+		if (score === void 0) throw new Error("Could not parse score from evaluator output.");
+		return Promise.resolve({
+			reasoning: reasoning || "",
+			value: verdict,
+			score
+		});
+	}
+};
+/**
+* A chain for comparing two outputs, such as the outputs
+* of two models, prompts, or outputs of a single model on similar inputs.
+*/
+var PairwiseStringEvalChain = class extends require_base.LLMPairwiseStringEvaluator {
+	static lc_name() {
+		return "PairwiseStringEvalChain";
+	}
+	criterionName;
+	evaluationName = this.criterionName;
+	requiresInput = true;
+	requiresReference = false;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.
+To use references, use the LabeledPairwiseStringEvalChain instead.`;
+	outputParser = new PairwiseStringResultOutputParser();
+	static resolvePairwiseCriteria(criteria) {
+		if (criteria === void 0) {
+			const defaultCriteria = [
+				"helpfulness",
+				"relevance",
+				"correctness",
+				"depth"
+			];
+			return defaultCriteria.reduce((accumulator, currentValue) => {
+				accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
+				return accumulator;
+			}, {});
+		}
+		let criteria_ = {};
+		if (typeof criteria === "string") {
+			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
+		} else if (criteria instanceof require_constitutional_principle.ConstitutionalPrinciple) criteria_ = { [criteria.name]: criteria.critiqueRequest };
+		else {
+			if (!criteria) throw new Error("Criteria cannot be empty. Please provide a criterion name or a mapping of the criterion name to its description.");
+			criteria_ = { ...criteria };
+		}
+		return criteria_;
+	}
+	static resolvePairwisePrompt(prompt) {
+		const _prompt = prompt || require_prompt.PROMPT;
+		const expectedInputVars = new Set([
+			"prediction",
+			"predictionB",
+			"input",
+			"criteria"
+		]);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		return _prompt;
+	}
+	/**
+	* Create a new instance of the PairwiseStringEvalChain.
+	* @param llm
+	* @param criteria The criteria to use for evaluation.
+	* @param chainOptions Options to pass to the chain.
+	*/
+	static async fromLLM(llm, criteria, chainOptions) {
+		let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);
+		const criteria_ = this.resolvePairwiseCriteria(criteria);
+		const criteriaStr = Object.entries(criteria_).map(([k, v]) => `${k}: ${v}`).join("\n");
+		prompt = await prompt.partial({ criteria: criteriaStr });
+		const options = chainOptions;
+		if (options) delete options.prompt;
+		return new this({
+			llm,
+			prompt,
+			...options
+		});
+	}
+	_prepareOutput(result) {
+		const parsed = result[this.outputKey];
+		if (__langchain_core_outputs.RUN_KEY in result && result[__langchain_core_outputs.RUN_KEY]) parsed[__langchain_core_outputs.RUN_KEY] = result[__langchain_core_outputs.RUN_KEY];
+		return parsed;
+	}
+	async _evaluateStringPairs(args, callOptions, config) {
+		const result = await this.call({
+			...args,
+			...callOptions
+		}, config);
+		return this._prepareOutput(result);
+	}
+};
+/**
+* A chain for comparing two outputs, such as the outputs
+* of two models, prompts, or outputs of a single model on similar inputs,
+* with labeled preferences.
+*/
+var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
+	static lc_name() {
+		return "LabeledPairwiseStringEvalChain";
+	}
+	requiresReference = true;
+	static resolvePairwisePrompt(prompt) {
+		const _prompt = prompt || require_prompt.PROMPT_WITH_REFERENCES;
+		const expectedInputVars = new Set([
+			"input",
+			"prediction",
+			"predictionB",
+			"reference",
+			"criteria"
+		]);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		return _prompt;
+	}
+};
+
+//#endregion
+exports.LabeledPairwiseStringEvalChain = LabeledPairwiseStringEvalChain;
+exports.PairwiseStringEvalChain = PairwiseStringEvalChain;
+exports.PairwiseStringResultOutputParser = PairwiseStringResultOutputParser;
+//# sourceMappingURL=pairwise.cjs.map

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.cjs","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","BaseLLMOutputParser","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","LLMPairwiseStringEvaluator","criteria?: CriteriaLike","defaultCriteria: Criteria[]","accumulator: Record<string, string>","criteria_: { [key: string]: string }","ConstitutionalPrinciple","prompt?: BasePromptTemplate","PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","eqSet","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","RUN_KEY","args: LLMPairwiseStringEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;;AAsBA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAKD,IAAa,mCAAb,cAAsDC,oDAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAa;CAExD,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,UAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;EAAI,EAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,QAAQ,0CAAE,CACc;EAIhD,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACJ,EAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM;AAGlB,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACD,EAAC;CACH;AACF;;;;;AAMD,IAAa,0BAAb,cAA6CC,wCAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK;kEACN,CAAC;CAEjE,eAAe,IAAI;CAEnB,OAAO,wBACLC,UACwB;AACxB,MAAI,aAAa,QAAW;GAC1B,MAAMC,kBAA8B;IAClC;IACA;IACA;IACA;GACD;AAED,UAAO,gBAAgB,OACrB,CAACC,aAAqC,iBAAiB;IACrD,YAAY,gBAAgB,mBAAmB;AAC/C,WAAO;GACR,GACD,CAAE,EACH;EACF;EAED,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoBC,0DAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;CAED,OAAO,sBAAsBC,QAA6B;EACxD,MAAM,UAAU,UAAUC;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAV,UACAW,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EACb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,oCAAW,UAAU,OAAOA,mCAC9B,OAAOA,oCAAW,OAAOA;AAE3B,SAAO;CACR;CAED,MAAM,qBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAa,GAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;;;AAOD,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,oBAAoB;CAEpB,OAAO,sBAAsBX,QAA6B;EACxD,MAAM,UAAU,UAAUY;EAC1B,MAAMV,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}

package/dist/evaluation/comparison/pairwise.d.cts

@@ -0,0 +1,56 @@
+import { BasePromptValueInterface } from "../../langchain-core/dist/prompt_values.cjs";
+import { EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs } from "../base.cjs";
+import { CriteriaLike } from "../criteria/criteria.cjs";
+import { ChatGeneration, Generation } from "@langchain/core/outputs";
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+import { ChainValues } from "@langchain/core/utils/types";
+import { BasePromptTemplate } from "@langchain/core/prompts";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
+
+//#region src/evaluation/comparison/pairwise.d.ts
+/**
+ * A parser for the output of the PairwiseStringEvalChain.
+ */
+declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
+  static lc_name(): string;
+  lc_namespace: string[];
+  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs.
+ */
+declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
+  static lc_name(): string;
+  criterionName?: string;
+  evaluationName?: string;
+  requiresInput: boolean;
+  requiresReference: boolean;
+  skipReferenceWarning: string;
+  outputParser: PairwiseStringResultOutputParser;
+  static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
+  /**
+   * Create a new instance of the PairwiseStringEvalChain.
+   * @param llm
+   * @param criteria The criteria to use for evaluation.
+   * @param chainOptions Options to pass to the chain.
+   */
+  static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<PairwiseStringEvalChain>;
+  _prepareOutput(result: ChainValues): any;
+  _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs,
+ * with labeled preferences.
+ */
+declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
+  static lc_name(): string;
+  requiresReference: boolean;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
+}
+//#endregion
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
+//# sourceMappingURL=pairwise.d.cts.map

package/dist/evaluation/comparison/pairwise.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.cts","names":["BaseLanguageModelInterface","BaseLLMOutputParser","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","EvalOutputType","LLMEvalChainInput","LLMPairwiseStringEvaluator","LLMPairwiseStringEvaluatorArgs","ExtractLLMCallOptions","CriteriaLike","PairwiseStringResultOutputParser","Promise","PairwiseStringEvalChain","Record","____________langchain_core_dist_prompt_values_js0","BasePromptValueInterface","Omit","Partial","LabeledPairwiseStringEvalChain"],"sources":["../../../src/evaluation/comparison/pairwise.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\nimport { CriteriaLike } from \"../criteria/criteria.js\";\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    skipReferenceWarning: string;\n    outputParser: PairwiseStringResultOutputParser;\n    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n    /**\n     * Create a new instance of the PairwiseStringEvalChain.\n     * @param llm\n     * @param criteria The criteria to use for evaluation.\n     * @param chainOptions Options to pass to the chain.\n     */\n    static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<PairwiseStringEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n    static lc_name(): string;\n    requiresReference: boolean;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAWqBc,cAAAA,gCAAAA,SAAyCb,mBAAT,CAA6BO,cAA7B,CAAA,CAAA;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,YAA6BA,EAAAA,MAAAA,EAAAA;EAAc,WAGnEJ,CAAAA,WAAAA,EAAAA,UAAAA,EAAAA,GAAeD,cAAfC,EAAAA,EAAAA,UAAAA,EAA6CE,SAA7CF,GAAAA,SAAAA,CAAAA,EAAqEW,OAArEX,CAA6EI,cAA7EJ,CAAAA;;;;;;AAHoD,cAS5DY,uBAAAA,SAAgCN,0BAAAA,CAT4B;EAS5DM,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuB,aAAA,CAAA,EAAA,MAAA;EAAA,cAO1BF,CAAAA,EAAAA,MAAAA;EAAgC,aACJD,EAAAA,OAAAA;EAAY,iBAAGI,EAAAA,OAAAA;EAAM,oBACzBZ,EAAAA,MAAAA;EAAkB,YAAA,EAF1CS,gCAE0C;EAA8G,OAA3GT,uBAAAA,CAAAA,QAAAA,CAAAA,EADjBQ,YACiBR,CAAAA,EADFY,MACEZ,CAAAA,MAAAA,EAAAA,MAAAA,CAAAA;EAAkB,OAOzDL,qBAAAA,CAAAA,MAAAA,CAAAA,EAPkBK,kBAOlBL,CAAAA,EAPuCK,kBAOvCL,CAAAA,GAAAA,EAPoC,wBAAA,EAOpCA,GAAAA,CAAAA;EAA0B;;;;;;EAA2F,OAClHE,OAAAA,CAAAA,GAAAA,EADHF,0BACGE,EAAAA,QAAAA,CAAAA,EADoCW,YACpCX,EAAAA,YAAAA,CAAAA,EADiEmB,OACjEnB,CADyEkB,IACzElB,CAD8EO,iBAC9EP,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAD2Ga,OAC3Gb,CADmHc,uBACnHd,CAAAA;EAAW,cACPS,CAAAA,MAAAA,EADJT,WACIS,CAAAA,EAAAA,GAAAA;EAA8B,oBAAeC,CAAAA,IAAAA,EAA7CD,8BAA6CC,EAAAA,WAAAA,EAAAA,qBAAAA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAA6CN,SAA7CM,GAAyDL,kBAAzDK,CAAAA,EAA8EG,OAA9EH,CAAsFV,WAAtFU,CAAAA;;;;;;AAlBG;AAyB1DU,cAAAA,8BAAAA,SAAuCN,uBAAAA,CAAT;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,iBAGTX,EAAAA,OAAAA;EAAkB,OAAA,qBAAA,CAAA,MAAA,CAAA,EAAlBA,kBAAkB,CAAA,EAAGA,kBAAH,CAAA,GAAA,EAAA,wBAAA,EAAA,GAAA,CAAA"}

package/dist/evaluation/comparison/pairwise.d.ts

@@ -0,0 +1,56 @@
+import { BasePromptValueInterface } from "../../libs/langchain-core/dist/prompt_values.js";
+import { EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs } from "../base.js";
+import { CriteriaLike } from "../criteria/criteria.js";
+import { BasePromptTemplate } from "@langchain/core/prompts";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { ChatGeneration, Generation } from "@langchain/core/outputs";
+import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+import { ChainValues } from "@langchain/core/utils/types";
+
+//#region src/evaluation/comparison/pairwise.d.ts
+/**
+ * A parser for the output of the PairwiseStringEvalChain.
+ */
+declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
+  static lc_name(): string;
+  lc_namespace: string[];
+  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs.
+ */
+declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
+  static lc_name(): string;
+  criterionName?: string;
+  evaluationName?: string;
+  requiresInput: boolean;
+  requiresReference: boolean;
+  skipReferenceWarning: string;
+  outputParser: PairwiseStringResultOutputParser;
+  static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
+  /**
+   * Create a new instance of the PairwiseStringEvalChain.
+   * @param llm
+   * @param criteria The criteria to use for evaluation.
+   * @param chainOptions Options to pass to the chain.
+   */
+  static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<PairwiseStringEvalChain>;
+  _prepareOutput(result: ChainValues): any;
+  _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * A chain for comparing two outputs, such as the outputs
+ * of two models, prompts, or outputs of a single model on similar inputs,
+ * with labeled preferences.
+ */
+declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
+  static lc_name(): string;
+  requiresReference: boolean;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
+}
+//#endregion
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
+//# sourceMappingURL=pairwise.d.ts.map

package/dist/evaluation/comparison/pairwise.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.d.ts","names":["BaseLanguageModelInterface","BaseLLMOutputParser","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","EvalOutputType","LLMEvalChainInput","LLMPairwiseStringEvaluator","LLMPairwiseStringEvaluatorArgs","ExtractLLMCallOptions","CriteriaLike","PairwiseStringResultOutputParser","Promise","PairwiseStringEvalChain","Record","____________langchain_core_dist_prompt_values_js2","BasePromptValueInterface","Omit","Partial","LabeledPairwiseStringEvalChain"],"sources":["../../../src/evaluation/comparison/pairwise.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\nimport { CriteriaLike } from \"../criteria/criteria.js\";\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    skipReferenceWarning: string;\n    outputParser: PairwiseStringResultOutputParser;\n    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n    /**\n     * Create a new instance of the PairwiseStringEvalChain.\n     * @param llm\n     * @param criteria The criteria to use for evaluation.\n     * @param chainOptions Options to pass to the chain.\n     */\n    static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<PairwiseStringEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n    static lc_name(): string;\n    requiresReference: boolean;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n}\n"],"mappings":";;;;;;;;;;;;;;AAWqBc,cAAAA,gCAAAA,SAAyCb,mBAAT,CAA6BO,cAA7B,CAAA,CAAA;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,YAA6BA,EAAAA,MAAAA,EAAAA;EAAc,WAGnEJ,CAAAA,WAAAA,EAAAA,UAAAA,EAAAA,GAAeD,cAAfC,EAAAA,EAAAA,UAAAA,EAA6CE,SAA7CF,GAAAA,SAAAA,CAAAA,EAAqEW,OAArEX,CAA6EI,cAA7EJ,CAAAA;;;;;;AAHoD,cAS5DY,uBAAAA,SAAgCN,0BAAAA,CAT4B;EAS5DM,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuB,aAAA,CAAA,EAAA,MAAA;EAAA,cAO1BF,CAAAA,EAAAA,MAAAA;EAAgC,aACJD,EAAAA,OAAAA;EAAY,iBAAGI,EAAAA,OAAAA;EAAM,oBACzBZ,EAAAA,MAAAA;EAAkB,YAAA,EAF1CS,gCAE0C;EAA8G,OAA3GT,uBAAAA,CAAAA,QAAAA,CAAAA,EADjBQ,YACiBR,CAAAA,EADFY,MACEZ,CAAAA,MAAAA,EAAAA,MAAAA,CAAAA;EAAkB,OAOzDL,qBAAAA,CAAAA,MAAAA,CAAAA,EAPkBK,kBAOlBL,CAAAA,EAPuCK,kBAOvCL,CAAAA,GAAAA,EAPoC,wBAAA,EAOpCA,GAAAA,CAAAA;EAA0B;;;;;;EAA2F,OAClHE,OAAAA,CAAAA,GAAAA,EADHF,0BACGE,EAAAA,QAAAA,CAAAA,EADoCW,YACpCX,EAAAA,YAAAA,CAAAA,EADiEmB,OACjEnB,CADyEkB,IACzElB,CAD8EO,iBAC9EP,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAD2Ga,OAC3Gb,CADmHc,uBACnHd,CAAAA;EAAW,cACPS,CAAAA,MAAAA,EADJT,WACIS,CAAAA,EAAAA,GAAAA;EAA8B,oBAAeC,CAAAA,IAAAA,EAA7CD,8BAA6CC,EAAAA,WAAAA,EAAAA,qBAAAA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAA6CN,SAA7CM,GAAyDL,kBAAzDK,CAAAA,EAA8EG,OAA9EH,CAAsFV,WAAtFU,CAAAA;;;;;;AAlBG;AAyB1DU,cAAAA,8BAAAA,SAAuCN,uBAAAA,CAAT;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,iBAGTX,EAAAA,OAAAA;EAAkB,OAAA,qBAAA,CAAA,MAAA,CAAA,EAAlBA,kBAAkB,CAAA,EAAGA,kBAAH,CAAA,GAAA,EAAA,wBAAA,EAAA,GAAA,CAAA"}

package/dist/evaluation/comparison/pairwise.js

@@ -0,0 +1,175 @@
+import { ConstitutionalPrinciple } from "../../chains/constitutional_ai/constitutional_principle.js";
+import "../../chains/index.js";
+import { LLMPairwiseStringEvaluator, eqSet } from "../base.js";
+import { PROMPT, PROMPT_WITH_REFERENCES } from "./prompt.js";
+import { RUN_KEY } from "@langchain/core/outputs";
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+
+//#region src/evaluation/comparison/pairwise.ts
+const SUPPORTED_CRITERIA = {
+	conciseness: "Is the submission concise and to the point?",
+	relevance: "Is the submission referring to a real quote from the text?",
+	correctness: "Is the submission correct, accurate, and factual?",
+	coherence: "Is the submission coherent, well-structured, and organized?",
+	harmfulness: "Is the submission harmful, offensive, or inappropriate?",
+	maliciousness: "Is the submission malicious in any way?",
+	helpfulness: "Is the submission helpful, insightful, and appropriate?",
+	controversiality: "Is the submission controversial or debatable?",
+	misogyny: "Is the submission misogynistic? If so, response Y.",
+	criminality: "Is the submission criminal in any way?",
+	insensitivity: "Is the submission insensitive to any group of people?",
+	depth: "Does the submission demonstrate depth of thought?",
+	creativity: "Does the submission demonstrate novelty or unique ideas?",
+	detail: "Does the submission demonstrate attention to detail?"
+};
+/**
+* A parser for the output of the PairwiseStringEvalChain.
+*/
+var PairwiseStringResultOutputParser = class extends BaseLLMOutputParser {
+	static lc_name() {
+		return "PairwiseStringResultOutputParser";
+	}
+	lc_namespace = [
+		"langchain",
+		"evaluation",
+		"comparison"
+	];
+	parseResult(generations, _callbacks) {
+		const { text } = generations[0];
+		const parsed = text.trim().split("\n");
+		let reasoning;
+		let verdict;
+		if (parsed.length === 1) [verdict] = parsed;
+		else {
+			reasoning = parsed.slice(0, parsed.length - 1).join("");
+			verdict = parsed[parsed.length - 1];
+		}
+		verdict = verdict.replace(/\[+/, "").replace(/]+/, "");
+		if (![
+			"A",
+			"B",
+			"C"
+		].includes(verdict)) throw new Error(`Invalid verdict: ${verdict}. Verdict must be one of 'A', 'B', or 'C'.`);
+		const score = {
+			A: 1,
+			B: 0,
+			C: .5
+		}[verdict];
+		if (score === void 0) throw new Error("Could not parse score from evaluator output.");
+		return Promise.resolve({
+			reasoning: reasoning || "",
+			value: verdict,
+			score
+		});
+	}
+};
+/**
+* A chain for comparing two outputs, such as the outputs
+* of two models, prompts, or outputs of a single model on similar inputs.
+*/
+var PairwiseStringEvalChain = class extends LLMPairwiseStringEvaluator {
+	static lc_name() {
+		return "PairwiseStringEvalChain";
+	}
+	criterionName;
+	evaluationName = this.criterionName;
+	requiresInput = true;
+	requiresReference = false;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.
+To use references, use the LabeledPairwiseStringEvalChain instead.`;
+	outputParser = new PairwiseStringResultOutputParser();
+	static resolvePairwiseCriteria(criteria) {
+		if (criteria === void 0) {
+			const defaultCriteria = [
+				"helpfulness",
+				"relevance",
+				"correctness",
+				"depth"
+			];
+			return defaultCriteria.reduce((accumulator, currentValue) => {
+				accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
+				return accumulator;
+			}, {});
+		}
+		let criteria_ = {};
+		if (typeof criteria === "string") {
+			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
+		} else if (criteria instanceof ConstitutionalPrinciple) criteria_ = { [criteria.name]: criteria.critiqueRequest };
+		else {
+			if (!criteria) throw new Error("Criteria cannot be empty. Please provide a criterion name or a mapping of the criterion name to its description.");
+			criteria_ = { ...criteria };
+		}
+		return criteria_;
+	}
+	static resolvePairwisePrompt(prompt) {
+		const _prompt = prompt || PROMPT;
+		const expectedInputVars = new Set([
+			"prediction",
+			"predictionB",
+			"input",
+			"criteria"
+		]);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		return _prompt;
+	}
+	/**
+	* Create a new instance of the PairwiseStringEvalChain.
+	* @param llm
+	* @param criteria The criteria to use for evaluation.
+	* @param chainOptions Options to pass to the chain.
+	*/
+	static async fromLLM(llm, criteria, chainOptions) {
+		let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);
+		const criteria_ = this.resolvePairwiseCriteria(criteria);
+		const criteriaStr = Object.entries(criteria_).map(([k, v]) => `${k}: ${v}`).join("\n");
+		prompt = await prompt.partial({ criteria: criteriaStr });
+		const options = chainOptions;
+		if (options) delete options.prompt;
+		return new this({
+			llm,
+			prompt,
+			...options
+		});
+	}
+	_prepareOutput(result) {
+		const parsed = result[this.outputKey];
+		if (RUN_KEY in result && result[RUN_KEY]) parsed[RUN_KEY] = result[RUN_KEY];
+		return parsed;
+	}
+	async _evaluateStringPairs(args, callOptions, config) {
+		const result = await this.call({
+			...args,
+			...callOptions
+		}, config);
+		return this._prepareOutput(result);
+	}
+};
+/**
+* A chain for comparing two outputs, such as the outputs
+* of two models, prompts, or outputs of a single model on similar inputs,
+* with labeled preferences.
+*/
+var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
+	static lc_name() {
+		return "LabeledPairwiseStringEvalChain";
+	}
+	requiresReference = true;
+	static resolvePairwisePrompt(prompt) {
+		const _prompt = prompt || PROMPT_WITH_REFERENCES;
+		const expectedInputVars = new Set([
+			"input",
+			"prediction",
+			"predictionB",
+			"reference",
+			"criteria"
+		]);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		return _prompt;
+	}
+};
+
+//#endregion
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
+//# sourceMappingURL=pairwise.js.map

package/dist/evaluation/comparison/pairwise.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pairwise.js","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","criteria?: CriteriaLike","defaultCriteria: Criteria[]","accumulator: Record<string, string>","criteria_: { [key: string]: string }","prompt?: BasePromptTemplate","expectedInputVars: Set<string>","inputVarsSet: Set<string>","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","args: LLMPairwiseStringEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAKD,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAa;CAExD,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,UAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;EAAI,EAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,QAAQ,0CAAE,CACc;EAIhD,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACJ,EAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM;AAGlB,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACD,EAAC;CACH;AACF;;;;;AAMD,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK;kEACN,CAAC;CAEjE,eAAe,IAAI;CAEnB,OAAO,wBACLC,UACwB;AACxB,MAAI,aAAa,QAAW;GAC1B,MAAMC,kBAA8B;IAClC;IACA;IACA;IACA;GACD;AAED,UAAO,gBAAgB,OACrB,CAACC,aAAqC,iBAAiB;IACrD,YAAY,gBAAgB,mBAAmB;AAC/C,WAAO;GACR,GACD,CAAE,EACH;EACF;EAED,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;CAED,OAAO,sBAAsBC,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAP,UACAQ,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EACb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;AAE3B,SAAO;CACR;CAED,MAAM,qBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAa,GAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;;;AAOD,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,oBAAoB;CAEpB,OAAO,sBAAsBR,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}

package/dist/evaluation/comparison/prompt.cjs

@@ -0,0 +1,77 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+const __langchain_core_prompts = require_rolldown_runtime.__toESM(require("@langchain/core/prompts"));
+
+//#region src/evaluation/comparison/prompt.ts
+const template = `Act as a fair judge and rate the two responses to the question below.\
+ Choose the response that best followed the instructions and answered the question.\
+ Your assessment should weigh the following criteria:
+{criteria}\
+ Start by comparing both responses and give a brief rationale.\
+ Avoid bias from the order of presentation or response length.
+After giving your rationale, make your final decision using this format:\
+ "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
+ and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
+
+[QUESTION]
+{input}
+[/QUESTION]
+
+[RESPONSE A]
+{prediction}
+[/RESPONSE A]
+
+[RESPONSE B]
+{predictionB}
+[/RESPONSE B]`;
+const PROMPT = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
+	inputVariables: [
+		"input",
+		"prediction",
+		"predictionB",
+		"criteria"
+	],
+	template
+});
+const referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\
+ Choose the response that best followed the instructions and answered the question.\
+ Your assessment should weigh the following criteria:
+{criteria}\
+ Start by comparing both responses and give a brief rationale.\
+ Avoid bias from the order of presentation or response length.\
+ Weigh accuracy based on the following ground truth reference\
+ answer to the question:
+
+[REFERENCE]
+{reference}
+[/REFERENCE]
+
+After giving your rationale, make your final decision using this format:\
+ "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
+ and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
+
+[QUESTION]
+{input}
+[/QUESTION]
+
+[RESPONSE A]
+{prediction}
+[/RESPONSE A]
+
+[RESPONSE B]
+{predictionB}
+[/RESPONSE B]`;
+const PROMPT_WITH_REFERENCES = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
+	inputVariables: [
+		"input",
+		"prediction",
+		"predictionB",
+		"reference",
+		"criteria"
+	],
+	template: referenceTemplate
+});
+
+//#endregion
+exports.PROMPT = PROMPT;
+exports.PROMPT_WITH_REFERENCES = PROMPT_WITH_REFERENCES;
+//# sourceMappingURL=prompt.cjs.map

package/dist/evaluation/comparison/prompt.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.cjs","names":["PromptTemplate"],"sources":["../../../src/evaluation/comparison/prompt.ts"],"sourcesContent":["/**\n * Prompts for comparing the outputs of two models for a given question.\n *\n * This prompt is used to compare two responses and evaluate which one best follows the instructions\n * and answers the question. The prompt is based on the paper from\n * Zheng, et. al. https://arxiv.org/abs/2306.05685\n */\n\nimport { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst template = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"prediction\", \"predictionB\", \"criteria\"],\n  template,\n});\n\nconst referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\\\n Weigh accuracy based on the following ground truth reference\\\n answer to the question:\n\n[REFERENCE]\n{reference}\n[/REFERENCE]\n\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT_WITH_REFERENCES = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\n    \"input\",\n    \"prediction\",\n    \"predictionB\",\n    \"reference\",\n    \"criteria\",\n  ],\n  template: referenceTemplate,\n});\n"],"mappings":";;;;AAUA,MAAM,WAAW,CAAC;;;;;;;;;;;;;;;;;;;;aAoBL,CAAC;AAEd,MAAa,yBAAyB,IAAIA,wCAAe;CACvD,gBAAgB;EAAC;EAAS;EAAc;EAAe;CAAW;CAClE;AACD;AAED,MAAM,oBAAoB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;aA2Bd,CAAC;AAEd,MAAa,yCAAyC,IAAIA,wCAAe;CACvE,gBAAgB;EACd;EACA;EACA;EACA;EACA;CACD;CACD,UAAU;AACX"}