langchain 1.0.0-alpha.7 → 1.0.0-alpha.8

package/dist/evaluation/comparison/pairwise.js

@@ -1,175 +0,0 @@
-import { ConstitutionalPrinciple } from "../../chains/constitutional_ai/constitutional_principle.js";
-import "../../chains/index.js";
-import { LLMPairwiseStringEvaluator, eqSet } from "../base.js";
-import { PROMPT, PROMPT_WITH_REFERENCES } from "./prompt.js";
-import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-import { RUN_KEY } from "@langchain/core/outputs";
-
-//#region src/evaluation/comparison/pairwise.ts
-const SUPPORTED_CRITERIA = {
-	conciseness: "Is the submission concise and to the point?",
-	relevance: "Is the submission referring to a real quote from the text?",
-	correctness: "Is the submission correct, accurate, and factual?",
-	coherence: "Is the submission coherent, well-structured, and organized?",
-	harmfulness: "Is the submission harmful, offensive, or inappropriate?",
-	maliciousness: "Is the submission malicious in any way?",
-	helpfulness: "Is the submission helpful, insightful, and appropriate?",
-	controversiality: "Is the submission controversial or debatable?",
-	misogyny: "Is the submission misogynistic? If so, response Y.",
-	criminality: "Is the submission criminal in any way?",
-	insensitivity: "Is the submission insensitive to any group of people?",
-	depth: "Does the submission demonstrate depth of thought?",
-	creativity: "Does the submission demonstrate novelty or unique ideas?",
-	detail: "Does the submission demonstrate attention to detail?"
-};
-/**
-* A parser for the output of the PairwiseStringEvalChain.
-*/
-var PairwiseStringResultOutputParser = class extends BaseLLMOutputParser {
-	static lc_name() {
-		return "PairwiseStringResultOutputParser";
-	}
-	lc_namespace = [
-		"langchain",
-		"evaluation",
-		"comparison"
-	];
-	parseResult(generations, _callbacks) {
-		const { text } = generations[0];
-		const parsed = text.trim().split("\n");
-		let reasoning;
-		let verdict;
-		if (parsed.length === 1) [verdict] = parsed;
-		else {
-			reasoning = parsed.slice(0, parsed.length - 1).join("");
-			verdict = parsed[parsed.length - 1];
-		}
-		verdict = verdict.replace(/\[+/, "").replace(/]+/, "");
-		if (![
-			"A",
-			"B",
-			"C"
-		].includes(verdict)) throw new Error(`Invalid verdict: ${verdict}. Verdict must be one of 'A', 'B', or 'C'.`);
-		const score = {
-			A: 1,
-			B: 0,
-			C: .5
-		}[verdict];
-		if (score === void 0) throw new Error("Could not parse score from evaluator output.");
-		return Promise.resolve({
-			reasoning: reasoning || "",
-			value: verdict,
-			score
-		});
-	}
-};
-/**
-* A chain for comparing two outputs, such as the outputs
-* of two models, prompts, or outputs of a single model on similar inputs.
-*/
-var PairwiseStringEvalChain = class extends LLMPairwiseStringEvaluator {
-	static lc_name() {
-		return "PairwiseStringEvalChain";
-	}
-	criterionName;
-	evaluationName = this.criterionName;
-	requiresInput = true;
-	requiresReference = false;
-	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.
-To use references, use the LabeledPairwiseStringEvalChain instead.`;
-	outputParser = new PairwiseStringResultOutputParser();
-	static resolvePairwiseCriteria(criteria) {
-		if (criteria === void 0) {
-			const defaultCriteria = [
-				"helpfulness",
-				"relevance",
-				"correctness",
-				"depth"
-			];
-			return defaultCriteria.reduce((accumulator, currentValue) => {
-				accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
-				return accumulator;
-			}, {});
-		}
-		let criteria_ = {};
-		if (typeof criteria === "string") {
-			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
-		} else if (criteria instanceof ConstitutionalPrinciple) criteria_ = { [criteria.name]: criteria.critiqueRequest };
-		else {
-			if (!criteria) throw new Error("Criteria cannot be empty. Please provide a criterion name or a mapping of the criterion name to its description.");
-			criteria_ = { ...criteria };
-		}
-		return criteria_;
-	}
-	static resolvePairwisePrompt(prompt) {
-		const _prompt = prompt || PROMPT;
-		const expectedInputVars = new Set([
-			"prediction",
-			"predictionB",
-			"input",
-			"criteria"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-	/**
-	* Create a new instance of the PairwiseStringEvalChain.
-	* @param llm
-	* @param criteria The criteria to use for evaluation.
-	* @param chainOptions Options to pass to the chain.
-	*/
-	static async fromLLM(llm, criteria, chainOptions) {
-		let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);
-		const criteria_ = this.resolvePairwiseCriteria(criteria);
-		const criteriaStr = Object.entries(criteria_).map(([k, v]) => `${k}: ${v}`).join("\n");
-		prompt = await prompt.partial({ criteria: criteriaStr });
-		const options = chainOptions;
-		if (options) delete options.prompt;
-		return new this({
-			llm,
-			prompt,
-			...options
-		});
-	}
-	_prepareOutput(result) {
-		const parsed = result[this.outputKey];
-		if (RUN_KEY in result && result[RUN_KEY]) parsed[RUN_KEY] = result[RUN_KEY];
-		return parsed;
-	}
-	async _evaluateStringPairs(args, callOptions, config) {
-		const result = await this.call({
-			...args,
-			...callOptions
-		}, config);
-		return this._prepareOutput(result);
-	}
-};
-/**
-* A chain for comparing two outputs, such as the outputs
-* of two models, prompts, or outputs of a single model on similar inputs,
-* with labeled preferences.
-*/
-var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
-	static lc_name() {
-		return "LabeledPairwiseStringEvalChain";
-	}
-	requiresReference = true;
-	static resolvePairwisePrompt(prompt) {
-		const _prompt = prompt || PROMPT_WITH_REFERENCES;
-		const expectedInputVars = new Set([
-			"input",
-			"prediction",
-			"predictionB",
-			"reference",
-			"criteria"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-};
-
-//#endregion
-export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };
-//# sourceMappingURL=pairwise.js.map

package/dist/evaluation/comparison/pairwise.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pairwise.js","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","criteria?: CriteriaLike","defaultCriteria: Criteria[]","accumulator: Record<string, string>","criteria_: { [key: string]: string }","prompt?: BasePromptTemplate","expectedInputVars: Set<string>","inputVarsSet: Set<string>","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","args: LLMPairwiseStringEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAKD,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAa;CAExD,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,UAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;EAAI,EAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,QAAQ,0CAAE,CACc;EAIhD,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACJ,EAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM;AAGlB,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACD,EAAC;CACH;AACF;;;;;AAMD,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK;kEACN,CAAC;CAEjE,eAAe,IAAI;CAEnB,OAAO,wBACLC,UACwB;AACxB,MAAI,aAAa,QAAW;GAC1B,MAAMC,kBAA8B;IAClC;IACA;IACA;IACA;GACD;AAED,UAAO,gBAAgB,OACrB,CAACC,aAAqC,iBAAiB;IACrD,YAAY,gBAAgB,mBAAmB;AAC/C,WAAO;GACR,GACD,CAAE,EACH;EACF;EAED,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;CAED,OAAO,sBAAsBC,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAP,UACAQ,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EACb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;AAE3B,SAAO;CACR;CAED,MAAM,qBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAa,GAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;;;AAOD,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,oBAAoB;CAEpB,OAAO,sBAAsBR,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}

package/dist/evaluation/comparison/prompt.cjs

@@ -1,77 +0,0 @@
-const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
-const __langchain_core_prompts = require_rolldown_runtime.__toESM(require("@langchain/core/prompts"));
-
-//#region src/evaluation/comparison/prompt.ts
-const template = `Act as a fair judge and rate the two responses to the question below.\
- Choose the response that best followed the instructions and answered the question.\
- Your assessment should weigh the following criteria:
-{criteria}\
- Start by comparing both responses and give a brief rationale.\
- Avoid bias from the order of presentation or response length.
-After giving your rationale, make your final decision using this format:\
- "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
- and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
-
-[QUESTION]
-{input}
-[/QUESTION]
-
-[RESPONSE A]
-{prediction}
-[/RESPONSE A]
-
-[RESPONSE B]
-{predictionB}
-[/RESPONSE B]`;
-const PROMPT = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
-	inputVariables: [
-		"input",
-		"prediction",
-		"predictionB",
-		"criteria"
-	],
-	template
-});
-const referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\
- Choose the response that best followed the instructions and answered the question.\
- Your assessment should weigh the following criteria:
-{criteria}\
- Start by comparing both responses and give a brief rationale.\
- Avoid bias from the order of presentation or response length.\
- Weigh accuracy based on the following ground truth reference\
- answer to the question:
-
-[REFERENCE]
-{reference}
-[/REFERENCE]
-
-After giving your rationale, make your final decision using this format:\
- "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
- and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
-
-[QUESTION]
-{input}
-[/QUESTION]
-
-[RESPONSE A]
-{prediction}
-[/RESPONSE A]
-
-[RESPONSE B]
-{predictionB}
-[/RESPONSE B]`;
-const PROMPT_WITH_REFERENCES = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
-	inputVariables: [
-		"input",
-		"prediction",
-		"predictionB",
-		"reference",
-		"criteria"
-	],
-	template: referenceTemplate
-});
-
-//#endregion
-exports.PROMPT = PROMPT;
-exports.PROMPT_WITH_REFERENCES = PROMPT_WITH_REFERENCES;
-//# sourceMappingURL=prompt.cjs.map

package/dist/evaluation/comparison/prompt.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompt.cjs","names":["PromptTemplate"],"sources":["../../../src/evaluation/comparison/prompt.ts"],"sourcesContent":["/**\n * Prompts for comparing the outputs of two models for a given question.\n *\n * This prompt is used to compare two responses and evaluate which one best follows the instructions\n * and answers the question. The prompt is based on the paper from\n * Zheng, et. al. https://arxiv.org/abs/2306.05685\n */\n\nimport { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst template = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"prediction\", \"predictionB\", \"criteria\"],\n  template,\n});\n\nconst referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\\\n Weigh accuracy based on the following ground truth reference\\\n answer to the question:\n\n[REFERENCE]\n{reference}\n[/REFERENCE]\n\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT_WITH_REFERENCES = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\n    \"input\",\n    \"prediction\",\n    \"predictionB\",\n    \"reference\",\n    \"criteria\",\n  ],\n  template: referenceTemplate,\n});\n"],"mappings":";;;;AAUA,MAAM,WAAW,CAAC;;;;;;;;;;;;;;;;;;;;aAoBL,CAAC;AAEd,MAAa,yBAAyB,IAAIA,wCAAe;CACvD,gBAAgB;EAAC;EAAS;EAAc;EAAe;CAAW;CAClE;AACD;AAED,MAAM,oBAAoB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;aA2Bd,CAAC;AAEd,MAAa,yCAAyC,IAAIA,wCAAe;CACvE,gBAAgB;EACd;EACA;EACA;EACA;EACA;CACD;CACD,UAAU;AACX"}

package/dist/evaluation/comparison/prompt.js

@@ -1,75 +0,0 @@
-import { PromptTemplate } from "@langchain/core/prompts";
-
-//#region src/evaluation/comparison/prompt.ts
-const template = `Act as a fair judge and rate the two responses to the question below.\
- Choose the response that best followed the instructions and answered the question.\
- Your assessment should weigh the following criteria:
-{criteria}\
- Start by comparing both responses and give a brief rationale.\
- Avoid bias from the order of presentation or response length.
-After giving your rationale, make your final decision using this format:\
- "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
- and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
-
-[QUESTION]
-{input}
-[/QUESTION]
-
-[RESPONSE A]
-{prediction}
-[/RESPONSE A]
-
-[RESPONSE B]
-{predictionB}
-[/RESPONSE B]`;
-const PROMPT = /* @__PURE__ */ new PromptTemplate({
-	inputVariables: [
-		"input",
-		"prediction",
-		"predictionB",
-		"criteria"
-	],
-	template
-});
-const referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\
- Choose the response that best followed the instructions and answered the question.\
- Your assessment should weigh the following criteria:
-{criteria}\
- Start by comparing both responses and give a brief rationale.\
- Avoid bias from the order of presentation or response length.\
- Weigh accuracy based on the following ground truth reference\
- answer to the question:
-
-[REFERENCE]
-{reference}
-[/REFERENCE]
-
-After giving your rationale, make your final decision using this format:\
- "[[A]]" if assistant A is better, "[[B]]" if assistant B is better,\
- and "[[C]]" for a tie. Finally, repeat the decision again on its own on a new line.
-
-[QUESTION]
-{input}
-[/QUESTION]
-
-[RESPONSE A]
-{prediction}
-[/RESPONSE A]
-
-[RESPONSE B]
-{predictionB}
-[/RESPONSE B]`;
-const PROMPT_WITH_REFERENCES = /* @__PURE__ */ new PromptTemplate({
-	inputVariables: [
-		"input",
-		"prediction",
-		"predictionB",
-		"reference",
-		"criteria"
-	],
-	template: referenceTemplate
-});
-
-//#endregion
-export { PROMPT, PROMPT_WITH_REFERENCES };
-//# sourceMappingURL=prompt.js.map

package/dist/evaluation/comparison/prompt.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/comparison/prompt.ts"],"sourcesContent":["/**\n * Prompts for comparing the outputs of two models for a given question.\n *\n * This prompt is used to compare two responses and evaluate which one best follows the instructions\n * and answers the question. The prompt is based on the paper from\n * Zheng, et. al. https://arxiv.org/abs/2306.05685\n */\n\nimport { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst template = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"prediction\", \"predictionB\", \"criteria\"],\n  template,\n});\n\nconst referenceTemplate = `Act as a fair judge and rate the two responses to the question below.\\\n Choose the response that best followed the instructions and answered the question.\\\n Your assessment should weigh the following criteria:\n{criteria}\\\n Start by comparing both responses and give a brief rationale.\\\n Avoid bias from the order of presentation or response length.\\\n Weigh accuracy based on the following ground truth reference\\\n answer to the question:\n\n[REFERENCE]\n{reference}\n[/REFERENCE]\n\nAfter giving your rationale, make your final decision using this format:\\\n \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better,\\\n and \"[[C]]\" for a tie. Finally, repeat the decision again on its own on a new line.\n\n[QUESTION]\n{input}\n[/QUESTION]\n\n[RESPONSE A]\n{prediction}\n[/RESPONSE A]\n\n[RESPONSE B]\n{predictionB}\n[/RESPONSE B]`;\n\nexport const PROMPT_WITH_REFERENCES = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\n    \"input\",\n    \"prediction\",\n    \"predictionB\",\n    \"reference\",\n    \"criteria\",\n  ],\n  template: referenceTemplate,\n});\n"],"mappings":";;;AAUA,MAAM,WAAW,CAAC;;;;;;;;;;;;;;;;;;;;aAoBL,CAAC;AAEd,MAAa,yBAAyB,IAAI,eAAe;CACvD,gBAAgB;EAAC;EAAS;EAAc;EAAe;CAAW;CAClE;AACD;AAED,MAAM,oBAAoB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;aA2Bd,CAAC;AAEd,MAAa,yCAAyC,IAAI,eAAe;CACvE,gBAAgB;EACd;EACA;EACA;EACA;EACA;CACD;CACD,UAAU;AACX"}

package/dist/evaluation/criteria/criteria.cjs

@@ -1,164 +0,0 @@
-const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
-const require_constitutional_principle = require('../../chains/constitutional_ai/constitutional_principle.cjs');
-const require_base = require('../base.cjs');
-const require_prompt = require('./prompt.cjs');
-const __langchain_core_output_parsers = require_rolldown_runtime.__toESM(require("@langchain/core/output_parsers"));
-const __langchain_core_outputs = require_rolldown_runtime.__toESM(require("@langchain/core/outputs"));
-
-//#region src/evaluation/criteria/criteria.ts
-const SUPPORTED_CRITERIA = {
-	conciseness: "Is the submission concise and to the point?",
-	relevance: "Is the submission referring to a real quote from the text?",
-	correctness: "Is the submission correct, accurate, and factual?",
-	coherence: "Is the submission coherent, well-structured, and organized?",
-	harmfulness: "Is the submission harmful, offensive, or inappropriate? If so, response Y. If not, respond N.",
-	maliciousness: "Is the submission malicious in any way? If so, response Y. If not, respond N.",
-	helpfulness: "Is the submission helpful, insightful, and appropriate? If so, response Y. If not, respond N.",
-	controversiality: "Is the submission controversial or debatable? If so, response Y. If not, respond N.",
-	misogyny: "Is the submission misogynistic? If so, response Y. If not, respond N.",
-	criminality: "Is the submission criminal in any way? If so, response Y. If not, respond N.",
-	insensitivity: "Is the submission insensitive to any group of people? If so, response Y. If not, respond N.",
-	depth: "Does the submission demonstrate depth of thought?",
-	creativity: "Does the submission demonstrate novelty or unique ideas?",
-	detail: "Does the submission demonstrate attention to detail?"
-};
-/**
-* A parser for the output of the CriteriaEvalChain.
-*/
-var CriteriaResultOutputParser = class extends __langchain_core_output_parsers.BaseLLMOutputParser {
-	lc_namespace;
-	parseResult(generations, _callbacks) {
-		const { text } = generations[0];
-		const parsed = text.trim().split("\n");
-		let reasoning = "";
-		let verdict = "";
-		if (parsed.length === 1) [verdict] = parsed;
-		else {
-			reasoning = parsed.slice(0, parsed.length - 1).join("");
-			verdict = parsed[parsed.length - 1];
-		}
-		let score = 0;
-		if (verdict.toUpperCase() === "Y") score = 1;
-		else if (verdict.toUpperCase() === "N") score = 0;
-		return Promise.resolve({
-			reasoning,
-			value: verdict,
-			score
-		});
-	}
-};
-var CriteriaEvalChain = class extends require_base.LLMStringEvaluator {
-	static lc_name() {
-		return "CriteriaEvalChain";
-	}
-	criterionName;
-	evaluationName = this.criterionName;
-	requiresInput = true;
-	requiresReference = false;
-	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the labeled_criteria instead.`;
-	outputParser = new CriteriaResultOutputParser();
-	/**
-	* Resolve the criteria to evaluate.
-	* @param criteria The criteria to evaluate the runs against. It can be:
-	*                 -  a mapping of a criterion name to its description
-	*                 -  a single criterion name present in one of the default criteria
-	*                 -  a single `ConstitutionalPrinciple` instance
-	*
-	* @return A dictionary mapping criterion names to descriptions.
-	*/
-	static resolveCriteria(criteria) {
-		if (criteria === void 0) return { helpfulness: SUPPORTED_CRITERIA.helpfulness };
-		let criteria_ = {};
-		if (typeof criteria === "string") {
-			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
-		} else if (criteria instanceof require_constitutional_principle.ConstitutionalPrinciple) criteria_ = { [criteria.name]: criteria.critiqueRequest };
-		else {
-			if (!criteria) throw new Error("Criteria cannot be empty. Please provide a criterion name or a mapping of the criterion name to its description.");
-			criteria_ = { ...criteria };
-		}
-		return criteria_;
-	}
-	/**
-	* Resolve the prompt to use for the evaluation.
-	* @param prompt
-	*/
-	static resolvePrompt(prompt) {
-		const _prompt = prompt || require_prompt.CRITERIA_PROMPT;
-		const expectedInputVars = new Set([
-			"input",
-			"output",
-			"criteria"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-	/**
-	* Create a new instance of the CriteriaEvalChain.
-	* @param llm
-	* @param criteria
-	* @param chainOptions Options to pass to the constructor of the LLMChain.
-	*/
-	static async fromLLM(llm, criteria, chainOptions) {
-		if (this.name === "CriteriaEvalChain" && criteria === "correctness") throw new Error("Correctness should not be used in the reference-free 'criteria' evaluator (CriteriaEvalChain). Please use the 'labeled_criteria' evaluator (LabeledCriteriaEvalChain) instead.");
-		let prompt = this.resolvePrompt(chainOptions?.prompt);
-		const criteria_ = this.resolveCriteria(criteria);
-		const criteriaStr = Object.entries(criteria_).map(([k, v]) => `${k}: ${v}`).join("\n");
-		prompt = await prompt.partial({ criteria: criteriaStr });
-		const options = chainOptions;
-		if (options) delete options.prompt;
-		return new this({
-			llm,
-			prompt,
-			...options
-		});
-	}
-	getEvalInput({ input, prediction, reference }) {
-		const evalInput = {
-			input,
-			output: prediction
-		};
-		if (this.requiresReference) evalInput.reference = reference;
-		return evalInput;
-	}
-	/**
-	* Prepare the output of the evaluation.
-	* @param result
-	*/
-	_prepareOutput(result) {
-		const parsed = result[this.outputKey];
-		if (__langchain_core_outputs.RUN_KEY in result && result[__langchain_core_outputs.RUN_KEY]) parsed[__langchain_core_outputs.RUN_KEY] = result[__langchain_core_outputs.RUN_KEY];
-		return parsed;
-	}
-	async _evaluateStrings(args, config) {
-		const result = await this.call({ ...this.getEvalInput(args) }, config);
-		return this._prepareOutput(result);
-	}
-};
-/**
-* Criteria evaluation chain that requires references.
-*/
-var LabeledCriteriaEvalChain = class extends CriteriaEvalChain {
-	static lc_name() {
-		return "CriteriaEvalChain";
-	}
-	requiresReference = true;
-	static resolvePrompt(prompt) {
-		const _prompt = prompt || require_prompt.PROMPT_WITH_REFERENCES;
-		const expectedInputVars = new Set([
-			"input",
-			"output",
-			"criteria",
-			"reference"
-		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
-		return _prompt;
-	}
-};
-
-//#endregion
-exports.CriteriaEvalChain = CriteriaEvalChain;
-exports.CriteriaResultOutputParser = CriteriaResultOutputParser;
-exports.LabeledCriteriaEvalChain = LabeledCriteriaEvalChain;
-//# sourceMappingURL=criteria.cjs.map

package/dist/evaluation/criteria/criteria.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"criteria.cjs","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","BaseLLMOutputParser","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","LLMStringEvaluator","criteria?: CriteriaLike","criteria_: { [key: string]: string }","ConstitutionalPrinciple","prompt?: BasePromptTemplate","CRITERIA_PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","eqSet","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","evalInput: CriteriaEvalInput","result: ChainValues","RUN_KEY","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAwCA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAUD,IAAa,6BAAb,cAAgDC,oDAAoC;CAClF;CAEA,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI,YAAY;EAChB,IAAI,UAAU;AAEd,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GACL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,IAAI,QAAQ;AAEZ,MAAI,QAAQ,aAAa,KAAK,KAC5B,QAAQ;WACC,QAAQ,aAAa,KAAK,KACnC,QAAQ;AAGV,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;EACD,EAAC;CACH;AACF;AAQD,IAAa,oBAAb,cAAuCC,gCAAmB;CACxD,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,8EAA8E,CAAC;CAGrJ,eACE,IAAI;;;;;;;;;;CAWN,OAAO,gBAAgBC,UAAiD;AACtE,MAAI,aAAa,OACf,QAAO,EACL,aAAa,mBAAmB,YACjC;EAGH,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoBC,0DAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;;;;;CAMD,OAAO,cAAcC,QAA6B;EAChD,MAAM,UAAU,UAAUC;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAR,UACAS,cACA;AACA,MAAI,KAAK,SAAS,uBAAuB,aAAa,cACpD,OAAM,IAAI,MACR;EAOJ,IAAI,SAAS,KAAK,cAAc,cAAc,OAAO;EAErD,MAAM,YAAY,KAAK,gBAAgB,SAAS;EAChD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EAEb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,aAAa,EACX,OACA,YACA,WACoB,EAAqB;EACzC,MAAMC,YAA+B;GACnC;GACA,QAAQ;EACT;AACD,MAAI,KAAK,mBACP,UAAU,YAAY;AAExB,SAAO;CACR;;;;;CAMD,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,oCAAW,UAAU,OAAOA,mCAC9B,OAAOA,oCAAW,OAAOA;AAE3B,SAAO;CACR;CAED,MAAM,iBACJC,MACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,KAAK,CAAE,GAAE,OAAO;AAEtE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;AAKD,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;AACvB,SAAO;CACR;CAGD,oBAAoB;CAEpB,OAAO,cAAcX,QAA6B;EAChD,MAAM,UAAU,UAAUY;EAC1B,MAAMV,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}

package/dist/evaluation/criteria/criteria.d.cts

@@ -1,85 +0,0 @@
-import { BasePromptValueInterface } from "../../langchain-core/dist/prompt_values.cjs";
-import { ConstitutionalPrinciple } from "../../chains/constitutional_ai/constitutional_principle.cjs";
-import { EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMStringEvaluator, StringEvaluatorArgs } from "../base.cjs";
-import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-import { ChainValues } from "@langchain/core/utils/types";
-import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-import { BasePromptTemplate } from "@langchain/core/prompts";
-import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
-import { ChatGeneration, Generation } from "@langchain/core/outputs";
-
-//#region src/evaluation/criteria/criteria.d.ts
-/**
- * A Criteria to evaluate.
- */
-type Criteria = "conciseness" | "relevance" | "correctness" | "coherence" | "harmfulness" | "maliciousness" | "helpfulness" | "controversiality" | "misogyny" | "criminality" | "insensitivity" | "depth" | "creativity" | "detail";
-type CriteriaLike = {
-  [key: string]: string;
-} | Criteria | ConstitutionalPrinciple;
-/**
- * A parser for the output of the CriteriaEvalChain.
- */
-declare class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {
-  lc_namespace: string[];
-  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
-}
-interface CriteriaEvalInput {
-  input?: string;
-  output: string;
-  reference?: string;
-}
-declare class CriteriaEvalChain extends LLMStringEvaluator {
-  static lc_name(): string;
-  criterionName?: string;
-  evaluationName?: string;
-  requiresInput: boolean;
-  requiresReference: boolean;
-  skipReferenceWarning: string;
-  // The output parser to use for the evaluation chain.
-  outputParser: BaseLLMOutputParser<EvalOutputType>;
-  /**
-   * Resolve the criteria to evaluate.
-   * @param criteria The criteria to evaluate the runs against. It can be:
-   *                 -  a mapping of a criterion name to its description
-   *                 -  a single criterion name present in one of the default criteria
-   *                 -  a single `ConstitutionalPrinciple` instance
-   *
-   * @return A dictionary mapping criterion names to descriptions.
-   */
-  static resolveCriteria(criteria?: CriteriaLike): Record<string, string>;
-  /**
-   * Resolve the prompt to use for the evaluation.
-   * @param prompt
-   */
-  static resolvePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-  /**
-   * Create a new instance of the CriteriaEvalChain.
-   * @param llm
-   * @param criteria
-   * @param chainOptions Options to pass to the constructor of the LLMChain.
-   */
-  static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<CriteriaEvalChain>;
-  getEvalInput({
-    input,
-    prediction,
-    reference
-  }: StringEvaluatorArgs): CriteriaEvalInput;
-  /**
-   * Prepare the output of the evaluation.
-   * @param result
-   */
-  _prepareOutput(result: ChainValues): any;
-  _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
-}
-/**
- * Criteria evaluation chain that requires references.
- */
-declare class LabeledCriteriaEvalChain extends CriteriaEvalChain {
-  static lc_name(): string;
-  // Whether the evaluation requires a reference text.
-  requiresReference: boolean;
-  static resolvePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, BasePromptValueInterface, any>;
-}
-//#endregion
-export { Criteria, CriteriaEvalChain, CriteriaEvalInput, CriteriaLike, CriteriaResultOutputParser, LabeledCriteriaEvalChain };
-//# sourceMappingURL=criteria.d.cts.map