@langchain/classic 1.0.0-alpha.1

package/dist/evaluation/agents/trajectory.d.ts

@@ -0,0 +1,61 @@
+import { BasePromptValueInterface } from "../../libs/langchain-core/dist/prompt_values.js";
+import { AgentTrajectoryEvaluator, EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMTrajectoryEvaluatorArgs } from "../base.js";
+import { BasePromptTemplate } from "@langchain/core/prompts";
+import { StructuredToolInterface } from "@langchain/core/tools";
+import { ChatGeneration, Generation } from "@langchain/core/outputs";
+import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+import { ChainValues } from "@langchain/core/utils/types";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+import { AgentStep } from "@langchain/core/agents";
+
+//#region src/evaluation/agents/trajectory.d.ts
+/**
+ * A parser for the output of the TrajectoryEvalChain.
+ */
+declare class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {
+  static lc_name(): string;
+  lc_namespace: string[];
+  parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;
+}
+/**
+ * A chain for evaluating ReAct style agents.
+ *
+ * This chain is used to evaluate ReAct style agents by reasoning about
+ * the sequence of actions taken and their outcomes.
+ */
+declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
+  static lc_name(): string;
+  criterionName?: string;
+  evaluationName?: string;
+  requiresInput: boolean;
+  requiresReference: boolean;
+  outputParser: TrajectoryOutputParser;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, BasePromptValueInterface, any>;
+  /**
+   * Get the description of the agent tools.
+   *
+   * @returns The description of the agent tools.
+   */
+  static toolsDescription(agentTools: StructuredToolInterface[]): string;
+  /**
+   * Create a new TrajectoryEvalChain.
+   * @param llm
+   * @param agentTools - The tools used by the agent.
+   * @param chainOptions - The options for the chain.
+   */
+  static fromLLM(llm: BaseChatModel, agentTools?: StructuredToolInterface[], chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>): Promise<TrajectoryEvalChain>;
+  _prepareOutput(result: ChainValues): any;
+  /**
+   * Get the agent trajectory as a formatted string.
+   *
+   * @param steps - The agent trajectory.
+   * @returns The formatted agent trajectory.
+   */
+  getAgentTrajectory(steps: AgentStep[]): string;
+  formatReference(reference?: string): string;
+  _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+//#endregion
+export { TrajectoryEvalChain, TrajectoryOutputParser };
+//# sourceMappingURL=trajectory.d.ts.map

package/dist/evaluation/agents/trajectory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.d.ts","names":["StructuredToolInterface","BaseLLMOutputParser","AgentStep","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","BaseChatModel","AgentTrajectoryEvaluator","EvalOutputType","LLMEvalChainInput","LLMTrajectoryEvaluatorArgs","ExtractLLMCallOptions","TrajectoryOutputParser","Promise","TrajectoryEvalChain","____________langchain_core_dist_prompt_values_js0","BasePromptValueInterface","Omit","Partial"],"sources":["../../../src/evaluation/agents/trajectory.d.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport { AgentTrajectoryEvaluator, EvalOutputType, LLMEvalChainInput, LLMTrajectoryEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport declare class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    outputParser: TrajectoryOutputParser;\n    static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, import(\"../../../../langchain-core/dist/prompt_values.js\").BasePromptValueInterface, any>;\n    /**\n     * Get the description of the agent tools.\n     *\n     * @returns The description of the agent tools.\n     */\n    static toolsDescription(agentTools: StructuredToolInterface[]): string;\n    /**\n     * Create a new TrajectoryEvalChain.\n     * @param llm\n     * @param agentTools - The tools used by the agent.\n     * @param chainOptions - The options for the chain.\n     */\n    static fromLLM(llm: BaseChatModel, agentTools?: StructuredToolInterface[], chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<TrajectoryEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    /**\n     * Get the agent trajectory as a formatted string.\n     *\n     * @param steps - The agent trajectory.\n     * @returns The formatted agent trajectory.\n     */\n    getAgentTrajectory(steps: AgentStep[]): string;\n    formatReference(reference?: string): string;\n    _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAYqBe,cAAAA,sBAAAA,SAA+Bd,mBAAT,CAA6BU,cAA7B,CAAA,CAAA;EAAA,OAAA,OAAA,CAAA,CAAA,EAAA,MAAA;EAAA,YAA6BA,EAAAA,MAAAA,EAAAA;EAAc,WAGzDN,CAAAA,WAAAA,EAAAA,UAAAA,EAAAA,GAAeD,cAAfC,EAAAA,EAAAA,UAAAA,EAA6CE,SAA7CF,GAAAA,SAAAA,CAAAA,EAAqEW,OAArEX,CAA6EM,cAA7EN,CAAAA;;;;;;AAH0C;AAWvE;AAAwC,cAAnBY,mBAAAA,SAA4BP,wBAAAA,CAAT;EAAA,OAMtBK,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAsB,aACIT,CAAAA,EAAAA,MAAAA;EAAkB,cAA2BN,CAAAA,EAAAA,MAAAA;EAAuB,aAAA,EAAA,OAAA;EAAgH,iBAA3GM,EAAAA,OAAAA;EAAkB,YAM/FN,EAPtBe,sBAOsBf;EAAuB,OAOvCS,uBAAAA,CAAAA,MAAAA,CAAAA,EAboBH,kBAapBG,GAAAA,SAAAA,EAAAA,UAAAA,CAAAA,EAbiET,uBAajES,EAAAA,CAAAA,EAb6FH,kBAa7FG,CAAAA,GAAAA,EAbwF,wBAAA,EAaxFA,GAAAA,CAAAA;EAAa;;;;;EAA8H,OAA3BO,gBAAAA,CAAAA,UAAAA,EAPhGhB,uBAOgGgB,EAAAA,CAAAA,EAAAA,MAAAA;EAAO;;;;;;EAUQ,OAAWb,OAAAA,CAAAA,GAAAA,EAV1IM,aAU0IN,EAAAA,UAAAA,CAAAA,EAV9GH,uBAU8GG,EAAAA,EAAAA,YAAAA,CAAAA,EAVpEkB,OAUoElB,CAV5DiB,IAU4DjB,CAVvDS,iBAUuDT,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAV1Ba,OAU0Bb,CAVlBc,mBAUkBd,CAAAA;EAAW,cAAnBa,CAAAA,MAAAA,EAT/Hb,WAS+Ha,CAAAA,EAAAA,GAAAA;EAAO;AA9BxF;;;;;4BA4B3Cd;;iCAEKW,yCAAyCC,6CAA6CP,YAAYC,qBAAqBQ,QAAQb"}

package/dist/evaluation/agents/trajectory.js

@@ -0,0 +1,130 @@
+import { AgentTrajectoryEvaluator } from "../base.js";
+import { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from "./prompt.js";
+import { RUN_KEY } from "@langchain/core/outputs";
+import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
+
+//#region src/evaluation/agents/trajectory.ts
+/**
+* A parser for the output of the TrajectoryEvalChain.
+*/
+var TrajectoryOutputParser = class extends BaseLLMOutputParser {
+	static lc_name() {
+		return "TrajectoryOutputParser";
+	}
+	lc_namespace = [
+		"langchain",
+		"evaluation",
+		"agents"
+	];
+	parseResult(generations, _callbacks) {
+		const { text } = generations[0];
+		if (!text.includes("Score:")) throw new Error(`Could not find score in model eval output: ${text}`);
+		let [reasoning, scoreStr] = text.split("Score:", 2);
+		reasoning = reasoning.trim();
+		scoreStr = scoreStr.trim();
+		const scoreMatch = scoreStr.match(/(\d+(\.\d+)?)/);
+		if (scoreMatch === null || scoreMatch[1].includes(".")) throw new Error(`Score is not an integer digit in the range 1-5: ${text}`);
+		const score = +scoreMatch[1];
+		if (score < 1 || score > 5) throw new Error(`Score is not a digit in the range 1-5: ${text}`);
+		const normalizedScore = (score - 1) / 4;
+		return Promise.resolve({
+			reasoning,
+			score: normalizedScore
+		});
+	}
+};
+/**
+* A chain for evaluating ReAct style agents.
+*
+* This chain is used to evaluate ReAct style agents by reasoning about
+* the sequence of actions taken and their outcomes.
+*/
+var TrajectoryEvalChain = class extends AgentTrajectoryEvaluator {
+	static lc_name() {
+		return "TrajectoryEvalChain";
+	}
+	criterionName;
+	evaluationName = this.criterionName;
+	requiresInput = true;
+	requiresReference = false;
+	outputParser = new TrajectoryOutputParser();
+	static resolveTrajectoryPrompt(prompt, agentTools) {
+		let _prompt;
+		if (prompt) _prompt = prompt;
+		else if (agentTools) _prompt = EVAL_CHAT_PROMPT;
+		else _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;
+		return _prompt;
+	}
+	/**
+	* Get the description of the agent tools.
+	*
+	* @returns The description of the agent tools.
+	*/
+	static toolsDescription(agentTools) {
+		return agentTools.map((tool, i) => `Tool ${i + 1}: ${tool.name}\n Description: ${tool.description}`).join("\n\n");
+	}
+	/**
+	* Create a new TrajectoryEvalChain.
+	* @param llm
+	* @param agentTools - The tools used by the agent.
+	* @param chainOptions - The options for the chain.
+	*/
+	static async fromLLM(llm, agentTools, chainOptions) {
+		let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);
+		if (agentTools) {
+			const toolDescriptions = this.toolsDescription(agentTools);
+			prompt = await prompt.partial({ toolDescriptions });
+		}
+		const options = chainOptions;
+		if (options) delete options.prompt;
+		return new this({
+			llm,
+			prompt,
+			...options
+		});
+	}
+	_prepareOutput(result) {
+		const parsed = result[this.outputKey];
+		if (RUN_KEY in result && result[RUN_KEY]) parsed[RUN_KEY] = result[RUN_KEY];
+		return parsed;
+	}
+	/**
+	* Get the agent trajectory as a formatted string.
+	*
+	* @param steps - The agent trajectory.
+	* @returns The formatted agent trajectory.
+	*/
+	getAgentTrajectory(steps) {
+		return steps.map((step, i) => {
+			const { action, observation } = step;
+			return `Step ${i + 1}:\nTool used: ${action.tool}\nTool input: ${action.toolInput}\nTool output: ${observation}`;
+		}).join("\n\n");
+	}
+	formatReference(reference) {
+		if (!reference) return "";
+		return `
+The following is the expected answer. Use this to measure correctness:
+[GROUND_TRUTH]
+${reference}
+[END_GROUND_TRUTH]
+        `;
+	}
+	async _evaluateAgentTrajectory(args, callOptions, config) {
+		const { input, prediction, reference, agentTrajectory } = args;
+		const inputs = {
+			question: input,
+			agentTrajectory: this.getAgentTrajectory(agentTrajectory),
+			answer: prediction,
+			reference: this.formatReference(reference)
+		};
+		const result = await this.call({
+			...inputs,
+			...callOptions
+		}, config);
+		return this._prepareOutput(result);
+	}
+};
+
+//#endregion
+export { TrajectoryEvalChain, TrajectoryOutputParser };
+//# sourceMappingURL=trajectory.js.map

package/dist/evaluation/agents/trajectory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trajectory.js","names":["generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","prompt?: BasePromptTemplate | undefined","agentTools?: StructuredToolInterface[]","agentTools: StructuredToolInterface[]","llm: BaseChatModel","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","steps: AgentStep[]","reference?: string","args: LLMTrajectoryEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAS;CAEpD,YACEA,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,CAAC,2CAA2C,EAAE,MAAM;EAGtE,IAAI,CAAC,WAAW,SAAS,GAAG,KAAK,MAAM,UAAU,EAAE;EACnD,YAAY,UAAU,MAAM;EAC5B,WAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,CAAC,gDAAgD,EAAE,MAAM;EAI7D,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,CAAC,uCAAuC,EAAE,MAAM;EAGlE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;EACR,EAAC;CACH;AACF;;;;;;;AAQD,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI;CAEnB,OAAO,wBACLC,QACAC,YACA;EACA,IAAI;AACJ,MAAI,QACF,UAAU;WACD,YACT,UAAU;OAEV,UAAU;AAGZ,SAAO;CACR;;;;;;CAOD,OAAO,iBAAiBC,YAA+C;AACrE,SAAO,WACJ,IACC,CAAC,MAAM,MACL,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,KAAK,gBAAgB,EAAE,KAAK,aAAa,CACnE,CACA,KAAK,OAAO;CAChB;;;;;;;CAQD,aAAa,QACXC,KACAF,YACAG,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;GAC1D,SAAS,MAAM,OAAO,QAAQ,EAAE,iBAAkB,EAAC;EACpD;EAED,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;AAE3B,SAAO;CACR;;;;;;;CAQD,mBAAmBC,OAA4B;AAC7C,SAAO,MACJ,IAAI,CAAC,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,aAAa,GAAG;AAEhC,UACE,CAAC,KAAK,EAAE,IAAI,EAAE,cAAG,EACH,OAAO,KAAK,cAAE,EACb,OAAO,UAAU,eAAE,EAClB,aAAa;EAEhC,EAAC,CACD,KAAK,OAAO;CAChB;CAED,gBAAgBC,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO,CAAC;;;AAGZ,EAAE,UAAU;;QAEJ,CAAC;CACN;CAED,MAAM,yBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,iBAAiB,GAAG;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;EAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;EAAa,GAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF"}

package/dist/evaluation/base.cjs

@@ -0,0 +1,169 @@
+const require_base = require('../chains/base.cjs');
+const require_llm_chain = require('../chains/llm_chain.cjs');
+require('../chains/index.cjs');
+
+//#region src/evaluation/base.ts
+/**
+* Compare two sets for equality
+*
+* @param xs
+* @param ys
+*/
+const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+/**
+* Base llm chain class for evaluators.
+*/
+var LLMEvalChain = class extends require_llm_chain.LLMChain {
+	requiresInput = false;
+	requiresReference = false;
+	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
+	/**
+	* Check if the evaluation arguments are valid.
+	* @param reference  The reference label.
+	* @param input The input string.
+	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+	*/
+	checkEvaluationArgs(reference, input) {
+		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
+		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
+		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
+		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
+	}
+};
+/**
+* Base chain class for evaluators.
+*/
+var EvalChain = class extends require_base.BaseChain {
+	requiresInput = false;
+	requiresReference = false;
+	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
+	/**
+	* Check if the evaluation arguments are valid.
+	* @param reference  The reference label.
+	* @param input The input string.
+	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+	*/
+	checkEvaluationArgs(reference, input) {
+		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
+		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
+		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
+		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
+	}
+};
+/**
+* Grade, tag, or otherwise evaluate predictions relative to their inputs
+* and/or reference labels
+*/
+var LLMStringEvaluator = class extends LLMEvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate Chain or LLM output, based on optional input and label.
+	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+	* - score: the score of the evaluation, if applicable.
+	* - value: the string value of the evaluation, if applicable.
+	* - reasoning: the reasoning for the evaluation, if applicable.
+	* @param args
+	* @param callOptions
+	* @param config
+	*/
+	evaluateStrings(args, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStrings(args, config);
+	}
+};
+/**
+* Grade, tag, or otherwise evaluate predictions relative to their inputs
+* and/or reference labels
+*/
+var StringEvaluator = class extends EvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate Chain or LLM output, based on optional input and label.
+	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+	* - score: the score of the evaluation, if applicable.
+	* - value: the string value of the evaluation, if applicable.
+	* - reasoning: the reasoning for the evaluation, if applicable.
+	* @param args
+	* @param config
+	*/
+	evaluateStrings(args, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStrings(args, config);
+	}
+};
+/**
+* Compare the output of two models (or two outputs of the same model).
+*/
+var PairwiseStringEvaluator = class extends EvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate the output string pairs.
+	* @param args
+	* @param config
+	* @return A dictionary containing the preference, scores, and/or other information.
+	*/
+	evaluateStringPairs(args, config) {
+		return this._evaluateStringPairs(args, config);
+	}
+};
+/**
+* Compare the output of two models (or two outputs of the same model).
+*/
+var LLMPairwiseStringEvaluator = class extends LLMEvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate the output string pairs.
+	* @param args
+	* @param callOptions
+	* @param config
+	* @return A dictionary containing the preference, scores, and/or other information.
+	*/
+	evaluateStringPairs(args, callOptions, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStringPairs(args, callOptions, config);
+	}
+};
+/**
+* Interface for evaluating agent trajectories.
+*/
+var AgentTrajectoryEvaluator = class extends LLMEvalChain {
+	requiresInput = true;
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate a trajectory.
+	* @return The evaluation result.
+	* @param args
+	* @param callOptions
+	* @param config
+	*/
+	evaluateAgentTrajectory(args, callOptions, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateAgentTrajectory(args, callOptions, config);
+	}
+};
+
+//#endregion
+exports.AgentTrajectoryEvaluator = AgentTrajectoryEvaluator;
+exports.LLMPairwiseStringEvaluator = LLMPairwiseStringEvaluator;
+exports.LLMStringEvaluator = LLMStringEvaluator;
+exports.PairwiseStringEvaluator = PairwiseStringEvaluator;
+exports.StringEvaluator = StringEvaluator;
+exports.eqSet = eqSet;
+//# sourceMappingURL=base.cjs.map

package/dist/evaluation/base.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.cjs","names":["xs: Set<string>","ys: Set<string>","LLMChain","reference?: string","input?: string","BaseChain","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","args: StringEvaluatorArgs","args: PairwiseStringEvaluatorArgs","args: LLMPairwiseStringEvaluatorArgs","callOptions?: ExtractLLMCallOptions<this[\"llm\"]>","args: LLMTrajectoryEvaluatorArgs"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,QAAQ,CAACA,IAAiBC,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUC,2BAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBC,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;AAKD,IAAsB,YAAtB,cAGUC,uBAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBF,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;;AAqDD,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACEE,MACAC,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;;AAMD,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACEC,MACAD,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;AAKD,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACEE,MACAF,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;CAC/C;AACF;;;;AAKD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACEG,MACAC,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;CAC5D;AACF;;;;AAKD,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACEK,MACAD,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO;CAChE;AACF"}

package/dist/evaluation/base.d.cts

@@ -0,0 +1,240 @@
+import { BaseChain } from "../chains/base.cjs";
+import { LLMChain, LLMChainInput } from "../chains/llm_chain.cjs";
+import { AgentStep } from "@langchain/core/agents";
+import { ChainValues } from "@langchain/core/utils/types";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
+
+//#region src/evaluation/base.d.ts
+/**
+ * Base input for evaluators.
+ */
+interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {}
+type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+any, infer CallOptions> ? CallOptions : never;
+/**
+ * Compare two sets for equality
+ *
+ * @param xs
+ * @param ys
+ */
+
+/**
+ * The type of the output of an evaluation evaluator.
+ */
+type EvalOutputType = Record<string, string | number | boolean>;
+/**
+ * Base llm chain class for evaluators.
+ */
+declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {
+  requiresInput?: boolean;
+  requiresReference?: boolean;
+  skipInputWarning?: string;
+  skipReferenceWarning?: string;
+  /**
+   * Check if the evaluation arguments are valid.
+   * @param reference  The reference label.
+   * @param input The input string.
+   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+   */
+  checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * Base chain class for evaluators.
+ */
+declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {
+  requiresInput?: boolean;
+  requiresReference?: boolean;
+  skipInputWarning?: string;
+  skipReferenceWarning?: string;
+  /**
+   * Check if the evaluation arguments are valid.
+   * @param reference  The reference label.
+   * @param input The input string.
+   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+   */
+  checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * @field prediction The output string from the  model.
+ * @field reference The expected output / reference string.
+ * @field input The input string.
+ */
+interface StringEvaluatorArgs {
+  prediction: string;
+  reference?: string;
+  input?: string;
+}
+/**
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ */
+interface PairwiseStringEvaluatorArgs {
+  prediction: string;
+  predictionB: string;
+}
+/**
+ * @field The input string.
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ * @field reference The expected output / reference string.
+ */
+interface LLMPairwiseStringEvaluatorArgs {
+  input: string;
+  prediction: string;
+  predictionB: string;
+  reference?: string;
+}
+/**
+ * Args for AgentTrajectoryEvaluator
+ * @field input The input to the agent.
+ * @field prediction The final predicted response.
+ * @field reference The reference answer.
+ * @field agentTrajectory  The intermediate steps forming the agent trajectory.
+ */
+interface LLMTrajectoryEvaluatorArgs {
+  input: string;
+  prediction: string;
+  reference?: string;
+  agentTrajectory: AgentStep[];
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+declare abstract class StringEvaluator extends EvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param config
+   */
+  abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param config
+   */
+  evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+declare abstract class PairwiseStringEvaluator extends EvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param callOptions
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param callOptions
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Interface for evaluating agent trajectories.
+ */
+declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {
+  requiresInput: boolean;
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate a trajectory.
+   * @return The evaluation result.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate a trajectory.
+   * @return The evaluation result.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+//#endregion
+export { AgentTrajectoryEvaluator, EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
+//# sourceMappingURL=base.d.cts.map