@langchain/classic 1.0.40 → 1.0.41

package/dist/evaluation/agents/trajectory.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.cjs","names":["BaseLLMOutputParser","AgentTrajectoryEvaluator","EVAL_CHAT_PROMPT","TOOL_FREE_EVAL_CHAT_PROMPT","RUN_KEY"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4CA,+BAAAA,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyCC,aAAAA,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAUC,eAAAA;MAEV,WAAUC,eAAAA;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}
+{"version":3,"file":"trajectory.cjs","names":["BaseLLMOutputParser","AgentTrajectoryEvaluator","EVAL_CHAT_PROMPT","TOOL_FREE_EVAL_CHAT_PROMPT","RUN_KEY"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;AAwBA,IAAa,yBAAb,cAA4CA,+BAAAA,oBAAoC;CAC9E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAQ;CAEnD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,IAAI,CAAC,KAAK,SAAS,QAAQ,GACzB,MAAM,IAAI,MAAM,8CAA8C,MAAM;EAGtE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,CAAC;EAClD,YAAY,UAAU,KAAK;EAC3B,WAAW,SAAS,KAAK;EAQzB,MAAM,aAAa,SAAS,MAAM,eAAe;EACjD,IAAI,eAAe,QAAQ,WAAW,EAAE,CAAC,SAAS,GAAG,GACnD,MAAM,IAAI,MACR,mDAAmD,MACrD;EAGF,MAAM,QAAQ,CAAC,WAAW;EAC1B,IAAI,QAAQ,KAAK,QAAQ,GACvB,MAAM,IAAI,MAAM,0CAA0C,MAAM;EAGlE,MAAM,mBAAmB,QAAQ,KAAK;EAEtC,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;EACT,CAAC;CACH;AACF;;;;;;;AAQA,IAAa,sBAAb,cAAyCC,aAAAA,yBAAyB;CAChE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,uBAAuB;CAE1C,OAAO,wBACL,QACA,YACA;EACA,IAAI;EACJ,IAAI,QACF,UAAU;OACL,IAAI,YACT,UAAUC,eAAAA;OAEV,UAAUC,eAAAA;EAGZ,OAAO;CACT;;;;;;CAOA,OAAO,iBAAiB,YAA+C;EACrE,OAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,aACvD,CAAC,CACA,KAAK,MAAM;CAChB;;;;;;;CAQA,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,UAAU;EAC1E,IAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,UAAU;GACzD,SAAS,MAAM,OAAO,QAAQ,EAAE,iBAAiB,CAAC;EACpD;EAEA,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;;;;;;;CAQA,mBAAmB,OAA4B;EAC7C,OAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;GAEhC,OACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;EAEpB,CAAC,CAAC,CACD,KAAK,MAAM;CAChB;CAEA,gBAAgB,WAA4B;EAC1C,IAAI,CAAC,WACH,OAAO;EAET,OAAO;;;EAGT,UAAU;;;CAGV;CAEA,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,eAAe;GACxD,QAAQ;GACR,WAAW,KAAK,gBAAgB,SAAS;EAC3C;EAEA,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;EAAY,GAAG,MAAM;EAEpE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF"}

package/dist/evaluation/agents/trajectory.d.cts

@@ -4,11 +4,9 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
-import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { StructuredToolInterface } from "@langchain/core/tools";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
-
 //#region src/evaluation/agents/trajectory.d.ts
 /**
  * A parser for the output of the TrajectoryEvalChain.
@@ -31,7 +29,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.cts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,gCAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"trajectory.d.cts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;cAwBa,+BAA+B,oBAAoB;SACvD;EAIP;EAEA,YACE,aAAa,eAAe,kBAC5B,YAAY,wBACX,QAAQ;;;;;;;;cA4CA,4BAA4B;SAChC;EAIP;EAEA;EAEA;EAEA;EAEA,cAAY;SAEL,wBACL,SAAS,gCACT,aAAa,4BAAyB,gEAAA;;;;;;SAmBjC,iBAAiB,YAAY;;;;;;;SAevB,QACX,KAAK,eACL,aAAa,2BACb,eAAe,QAAQ,KAAK,6BAA0B,QAAA;EAqBxD,eAAe,QAAQ;;;;;;;EAcvB,mBAAmB,OAAO;EAe1B,gBAAgB;EAYV,yBACJ,MAAM,4BACN,aAAa,oCACb,SAAS,YAAY,qBACpB,QAAQ"}

package/dist/evaluation/agents/trajectory.d.ts

@@ -6,9 +6,7 @@ import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
-import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { AgentStep } from "@langchain/core/agents";
-
 //#region src/evaluation/agents/trajectory.d.ts
 /**
  * A parser for the output of the TrajectoryEvalChain.
@@ -31,7 +29,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.ts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,gCAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"trajectory.d.ts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;cAwBa,+BAA+B,oBAAoB;SACvD;EAIP;EAEA,YACE,aAAa,eAAe,kBAC5B,YAAY,wBACX,QAAQ;;;;;;;;cA4CA,4BAA4B;SAChC;EAIP;EAEA;EAEA;EAEA;EAEA,cAAY;SAEL,wBACL,SAAS,gCACT,aAAa,4BAAyB,gEAAA;;;;;;SAmBjC,iBAAiB,YAAY;;;;;;;SAevB,QACX,KAAK,eACL,aAAa,2BACb,eAAe,QAAQ,KAAK,6BAA0B,QAAA;EAqBxD,eAAe,QAAQ;;;;;;;EAcvB,mBAAmB,OAAO;EAe1B,gBAAgB;EAYV,yBACJ,MAAM,4BACN,aAAa,oCACb,SAAS,YAAY,qBACpB,QAAQ"}

package/dist/evaluation/agents/trajectory.js.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.js","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAU;MAEV,WAAU;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}
+{"version":3,"file":"trajectory.js","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAQ;CAEnD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,IAAI,CAAC,KAAK,SAAS,QAAQ,GACzB,MAAM,IAAI,MAAM,8CAA8C,MAAM;EAGtE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,CAAC;EAClD,YAAY,UAAU,KAAK;EAC3B,WAAW,SAAS,KAAK;EAQzB,MAAM,aAAa,SAAS,MAAM,eAAe;EACjD,IAAI,eAAe,QAAQ,WAAW,EAAE,CAAC,SAAS,GAAG,GACnD,MAAM,IAAI,MACR,mDAAmD,MACrD;EAGF,MAAM,QAAQ,CAAC,WAAW;EAC1B,IAAI,QAAQ,KAAK,QAAQ,GACvB,MAAM,IAAI,MAAM,0CAA0C,MAAM;EAGlE,MAAM,mBAAmB,QAAQ,KAAK;EAEtC,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;EACT,CAAC;CACH;AACF;;;;;;;AAQA,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,uBAAuB;CAE1C,OAAO,wBACL,QACA,YACA;EACA,IAAI;EACJ,IAAI,QACF,UAAU;OACL,IAAI,YACT,UAAU;OAEV,UAAU;EAGZ,OAAO;CACT;;;;;;CAOA,OAAO,iBAAiB,YAA+C;EACrE,OAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,aACvD,CAAC,CACA,KAAK,MAAM;CAChB;;;;;;;CAQA,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,UAAU;EAC1E,IAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,UAAU;GACzD,SAAS,MAAM,OAAO,QAAQ,EAAE,iBAAiB,CAAC;EACpD;EAEA,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;EAE3B,OAAO;CACT;;;;;;;CAQA,mBAAmB,OAA4B;EAC7C,OAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;GAEhC,OACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;EAEpB,CAAC,CAAC,CACD,KAAK,MAAM;CAChB;CAEA,gBAAgB,WAA4B;EAC1C,IAAI,CAAC,WACH,OAAO;EAET,OAAO;;;EAGT,UAAU;;;CAGV;CAEA,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,eAAe;GACxD,QAAQ;GACR,WAAW,KAAK,gBAAgB,SAAS;EAC3C;EAEA,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;EAAY,GAAG,MAAM;EAEpE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF"}

package/dist/evaluation/base.cjs

@@ -159,6 +159,8 @@ var AgentTrajectoryEvaluator = class extends LLMEvalChain {
 };
 //#endregion
 exports.AgentTrajectoryEvaluator = AgentTrajectoryEvaluator;
+exports.EvalChain = EvalChain;
+exports.LLMEvalChain = LLMEvalChain;
 exports.LLMPairwiseStringEvaluator = LLMPairwiseStringEvaluator;
 exports.LLMStringEvaluator = LLMStringEvaluator;
 exports.PairwiseStringEvaluator = PairwiseStringEvaluator;

package/dist/evaluation/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,kBAAAA,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,aAAAA,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}
+{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC;;;;AAUvD,IAAsB,eAAtB,cAGUA,kBAAAA,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;EAC5D,IAAI,KAAK,iBAAiB,SAAS,MACjC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,2BAA2B;OAC/D,IAAI,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,gBAAgB;EAEpC,IAAI,KAAK,qBAAqB,aAAa,MACzC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,8BAA8B;OAClE,IAAI,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,oBAAoB;CAE1C;AACF;;;;AAKA,IAAsB,YAAtB,cAGUC,aAAAA,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;EAC5D,IAAI,KAAK,iBAAiB,SAAS,MACjC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,2BAA2B;OAC/D,IAAI,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,gBAAgB;EAEpC,IAAI,KAAK,qBAAqB,aAAa,MACzC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,8BAA8B;OAClE,IAAI,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,oBAAoB;CAE1C;AACF;;;;;AAqDA,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,iBAAiB,MAAM,MAAM;CAC3C;AACF;;;;;AAMA,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,iBAAiB,MAAM,MAAM;CAC3C;AACF;;;;AAKA,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;EACtB,OAAO,KAAK,qBAAqB,MAAM,MAAM;CAC/C;AACF;;;;AAKA,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,qBAAqB,MAAM,aAAa,MAAM;CAC5D;AACF;;;;AAKA,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,yBAAyB,MAAM,aAAa,MAAM;CAChE;AACF"}

package/dist/evaluation/base.d.cts

@@ -1,10 +1,10 @@
 import { BaseChain } from "../chains/base.cjs";
 import { LLMChain, LLMChainInput } from "../chains/llm_chain.cjs";
+import "../chains/index.cjs";
 import { AgentStep } from "@langchain/core/agents";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
-
 //#region src/evaluation/base.d.ts
 /**
  * Base input for evaluators.
@@ -227,5 +227,5 @@ declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {
   evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
 }
 //#endregion
-export { AgentTrajectoryEvaluator, EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
+export { AgentTrajectoryEvaluator, EvalChain, EvalOutputType, ExtractLLMCallOptions, LLMEvalChain, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
 //# sourceMappingURL=base.d.cts.map

package/dist/evaluation/base.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,kBACf,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,cAAc,GAAG;KAEf,sBAAsB,0BAChC,+BAA+B,sCAGvB,eAEJ;;;;KAeM,iBAAiB;;;;uBAKP,aACpB,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,SAAS,GAAG;EACpB;EAEA;EAEA;EAEA;;;;;;;EAQA,oBAAoB,oBAAoB;;;;;uBAiBpB,UACpB,iBAAiB,cAAc,aAC/B,kBAAkB,cAAc,qBACxB,UAAU,UAAU;EAC5B;EAEA;EAEA;EAEA;;;;;;;EAQA,oBAAoB,oBAAoB;;;;;;;UAmBzB;EACf;EACA;EACA;;;;;;UAOe;EACf;EACA;;;;;;;;UASe;EACf;EACA;EACA;EACA;;;;;;;;;UAUe;EACf;EACA;EACA;EACA,iBAAiB;;;;;;uBAOG,mBACpB,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,aAAa,GAAG;;;;EAIxB;;;;;;;;;;;WAYS,iBACP,MAAM,sBAAsB,oCAC5B,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;;;;EAYX,gBACE,MAAM,sBAAsB,oCAC5B,SAAS,YAAY,qBACpB,QAAQ;;;;;;uBAUS,wBAAwB;;;;EAI5C;;;;;;;;;;WAWS,iBACP,MAAM,qBACN,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;;;EAWX,gBACE,MAAM,qBACN,SAAS,YAAY,qBACpB,QAAQ;;;;;uBASS,gCAAgC;;;;EAIpD;;;;;;;WAQS,qBACP,MAAM,6BACN,SAAS,YAAY,qBACpB,QAAQ;;;;;;;EAQX,oBACE,MAAM,6BACN,SAAS,YAAY,qBACpB,QAAQ;;;;;uBAQS,mCAAmC;;;;EAIvD;;;;;;;;WASS,qBACP,MAAM,gCACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;EASX,oBACE,MAAM,gCACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;uBASS,iCAAiC;EACrD;;;;EAKA;;;;;;;;WASS,yBACP,MAAM,4BACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;EASX,wBACE,MAAM,4BACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ"}

package/dist/evaluation/base.d.ts

@@ -1,10 +1,10 @@
 import { BaseChain } from "../chains/base.js";
 import { LLMChain, LLMChainInput } from "../chains/llm_chain.js";
+import "../chains/index.js";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { ChainValues } from "@langchain/core/utils/types";
 import { AgentStep } from "@langchain/core/agents";
-
 //#region src/evaluation/base.d.ts
 /**
  * Base input for evaluators.
@@ -227,5 +227,5 @@ declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {
   evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
 }
 //#endregion
-export { AgentTrajectoryEvaluator, EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
+export { AgentTrajectoryEvaluator, EvalChain, EvalOutputType, ExtractLLMCallOptions, LLMEvalChain, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
 //# sourceMappingURL=base.d.ts.map

package/dist/evaluation/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,kBACf,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,cAAc,GAAG;KAEf,sBAAsB,0BAChC,+BAA+B,sCAGvB,eAEJ;;;;KAeM,iBAAiB;;;;uBAKP,aACpB,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,SAAS,GAAG;EACpB;EAEA;EAEA;EAEA;;;;;;;EAQA,oBAAoB,oBAAoB;;;;;uBAiBpB,UACpB,iBAAiB,cAAc,aAC/B,kBAAkB,cAAc,qBACxB,UAAU,UAAU;EAC5B;EAEA;EAEA;EAEA;;;;;;;EAQA,oBAAoB,oBAAoB;;;;;;;UAmBzB;EACf;EACA;EACA;;;;;;UAOe;EACf;EACA;;;;;;;;UASe;EACf;EACA;EACA;EACA;;;;;;;;;UAUe;EACf;EACA;EACA;EACA,iBAAiB;;;;;;uBAOG,mBACpB,UAAU,iBAAiB,gBAC3B,UAAU,6BAA6B,oCAC/B,aAAa,GAAG;;;;EAIxB;;;;;;;;;;;WAYS,iBACP,MAAM,sBAAsB,oCAC5B,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;;;;EAYX,gBACE,MAAM,sBAAsB,oCAC5B,SAAS,YAAY,qBACpB,QAAQ;;;;;;uBAUS,wBAAwB;;;;EAI5C;;;;;;;;;;WAWS,iBACP,MAAM,qBACN,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;;;EAWX,gBACE,MAAM,qBACN,SAAS,YAAY,qBACpB,QAAQ;;;;;uBASS,gCAAgC;;;;EAIpD;;;;;;;WAQS,qBACP,MAAM,6BACN,SAAS,YAAY,qBACpB,QAAQ;;;;;;;EAQX,oBACE,MAAM,6BACN,SAAS,YAAY,qBACpB,QAAQ;;;;;uBAQS,mCAAmC;;;;EAIvD;;;;;;;;WASS,qBACP,MAAM,gCACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;EASX,oBACE,MAAM,gCACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;uBASS,iCAAiC;EACrD;;;;EAKA;;;;;;;;WASS,yBACP,MAAM,4BACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ;;;;;;;;EASX,wBACE,MAAM,4BACN,cAAc,oCACd,SAAS,YAAY,qBACpB,QAAQ"}

package/dist/evaluation/base.js

@@ -158,6 +158,6 @@ var AgentTrajectoryEvaluator = class extends LLMEvalChain {
 	}
 };
 //#endregion
-export { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator, eqSet };
+export { AgentTrajectoryEvaluator, EvalChain, LLMEvalChain, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator, eqSet };
 
 //# sourceMappingURL=base.js.map

package/dist/evaluation/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":[],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}
+{"version":3,"file":"base.js","names":[],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC;;;;AAUvD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;EAC5D,IAAI,KAAK,iBAAiB,SAAS,MACjC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,2BAA2B;OAC/D,IAAI,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,gBAAgB;EAEpC,IAAI,KAAK,qBAAqB,aAAa,MACzC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,8BAA8B;OAClE,IAAI,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,oBAAoB;CAE1C;AACF;;;;AAKA,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;EAC5D,IAAI,KAAK,iBAAiB,SAAS,MACjC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,2BAA2B;OAC/D,IAAI,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,gBAAgB;EAEpC,IAAI,KAAK,qBAAqB,aAAa,MACzC,MAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,8BAA8B;OAClE,IAAI,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,oBAAoB;CAE1C;AACF;;;;;AAqDA,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,iBAAiB,MAAM,MAAM;CAC3C;AACF;;;;;AAMA,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,iBAAiB,MAAM,MAAM;CAC3C;AACF;;;;AAKA,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;EACtB,OAAO,KAAK,qBAAqB,MAAM,MAAM;CAC/C;AACF;;;;AAKA,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,qBAAqB,MAAM,aAAa,MAAM;CAC5D;AACF;;;;AAKA,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,KAAK;EACnD,OAAO,KAAK,yBAAyB,MAAM,aAAa,MAAM;CAChE;AACF"}

package/dist/evaluation/comparison/index.cjs

@@ -1 +1,4 @@
-require("./pairwise.cjs");
+const require_pairwise = require("./pairwise.cjs");
+exports.LabeledPairwiseStringEvalChain = require_pairwise.LabeledPairwiseStringEvalChain;
+exports.PairwiseStringEvalChain = require_pairwise.PairwiseStringEvalChain;
+exports.PairwiseStringResultOutputParser = require_pairwise.PairwiseStringResultOutputParser;

package/dist/evaluation/comparison/index.d.cts

@@ -1 +1,2 @@
-import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.cjs";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.cjs";
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/index.d.ts

@@ -1 +1,2 @@
-import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/index.js

@@ -1,2 +1,2 @@
-import "./pairwise.js";
-export {};
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";
+export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.cjs

@@ -1,4 +1,3 @@
-require("../../_virtual/_rolldown/runtime.cjs");
 const require_constitutional_principle = require("../../chains/constitutional_ai/constitutional_principle.cjs");
 require("../../chains/index.cjs");
 const require_base = require("../base.cjs");
@@ -100,7 +99,7 @@ To use references, use the LabeledPairwiseStringEvalChain instead.`;
 	}
 	static resolvePairwisePrompt(prompt) {
 		const _prompt = prompt || require_prompt.PROMPT;
-		const expectedInputVars = new Set([
+		const expectedInputVars = /* @__PURE__ */ new Set([
 			"prediction",
 			"predictionB",
 			"input",
@@ -153,7 +152,7 @@ var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
 	requiresReference = true;
 	static resolvePairwisePrompt(prompt) {
 		const _prompt = prompt || require_prompt.PROMPT_WITH_REFERENCES;
-		const expectedInputVars = new Set([
+		const expectedInputVars = /* @__PURE__ */ new Set([
 			"input",
 			"prediction",
 			"predictionB",

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,KAAA,EACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,KAAA,EAQf,QAAO;GANL;GACA;GACA;GACA;GAGoB,CAAC,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,iCAAAA,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,eAER,CAAC,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,eAER,CAAC,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}
+{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAKA,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAY;CAEvD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI;EACJ,IAAI;EAEJ,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,UAAU,QAAQ,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,MAAM,EAAE;EACrD,IAAI,CAAC;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,SAAS,OAAO,GACnC,MAAM,IAAI,MACR,oBAAoB,QAAQ,2CAE9B;EAGF,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACL,EAAE;EAEF,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MAAM,8CAA8C;EAGhE,OAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACF,CAAC;CACH;AACF;;;;;AAMA,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,iCAAiC;CAEpD,OAAO,wBACL,UACwB;EACxB,IAAI,aAAa,KAAA,GAQf,OAAO;GANL;GACA;GACA;GACA;EAGmB,CAAC,CAAC,QACpB,aAAqC,iBAAiB;GACrD,YAAY,gBAAgB,mBAAmB;GAC/C,OAAO;EACT,GACA,CAAC,CACH;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoBC,iCAAAA,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;CAEA,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACC,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,MAAM;EAE5D,MAAM,YAAY,KAAK,wBAAwB,QAAQ;EACvD,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EACZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;CAEA,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAY,GAAG,MAAM;EAElE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;;;AAOA,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACF,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}

package/dist/evaluation/comparison/pairwise.d.cts

@@ -5,9 +5,7 @@ import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
-
 //#region src/evaluation/comparison/pairwise.d.ts
 /**
  * A parser for the output of the PairwiseStringEvalChain.
@@ -30,7 +28,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +47,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.d.cts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;AAAA"}
+{"version":3,"file":"pairwise.d.cts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;cA0Ca,yCAAyC,oBAAoB;SACjE;EAIP;EAEA,YACE,aAAa,eAAe,kBAC5B,YAAY,wBACX,QAAQ;;;;;;cA6CA,gCAAgC;SACpC;EAIP;EAEA;EAEA;EAEA;EAEA;EAGA,cAAY;SAEL,wBACL,WAAW,eACV;SAwCI,sBAAsB,SAAS,qBAAkB,gEAAA;;;;;;;SA2B3C,QACX,KAAK,4BACL,WAAW,cACX,eAAe,QAAQ,KAAK,6BAA0B,QAAA;EAuBxD,eAAe,QAAQ;EAQjB,qBACJ,MAAM,gCACN,aAAa,oCACb,SAAS,YAAY,qBACpB,QAAQ;;;;;;;cAYA,uCAAuC;SAC3C;EAIP;SAEO,sBAAsB,SAAS,qBAAkB,gEAAA"}

package/dist/evaluation/comparison/pairwise.d.ts

@@ -6,8 +6,6 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
-import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
-
 //#region src/evaluation/comparison/pairwise.d.ts
 /**
  * A parser for the output of the PairwiseStringEvalChain.
@@ -30,7 +28,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +47,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import("@langchain/core/prompt_values").BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.d.ts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;AAAA"}
+{"version":3,"file":"pairwise.d.ts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;cA0Ca,yCAAyC,oBAAoB;SACjE;EAIP;EAEA,YACE,aAAa,eAAe,kBAC5B,YAAY,wBACX,QAAQ;;;;;;cA6CA,gCAAgC;SACpC;EAIP;EAEA;EAEA;EAEA;EAEA;EAGA,cAAY;SAEL,wBACL,WAAW,eACV;SAwCI,sBAAsB,SAAS,qBAAkB,gEAAA;;;;;;;SA2B3C,QACX,KAAK,4BACL,WAAW,cACX,eAAe,QAAQ,KAAK,6BAA0B,QAAA;EAuBxD,eAAe,QAAQ;EAQjB,qBACJ,MAAM,gCACN,aAAa,oCACb,SAAS,YAAY,qBACpB,QAAQ;;;;;;;cAYA,uCAAuC;SAC3C;EAIP;SAEO,sBAAsB,SAAS,qBAAkB,gEAAA"}