@langchain/classic 1.0.17 → 1.0.18

package/dist/evaluation/agents/trajectory.d.cts

@@ -4,7 +4,7 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
-import * as _langchain_core_prompt_values1 from "@langchain/core/prompt_values";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { StructuredToolInterface } from "@langchain/core/tools";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
@@ -31,7 +31,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values1.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.cts","names":["StructuredToolInterface","BaseLLMOutputParser","AgentStep","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","BaseChatModel","AgentTrajectoryEvaluator","EvalOutputType","LLMEvalChainInput","LLMTrajectoryEvaluatorArgs","ExtractLLMCallOptions","TrajectoryOutputParser","Promise","TrajectoryEvalChain","_langchain_core_prompt_values1","BasePromptValueInterface","Omit","Partial"],"sources":["../../../src/evaluation/agents/trajectory.d.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport { AgentTrajectoryEvaluator, EvalOutputType, LLMEvalChainInput, LLMTrajectoryEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport declare class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    outputParser: TrajectoryOutputParser;\n    static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>;\n    /**\n     * Get the description of the agent tools.\n     *\n     * @returns The description of the agent tools.\n     */\n    static toolsDescription(agentTools: StructuredToolInterface[]): string;\n    /**\n     * Create a new TrajectoryEvalChain.\n     * @param llm\n     * @param agentTools - The tools used by the agent.\n     * @param chainOptions - The options for the chain.\n     */\n    static fromLLM(llm: BaseChatModel, agentTools?: StructuredToolInterface[], chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<TrajectoryEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    /**\n     * Get the agent trajectory as a formatted string.\n     *\n     * @param steps - The agent trajectory.\n     * @returns The formatted agent trajectory.\n     */\n    getAgentTrajectory(steps: AgentStep[]): string;\n    formatReference(reference?: string): string;\n    _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n//# sourceMappingURL=trajectory.d.ts.map"],"mappings":";;;;;;;;;;;;;;;AAYqBe,cAAAA,sBAAAA,SAA+Bd,mBAAT,CAA6BU,cAA7B,CAAA,CAAA;EAA6BA,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAG3CN,YAAAA,EAAAA,MAAAA,EAAAA;EAAeD,WAAAA,CAAAA,WAAAA,EAAfC,UAAeD,EAAAA,GAAAA,cAAAA,EAAAA,EAAAA,UAAAA,EAA8BG,SAA9BH,GAAAA,SAAAA,CAAAA,EAAsDY,OAAtDZ,CAA8DO,cAA9DP,CAAAA;;;;;AAH2B;AAWvE;;AAO4CE,cAPvBW,mBAAAA,SAA4BP,wBAAAA,CAOLJ;EAA6CN,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuBkB,aAAAA,CAAAA,EAAAA,MAAAA;EAAKZ,cAAAA,CAAAA,EAAAA,MAAAA;EAM7EN,aAAAA,EAAAA,OAAAA;EAOhBS,iBAAAA,EAAAA,OAAAA;EAA4BT,YAAAA,EAdlCe,sBAckCf;EAAuDY,OAAAA,uBAAAA,CAAAA,MAAAA,CAAAA,EAb/DN,kBAa+DM,GAAAA,SAAAA,EAAAA,UAAAA,CAAAA,EAblBZ,uBAakBY,EAAAA,CAAAA,EAbUN,kBAaVM,CAAAA,GAAAA,EAbKM,8BAAAA,CAAqEC,wBAAAA,EAa1EP,GAAAA,CAAAA;EAALQ;;;;;EAQxElB,OAAAA,gBAAAA,CAAAA,UAAAA,EAfUF,uBAeVE,EAAAA,CAAAA,EAAAA,MAAAA;EAEKW;;;;;;EA9BcH,OAAAA,OAAAA,CAAAA,GAAAA,EAoBzBD,aApByBC,EAAAA,UAAAA,CAAAA,EAoBGV,uBApBHU,EAAAA,EAAAA,YAAAA,CAAAA,EAoB6CW,OApB7CX,CAoBqDU,IApBrDV,CAoB0DE,iBApB1DF,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAoBuFM,OApBvFN,CAoB+FO,mBApB/FP,CAAAA;EAAwB,cAAA,CAAA,MAAA,EAqB9CP,WArB8C,CAAA,EAAA,GAAA;;;;;;;4BA4B3CD;;iCAEKW,yCAAyCC,6CAA6CP,YAAYC,qBAAqBQ,QAAQb"}
+{"version":3,"file":"trajectory.d.cts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,8BAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/agents/trajectory.d.ts

@@ -6,7 +6,7 @@ import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
-import * as _langchain_core_prompt_values1 from "@langchain/core/prompt_values";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { AgentStep } from "@langchain/core/agents";
 
 //#region src/evaluation/agents/trajectory.d.ts
@@ -31,7 +31,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values1.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.ts","names":["StructuredToolInterface","BaseLLMOutputParser","AgentStep","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","BaseChatModel","AgentTrajectoryEvaluator","EvalOutputType","LLMEvalChainInput","LLMTrajectoryEvaluatorArgs","ExtractLLMCallOptions","TrajectoryOutputParser","Promise","TrajectoryEvalChain","_langchain_core_prompt_values1","BasePromptValueInterface","Omit","Partial"],"sources":["../../../src/evaluation/agents/trajectory.d.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport { AgentTrajectoryEvaluator, EvalOutputType, LLMEvalChainInput, LLMTrajectoryEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport declare class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    outputParser: TrajectoryOutputParser;\n    static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>;\n    /**\n     * Get the description of the agent tools.\n     *\n     * @returns The description of the agent tools.\n     */\n    static toolsDescription(agentTools: StructuredToolInterface[]): string;\n    /**\n     * Create a new TrajectoryEvalChain.\n     * @param llm\n     * @param agentTools - The tools used by the agent.\n     * @param chainOptions - The options for the chain.\n     */\n    static fromLLM(llm: BaseChatModel, agentTools?: StructuredToolInterface[], chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<TrajectoryEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    /**\n     * Get the agent trajectory as a formatted string.\n     *\n     * @param steps - The agent trajectory.\n     * @returns The formatted agent trajectory.\n     */\n    getAgentTrajectory(steps: AgentStep[]): string;\n    formatReference(reference?: string): string;\n    _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n//# sourceMappingURL=trajectory.d.ts.map"],"mappings":";;;;;;;;;;;;;;;AAYqBe,cAAAA,sBAAAA,SAA+Bd,mBAAT,CAA6BU,cAA7B,CAAA,CAAA;EAA6BA,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAG3CN,YAAAA,EAAAA,MAAAA,EAAAA;EAAeD,WAAAA,CAAAA,WAAAA,EAAfC,UAAeD,EAAAA,GAAAA,cAAAA,EAAAA,EAAAA,UAAAA,EAA8BG,SAA9BH,GAAAA,SAAAA,CAAAA,EAAsDY,OAAtDZ,CAA8DO,cAA9DP,CAAAA;;;;;AAH2B;AAWvE;;AAO4CE,cAPvBW,mBAAAA,SAA4BP,wBAAAA,CAOLJ;EAA6CN,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAuBkB,aAAAA,CAAAA,EAAAA,MAAAA;EAAKZ,cAAAA,CAAAA,EAAAA,MAAAA;EAM7EN,aAAAA,EAAAA,OAAAA;EAOhBS,iBAAAA,EAAAA,OAAAA;EAA4BT,YAAAA,EAdlCe,sBAckCf;EAAuDY,OAAAA,uBAAAA,CAAAA,MAAAA,CAAAA,EAb/DN,kBAa+DM,GAAAA,SAAAA,EAAAA,UAAAA,CAAAA,EAblBZ,uBAakBY,EAAAA,CAAAA,EAbUN,kBAaVM,CAAAA,GAAAA,EAbKM,8BAAAA,CAAqEC,wBAAAA,EAa1EP,GAAAA,CAAAA;EAALQ;;;;;EAQxElB,OAAAA,gBAAAA,CAAAA,UAAAA,EAfUF,uBAeVE,EAAAA,CAAAA,EAAAA,MAAAA;EAEKW;;;;;;EA9BcH,OAAAA,OAAAA,CAAAA,GAAAA,EAoBzBD,aApByBC,EAAAA,UAAAA,CAAAA,EAoBGV,uBApBHU,EAAAA,EAAAA,YAAAA,CAAAA,EAoB6CW,OApB7CX,CAoBqDU,IApBrDV,CAoB0DE,iBApB1DF,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAoBuFM,OApBvFN,CAoB+FO,mBApB/FP,CAAAA;EAAwB,cAAA,CAAA,MAAA,EAqB9CP,WArB8C,CAAA,EAAA,GAAA;;;;;;;4BA4B3CD;;iCAEKW,yCAAyCC,6CAA6CP,YAAYC,qBAAqBQ,QAAQb"}
+{"version":3,"file":"trajectory.d.ts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,8BAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/agents/trajectory.js.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.js","names":["generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","prompt?: BasePromptTemplate | undefined","agentTools?: StructuredToolInterface[]","agentTools: StructuredToolInterface[]","llm: BaseChatModel","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","steps: AgentStep[]","reference?: string","args: LLMTrajectoryEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAS;CAEpD,YACEA,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,CAAC,2CAA2C,EAAE,MAAM;EAGtE,IAAI,CAAC,WAAW,SAAS,GAAG,KAAK,MAAM,UAAU,EAAE;EACnD,YAAY,UAAU,MAAM;EAC5B,WAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,CAAC,gDAAgD,EAAE,MAAM;EAI7D,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,CAAC,uCAAuC,EAAE,MAAM;EAGlE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;EACR,EAAC;CACH;AACF;;;;;;;AAQD,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI;CAEnB,OAAO,wBACLC,QACAC,YACA;EACA,IAAI;AACJ,MAAI,QACF,UAAU;WACD,YACT,UAAU;OAEV,UAAU;AAGZ,SAAO;CACR;;;;;;CAOD,OAAO,iBAAiBC,YAA+C;AACrE,SAAO,WACJ,IACC,CAAC,MAAM,MACL,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,KAAK,gBAAgB,EAAE,KAAK,aAAa,CACnE,CACA,KAAK,OAAO;CAChB;;;;;;;CAQD,aAAa,QACXC,KACAF,YACAG,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;GAC1D,SAAS,MAAM,OAAO,QAAQ,EAAE,iBAAkB,EAAC;EACpD;EAED,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;AAE3B,SAAO;CACR;;;;;;;CAQD,mBAAmBC,OAA4B;AAC7C,SAAO,MACJ,IAAI,CAAC,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,aAAa,GAAG;AAEhC,UACE,CAAC,KAAK,EAAE,IAAI,EAAE,cAAG,EACH,OAAO,KAAK,cAAE,EACb,OAAO,UAAU,eAAE,EAClB,aAAa;EAEhC,EAAC,CACD,KAAK,OAAO;CAChB;CAED,gBAAgBC,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO,CAAC;;;AAGZ,EAAE,UAAU;;QAEJ,CAAC;CACN;CAED,MAAM,yBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,iBAAiB,GAAG;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;EAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;EAAa,GAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF"}
+{"version":3,"file":"trajectory.js","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAU;MAEV,WAAU;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}

package/dist/evaluation/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["xs: Set<string>","ys: Set<string>","LLMChain","reference?: string","input?: string","BaseChain","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","args: StringEvaluatorArgs","args: PairwiseStringEvaluatorArgs","args: LLMPairwiseStringEvaluatorArgs","callOptions?: ExtractLLMCallOptions<this[\"llm\"]>","args: LLMTrajectoryEvaluatorArgs"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,QAAQ,CAACA,IAAiBC,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUC,2BAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBC,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;AAKD,IAAsB,YAAtB,cAGUC,uBAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBF,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;;AAqDD,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACEE,MACAC,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;;AAMD,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACEC,MACAD,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;AAKD,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACEE,MACAF,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;CAC/C;AACF;;;;AAKD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACEG,MACAC,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;CAC5D;AACF;;;;AAKD,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACEK,MACAD,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO;CAChE;AACF"}
+{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,2BAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,uBAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}

package/dist/evaluation/base.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.cts","names":["BaseLanguageModelInterface","AgentStep","ChainValues","BaseCallbackConfig","Callbacks","BaseChain","LLMChain","LLMChainInput","LLMEvalChainInput","T","L","EvalOutputType","ExtractLLMCallOptions","LanguageModelInterface","CallOptions","eqSet","Set","Record","LLMEvalChain","EvalChain","RunInput","RunOutput","StringEvaluatorArgs","PairwiseStringEvaluatorArgs","LLMPairwiseStringEvaluatorArgs","LLMTrajectoryEvaluatorArgs","LLMStringEvaluator","Promise","StringEvaluator","PairwiseStringEvaluator","LLMPairwiseStringEvaluator","AgentTrajectoryEvaluator"],"sources":["../../src/evaluation/base.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { BaseCallbackConfig, Callbacks } from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {\n}\nexport type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<any, infer CallOptions> ? CallOptions : never;\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport declare const eqSet: (xs: Set<string>, ys: Set<string>) => boolean;\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n/**\n * Base llm chain class for evaluators.\n */\nexport declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * Base chain class for evaluators.\n */\nexport declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n    prediction: string;\n    reference?: string;\n    input?: string;\n}\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n    prediction: string;\n    predictionB: string;\n}\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n    input: string;\n    prediction: string;\n    predictionB: string;\n    reference?: string;\n}\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n    input: string;\n    prediction: string;\n    reference?: string;\n    agentTrajectory: AgentStep[];\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class StringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class PairwiseStringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Interface for evaluating agent trajectories.\n */\nexport declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n    requiresInput: boolean;\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;;;;;AAQiBQ,UAAAA,iBAAiBE,CAAAA,UAAWC,cAAX,GAA4BA,cAA5B,EAAA,UAAsDX,0BAAtD,GAAmFA,0BAAnF,CAAA,SAAuHO,aAAvH,CAAqIE,CAArI,EAAwIC,CAAxI,CAAA,CAAA;AAA4BC,KAElDC,qBAFkDD,CAAAA,sBAAAA,CAAAA,GAEFE,sBAFEF,SAE6BX,0BAF7BW,CAAAA,GAAAA,EAAAA,KAAAA,YAAAA,CAAAA,GAEkFG,WAFlFH,GAAAA,KAAAA;;;;AAE6F,KAW/IA,cAAAA,GAAiBM,MAX8H,CAAA,MAAA,EAAA,MAAA,GAAA,MAAA,GAAA,OAAA,CAAA;AAW3J;AAIA;;AAAsEN,uBAAxCO,YAAwCP,CAAAA,UAAjBA,cAAiBA,GAAAA,cAAAA,EAAAA,UAA0BX,0BAA1BW,GAAuDX,0BAAvDW,CAAAA,SAA2FL,QAA3FK,CAAoGF,CAApGE,EAAuGD,CAAvGC,CAAAA,CAAAA;EAA0BX,aAAAA,CAAAA,EAAAA,OAAAA;EAA6BA,iBAAAA,CAAAA,EAAAA,OAAAA;EAA6CS,gBAAAA,CAAAA,EAAAA,MAAAA;EAAGC,oBAAAA,CAAAA,EAAAA,MAAAA;EAAZJ;AAAQ;AAgBzK;;;;EAAoHJ,mBAAAA,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;AAA8B;AAkBjIoB,uBAlBaH,SAkBM,CAAA,iBAlBqBjB,WAkBrB,GAlBmCA,WAkBnC,EAAA,kBAlBkEA,WAkBlE,GAlBgFA,WAkBhF,CAAA,SAlBqGG,SAkBrG,CAlB+Ge,QAkB/G,EAlByHC,SAkBzH,CAAA,CAAA;EASnBE,aAAAA,CAAAA,EAAAA,OAAAA;EAUAC,iBAAAA,CAAAA,EAAAA,OAAAA;EAaAC,gBAAAA,CAAAA,EAAAA,MAAAA;EAUaC,oBAAAA,CAAkB,EAAA,MAAAjB;EAAWE;;;;;;EAevBW,mBAAAA,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;;;AAWYV,UApE/BU,mBAAAA,CAoE+BV;EAA6CR,UAAAA,EAAAA,MAAAA;EAAYD,SAAAA,CAAAA,EAAAA,MAAAA;EAA6BD,KAAAA,CAAAA,EAAAA,MAAAA;;;AA1B6C;AAgCnL;;AAckEE,UA/EjDmB,2BAAAA,CA+EiDnB;EAAYD,UAAAA,EAAAA,MAAAA;EAA6BD,WAAAA,EAAAA,MAAAA;;;;;;;;AAd5C,UAvD9CsB,8BAAAA,CAuD8C;EA6BjCK,KAAAA,EAAAA,MAAAA;EAWUN,UAAAA,EAAAA,MAAAA;EAAsCnB,WAAAA,EAAAA,MAAAA;EAAYD,SAAAA,CAAAA,EAAAA,MAAAA;;;;;;;;;AAXnB,UAvEtDsB,0BAAAA,CAuEsD;EAuBzCK,KAAAA,EAAAA,MAAAA;EAYUN,UAAAA,EAAAA,MAAAA;EAA8CZ,SAAAA,CAAAA,EAAAA,MAAAA;EAA6CR,eAAAA,EAtG9GH,SAsG8GG,EAAAA;;;;;;AAQVA,uBAxG3FsB,kBAwG2FtB,CAAAA,UAxG9DO,cAwG8DP,GAxG7CO,cAwG6CP,EAAAA,UAxGnBJ,0BAwGmBI,GAxGUJ,0BAwGVI,CAAAA,SAxG8Cc,YAwG9Cd,CAxG2DK,CAwG3DL,EAxG8DM,CAwG9DN,CAAAA,CAAAA;EAAYD;;;EApBpEe,cAAAA,CAAAA,EAAAA,MAAAA;EAAY;AAyB7E;;;;;;;;;EAqByHd,SAAAA,gBAAAA,CAAAA,IAAAA,EAnHrFkB,mBAmHqFlB,GAnH/DQ,qBAmH+DR,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAnHlBA,SAmHkBA,GAnHND,kBAmHMC,CAAAA,EAnHeuB,OAmHfvB,CAnHuBF,WAmHvBE,CAAAA;EAAYD;;;;AArB1D;;;;;;wBAnFjDmB,sBAAsBV,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;uBAMxG0B,eAAAA,SAAwBT,SAAAA;;;;;;;;;;;;;;kCAclBG,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;;;wBAUjFoB,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKnE2B,uBAAAA,SAAgCV,SAAAA;;;;;;;;;;;sCAWtBI,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;4BAOzFqB,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAK/E4B,0BAAAA,SAAmCZ,YAAAA;;;;;;;;;;;;sCAYzBM,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;4BAQ9IsB,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKpI6B,wBAAAA,SAAiCb,YAAAA;;;;;;;;;;;;;0CAanBO,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;gCAQ1IuB,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB"}
+{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;AAE5B;;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlDiC;AAKnC;;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;AAjDF;UA0DiB,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EApExB;;;EAwEA,cAAA;EAxEsD;AAmBxD;;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1D+B;;;;AAW5C;;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;AAhDb;;uBA0DsB,eAAA,SAAwB,SAAA;EAtDlB;;;EA0D1B,cAAA;EA1DA;;;;AAOF;;;;;EAPE,SAqES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/D4B;;;;;;;;;EA0EvC,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EArF5C;;;EAyFR,cAAA;EA3FU;;;;;;EAAA,SAmGD,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGa;;;;;;EA4GxB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAvF/C;;;EA2FR,cAAA;EA1FuB;;;;;;AAWzB;EAXyB,SAmGd,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;;;;;;;;EASX,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnH4C;;;EAwH5C,cAAA;EAxGE;;;;;;;EAAA,SAiHO,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGT;;;;;;;EAgHF,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","names":["BaseLanguageModelInterface","AgentStep","ChainValues","BaseCallbackConfig","Callbacks","BaseChain","LLMChain","LLMChainInput","LLMEvalChainInput","T","L","EvalOutputType","ExtractLLMCallOptions","LanguageModelInterface","CallOptions","eqSet","Set","Record","LLMEvalChain","EvalChain","RunInput","RunOutput","StringEvaluatorArgs","PairwiseStringEvaluatorArgs","LLMPairwiseStringEvaluatorArgs","LLMTrajectoryEvaluatorArgs","LLMStringEvaluator","Promise","StringEvaluator","PairwiseStringEvaluator","LLMPairwiseStringEvaluator","AgentTrajectoryEvaluator"],"sources":["../../src/evaluation/base.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { BaseCallbackConfig, Callbacks } from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {\n}\nexport type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<any, infer CallOptions> ? CallOptions : never;\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport declare const eqSet: (xs: Set<string>, ys: Set<string>) => boolean;\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n/**\n * Base llm chain class for evaluators.\n */\nexport declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * Base chain class for evaluators.\n */\nexport declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n    prediction: string;\n    reference?: string;\n    input?: string;\n}\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n    prediction: string;\n    predictionB: string;\n}\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n    input: string;\n    prediction: string;\n    predictionB: string;\n    reference?: string;\n}\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n    input: string;\n    prediction: string;\n    reference?: string;\n    agentTrajectory: AgentStep[];\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class StringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class PairwiseStringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Interface for evaluating agent trajectories.\n */\nexport declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n    requiresInput: boolean;\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;;;;;;UAQiBQ,4BAA4BG,iBAAiBA,0BAA0BX,6BAA6BA,oCAAoCO,cAAcE,GAAGC,IAA1K;AAA6CC,KAEjCC,qBAFiCD,CAAAA,sBAAAA,CAAAA,GAEeE,sBAFfF,SAE8CX,0BAF9CW,CAAAA,GAAAA,EAAAA,KAAAA,YAAAA,CAAAA,GAEmGG,WAFnGH,GAAAA,KAAAA;AAE7C;;;AAAgJG,KAWpIH,cAAAA,GAAiBM,MAXmHH,CAAAA,MAAAA,EAAAA,MAAAA,GAAAA,MAAAA,GAAAA,OAAAA,CAAAA;AAAW;AAW3J;AAIA;AAAqDH,uBAAvBO,YAAuBP,CAAAA,UAAAA,cAAAA,GAAiBA,cAAjBA,EAAAA,UAA2CX,0BAA3CW,GAAwEX,0BAAxEW,CAAAA,SAA4GL,QAA5GK,CAAqHF,CAArHE,EAAwHD,CAAxHC,CAAAA,CAAAA;EAAiBA,aAAAA,CAAAA,EAAAA,OAAAA;EAA0BX,iBAAAA,CAAAA,EAAAA,OAAAA;EAA6BA,gBAAAA,CAAAA,EAAAA,MAAAA;EAA6CS,oBAAAA,CAAAA,EAAAA,MAAAA;EAAGC;;AAAJ;AAgBzK;;;EAAsGR,mBAAAA,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;AAA4C,uBAApHiB,SAAoH,CAAA,iBAAzFjB,WAAyF,GAA3EA,WAA2E,EAAA,kBAA5CA,WAA4C,GAA9BA,WAA8B,CAAA,SAATG,SAAS,CAACe,QAAD,EAAWC,SAAX,CAAA,CAAA;EAkBjIC,aAAAA,CAAAA,EAAAA,OAAmB;EASnBC,iBAAAA,CAAAA,EAAAA,OAAAA;EAUAC,gBAAAA,CAAAA,EAAAA,MAAAA;EAaAC,oBAAAA,CAAAA,EAAAA,MAA0B;EAUbC;;;;;;EAAyJhB,mBAAAA,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;;;AA0B7JY,UApETA,mBAAAA,CAoESA;EAAsBV,UAAAA,EAAAA,MAAAA;EAA6CR,SAAAA,CAAAA,EAAAA,MAAAA;EAAYD,KAAAA,CAAAA,EAAAA,MAAAA;;;;AA1B0E;AAgCnL;AAcoCmB,UA/EnBC,2BAAAA,CA+EmBD;EAA8BlB,UAAAA,EAAAA,MAAAA;EAAYD,WAAAA,EAAAA,MAAAA;;;;;;;;AAdxBgB,UAvDrCK,8BAAAA,CAuDqCL;EAAS,KAAA,EAAA,MAAA;EA6BjCU,UAAAA,EAAAA,MAAAA;EAWUN,WAAAA,EAAAA,MAAAA;EAAsCnB,SAAAA,CAAAA,EAAAA,MAAAA;;;;;;;;;AAXhBe,UAvE7CM,0BAAAA,CAuE6CN;EAAS,KAAA,EAAA,MAAA;EAuBzCW,UAAAA,EAAAA,MAAAA;EAYUN,SAAAA,CAAAA,EAAAA,MAAAA;EAA8CZ,eAAAA,EAtGjEX,SAsGiEW,EAAAA;;;;;;AAQVA,uBAxG9Cc,kBAwG8Cd,CAAAA,UAxGjBD,cAwGiBC,GAxGAD,cAwGAC,EAAAA,UAxG0BZ,0BAwG1BY,GAxGuDZ,0BAwGvDY,CAAAA,SAxG2FM,YAwG3FN,CAxGwGH,CAwGxGG,EAxG2GF,CAwG3GE,CAAAA,CAAAA;EAA6CR;;;EAAiCuB,cAAAA,CAAAA,EAAAA,MAAAA;EApBzFT;AAAY;AAyB7E;;;;;;;;EAqB4EN,SAAAA,gBAAAA,CAAAA,IAAAA,EAnHxCU,mBAmHwCV,GAnHlBA,qBAmHkBA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAnH2BR,SAmH3BQ,GAnHuCT,kBAmHvCS,CAAAA,EAnH4De,OAmH5Df,CAnHoEV,WAmHpEU,CAAAA;EAA6CR;;;;;AArB9C;;;;;wBAnFjDkB,sBAAsBV,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;uBAMxG0B,eAAAA,SAAwBT,SAAAA;;;;;;;;;;;;;;kCAclBG,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;;;wBAUjFoB,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKnE2B,uBAAAA,SAAgCV,SAAAA;;;;;;;;;;;sCAWtBI,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;4BAOzFqB,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAK/E4B,0BAAAA,SAAmCZ,YAAAA;;;;;;;;;;;;sCAYzBM,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;4BAQ9IsB,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKpI6B,wBAAAA,SAAiCb,YAAAA;;;;;;;;;;;;;0CAanBO,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;gCAQ1IuB,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB"}
+{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":["xs: Set<string>","ys: Set<string>","reference?: string","input?: string","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","args: StringEvaluatorArgs","args: PairwiseStringEvaluatorArgs","args: LLMPairwiseStringEvaluatorArgs","callOptions?: ExtractLLMCallOptions<this[\"llm\"]>","args: LLMTrajectoryEvaluatorArgs"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,QAAQ,CAACA,IAAiBC,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBC,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;AAKD,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBD,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;;AAqDD,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACEC,MACAC,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;;AAMD,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACEC,MACAD,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;AAKD,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACEE,MACAF,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;CAC/C;AACF;;;;AAKD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACEG,MACAC,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;CAC5D;AACF;;;;AAKD,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACEK,MACAD,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO;CAChE;AACF"}
+{"version":3,"file":"base.js","names":[],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}

package/dist/evaluation/comparison/index.js

@@ -1 +1,3 @@
 import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";
+
+export {  };

package/dist/evaluation/comparison/pairwise.cjs

@@ -1,10 +1,10 @@
-const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
 const require_constitutional_principle = require('../../chains/constitutional_ai/constitutional_principle.cjs');
 require('../../chains/index.cjs');
 const require_base = require('../base.cjs');
 const require_prompt = require('./prompt.cjs');
-const __langchain_core_outputs = require_rolldown_runtime.__toESM(require("@langchain/core/outputs"));
-const __langchain_core_output_parsers = require_rolldown_runtime.__toESM(require("@langchain/core/output_parsers"));
+let _langchain_core_outputs = require("@langchain/core/outputs");
+let _langchain_core_output_parsers = require("@langchain/core/output_parsers");
 
 //#region src/evaluation/comparison/pairwise.ts
 const SUPPORTED_CRITERIA = {
@@ -26,7 +26,7 @@ const SUPPORTED_CRITERIA = {
 /**
 * A parser for the output of the PairwiseStringEvalChain.
 */
-var PairwiseStringResultOutputParser = class extends __langchain_core_output_parsers.BaseLLMOutputParser {
+var PairwiseStringResultOutputParser = class extends _langchain_core_output_parsers.BaseLLMOutputParser {
 	static lc_name() {
 		return "PairwiseStringResultOutputParser";
 	}
@@ -80,18 +80,15 @@ var PairwiseStringEvalChain = class extends require_base.LLMPairwiseStringEvalua
 To use references, use the LabeledPairwiseStringEvalChain instead.`;
 	outputParser = new PairwiseStringResultOutputParser();
 	static resolvePairwiseCriteria(criteria) {
-		if (criteria === void 0) {
-			const defaultCriteria = [
-				"helpfulness",
-				"relevance",
-				"correctness",
-				"depth"
-			];
-			return defaultCriteria.reduce((accumulator, currentValue) => {
-				accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
-				return accumulator;
-			}, {});
-		}
+		if (criteria === void 0) return [
+			"helpfulness",
+			"relevance",
+			"correctness",
+			"depth"
+		].reduce((accumulator, currentValue) => {
+			accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];
+			return accumulator;
+		}, {});
 		let criteria_ = {};
 		if (typeof criteria === "string") {
 			if (criteria in SUPPORTED_CRITERIA) criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };
@@ -110,8 +107,7 @@ To use references, use the LabeledPairwiseStringEvalChain instead.`;
 			"input",
 			"criteria"
 		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 	/**
@@ -135,7 +131,7 @@ To use references, use the LabeledPairwiseStringEvalChain instead.`;
 	}
 	_prepareOutput(result) {
 		const parsed = result[this.outputKey];
-		if (__langchain_core_outputs.RUN_KEY in result && result[__langchain_core_outputs.RUN_KEY]) parsed[__langchain_core_outputs.RUN_KEY] = result[__langchain_core_outputs.RUN_KEY];
+		if (_langchain_core_outputs.RUN_KEY in result && result[_langchain_core_outputs.RUN_KEY]) parsed[_langchain_core_outputs.RUN_KEY] = result[_langchain_core_outputs.RUN_KEY];
 		return parsed;
 	}
 	async _evaluateStringPairs(args, callOptions, config) {
@@ -165,8 +161,7 @@ var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
 			"reference",
 			"criteria"
 		]);
-		const inputVarsSet = new Set(_prompt.inputVariables);
-		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 };

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.cjs","names":["SUPPORTED_CRITERIA: Record<Criteria, string>","BaseLLMOutputParser","generations: Generation[] | ChatGeneration[]","_callbacks: Callbacks | undefined","LLMPairwiseStringEvaluator","criteria?: CriteriaLike","defaultCriteria: Criteria[]","accumulator: Record<string, string>","criteria_: { [key: string]: string }","ConstitutionalPrinciple","prompt?: BasePromptTemplate","PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","eqSet","llm: BaseLanguageModelInterface","chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>","result: ChainValues","RUN_KEY","args: LLMPairwiseStringEvaluatorArgs","callOptions: ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;;AAsBA,MAAMA,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACT;;;;AAKD,IAAa,mCAAb,cAAsDC,oDAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,eAAe;EAAC;EAAa;EAAc;CAAa;CAExD,YACEC,aACAC,YACyB;EACzB,MAAM,EAAE,MAAM,GAAG,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,GACpB,CAAC,QAAQ,GAAG;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;GACvD,UAAU,OAAO,OAAO,SAAS;EAClC;EAED,UAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;EAAI,EAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,CAAC,iBAAiB,EAAE,QAAQ,0CAAE,CACc;EAIhD,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACJ,EAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM;AAGlB,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACD,EAAC;CACH;AACF;;;;;AAMD,IAAa,0BAAb,cAA6CC,wCAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;CACR;CAED;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK;kEACN,CAAC;CAEjE,eAAe,IAAI;CAEnB,OAAO,wBACLC,UACwB;AACxB,MAAI,aAAa,QAAW;GAC1B,MAAMC,kBAA8B;IAClC;IACA;IACA;IACA;GACD;AAED,UAAO,gBAAgB,OACrB,CAACC,aAAqC,iBAAiB;IACrD,YAAY,gBAAgB,mBAAmB;AAC/C,WAAO;GACR,GACD,CAAE,EACH;EACF;EAED,IAAIC,YAAuC,CAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAW;EACzD,WAEQ,oBAAoBC,0DAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR;GAKJ,YAAY,EAAE,GAAG,SAAU;EAC5B;AACD,SAAO;CACR;CAED,OAAO,sBAAsBC,QAA6B;EACxD,MAAM,UAAU,UAAUC;EAC1B,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;;;;;;;CAQD,aAAa,QACXC,KACAV,UACAW,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,IAAI,CAAC,CAAC,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC,CAC7B,KAAK,KAAK;EACb,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAa,EAAC;EAExD,MAAM,UAAU;AAChB,MAAI,SAEF,OAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACJ;CACF;CAED,eAAeC,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,oCAAW,UAAU,OAAOA,mCAC9B,OAAOA,oCAAW,OAAOA;AAE3B,SAAO;CACR;CAED,MAAM,qBACJC,MACAC,aACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAa,GAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;CACnC;AACF;;;;;;AAOD,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;CACR;CAED,oBAAoB;CAEpB,OAAO,sBAAsBX,QAA6B;EACxD,MAAM,UAAU,UAAUY;EAC1B,MAAMV,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,QAAQ;AAElD,MAAI,CAACC,mBAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,QAAQ,gBACR;AAGN,SAAO;CACR;AACF"}
+{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsDA,mDAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,OACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6CC,wCAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,OAQf,QAPoC;GAClC;GACA;GACA;GACA;GACD,CAEsB,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa,UACtB;OAAI,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,yDAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,mBAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,mCAAW,UAAU,OAAOA,iCAC9B,QAAOA,mCAAW,OAAOA;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,mBAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}

package/dist/evaluation/comparison/pairwise.d.cts

@@ -5,7 +5,7 @@ import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-import * as _langchain_core_prompt_values2 from "@langchain/core/prompt_values";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 
 //#region src/evaluation/comparison/pairwise.d.ts
@@ -30,7 +30,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values2.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +49,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values2.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.d.cts","names":["BaseLanguageModelInterface","BaseLLMOutputParser","ChainValues","ChatGeneration","Generation","BasePromptTemplate","Callbacks","BaseCallbackConfig","EvalOutputType","LLMEvalChainInput","LLMPairwiseStringEvaluator","LLMPairwiseStringEvaluatorArgs","ExtractLLMCallOptions","CriteriaLike","PairwiseStringResultOutputParser","Promise","PairwiseStringEvalChain","Record","_langchain_core_prompt_values2","BasePromptValueInterface","Omit","Partial","LabeledPairwiseStringEvalChain"],"sources":["../../../src/evaluation/comparison/pairwise.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { EvalOutputType, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, type ExtractLLMCallOptions } from \"../base.js\";\nimport { CriteriaLike } from \"../criteria/criteria.js\";\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport declare class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n    static lc_name(): string;\n    lc_namespace: string[];\n    parseResult(generations: Generation[] | ChatGeneration[], _callbacks: Callbacks | undefined): Promise<EvalOutputType>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n    static lc_name(): string;\n    criterionName?: string;\n    evaluationName?: string;\n    requiresInput: boolean;\n    requiresReference: boolean;\n    skipReferenceWarning: string;\n    outputParser: PairwiseStringResultOutputParser;\n    static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>;\n    /**\n     * Create a new instance of the PairwiseStringEvalChain.\n     * @param llm\n     * @param criteria The criteria to use for evaluation.\n     * @param chainOptions Options to pass to the chain.\n     */\n    static fromLLM(llm: BaseLanguageModelInterface, criteria?: CriteriaLike, chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>): Promise<PairwiseStringEvalChain>;\n    _prepareOutput(result: ChainValues): any;\n    _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n    static lc_name(): string;\n    requiresReference: boolean;\n    static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>;\n}\n//# sourceMappingURL=pairwise.d.ts.map"],"mappings":";;;;;;;;;;;;;;AAWqBc,cAAAA,gCAAAA,SAAyCb,mBAAT,CAA6BO,cAA7B,CAAA,CAAA;EAA6BA,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAGrDJ,YAAAA,EAAAA,MAAAA,EAAAA;EAAeD,WAAAA,CAAAA,WAAAA,EAAfC,UAAeD,EAAAA,GAAAA,cAAAA,EAAAA,EAAAA,UAAAA,EAA8BG,SAA9BH,GAAAA,SAAAA,CAAAA,EAAsDY,OAAtDZ,CAA8DK,cAA9DL,CAAAA;;;;;AAHqC;AAS5Da,cAAAA,uBAAAA,SAAgCN,0BAAAA,CAAT;EAO1BI,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAC4BD,aAAAA,CAAAA,EAAAA,MAAAA;EAAeI,cAAAA,CAAAA,EAAAA,MAAAA;EACnBZ,aAAAA,EAAAA,OAAAA;EAAkBa,iBAAAA,EAAAA,OAAAA;EAAGb,oBAAAA,EAAAA,MAAAA;EAOvCL,YAAAA,EATNc,gCASMd;EAAuCa,OAAAA,uBAAAA,CAAAA,QAAAA,CAAAA,EARjBA,YAQiBA,CAAAA,EARFI,MAQEJ,CAAAA,MAAAA,EAAAA,MAAAA,CAAAA;EAA0CJ,OAAAA,qBAAAA,CAAAA,MAAAA,CAAAA,EAP/DJ,kBAO+DI,CAAAA,EAP1CJ,kBAO0CI,CAAAA,GAAAA,EAP7CS,8BAAAA,CAAmEC,wBAAAA,EAOtBV,GAAAA,CAAAA;EAALW;;;;;;EAExBR,OAAAA,OAAAA,CAAAA,GAAAA,EAFpDZ,0BAEoDY,EAAAA,QAAAA,CAAAA,EAFbC,YAEaD,EAAAA,YAAAA,CAAAA,EAFgBS,OAEhBT,CAFwBQ,IAExBR,CAF6BH,iBAE7BG,EAAAA,KAAAA,CAAAA,CAAAA,CAAAA,EAF0DG,OAE1DH,CAFkEI,uBAElEJ,CAAAA;EAA6CN,cAAAA,CAAAA,MAAAA,EAD9FJ,WAC8FI,CAAAA,EAAAA,GAAAA;EAAYC,oBAAAA,CAAAA,IAAAA,EAAtGI,8BAAsGJ,EAAAA,WAAAA,EAAzDK,qBAAyDL,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAZD,SAAYC,GAAAA,kBAAAA,CAAAA,EAAqBQ,OAArBR,CAA6BL,WAA7BK,CAAAA;;;;AAlBtD;AAyB/E;;AAG4DW,cAHvCI,8BAAAA,SAAuCN,uBAAAA,CAGmEG;EAAhEd,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAHHW,iBAAAA,EAAAA,OAAAA;EAAuB,OAAA,qBAAA,CAAA,MAAA,CAAA,EAGzCX,kBAHyC,CAAA,EAGpBA,kBAHoB,CAAA,GAAA,EAGvBa,8BAAAA,CAAmEC,wBAAAA,EAH5C,GAAA,CAAA"}
+{"version":3,"file":"pairwise.d.cts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;AAAA"}

package/dist/evaluation/comparison/pairwise.d.ts

@@ -6,7 +6,7 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
-import * as _langchain_core_prompt_values4 from "@langchain/core/prompt_values";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 
 //#region src/evaluation/comparison/pairwise.d.ts
 /**
@@ -30,7 +30,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values4.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +49,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values4.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };