@langchain/classic 1.0.0-alpha.1

package/dist/evaluation/base.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.cts","names":["BaseLanguageModelInterface","AgentStep","ChainValues","BaseCallbackConfig","Callbacks","BaseChain","LLMChain","LLMChainInput","LLMEvalChainInput","EvalOutputType","T","L","ExtractLLMCallOptions","LanguageModelInterface","CallOptions","eqSet","Set","Record","LLMEvalChain","EvalChain","RunInput","RunOutput","StringEvaluatorArgs","PairwiseStringEvaluatorArgs","LLMPairwiseStringEvaluatorArgs","LLMTrajectoryEvaluatorArgs","LLMStringEvaluator","Promise","StringEvaluator","PairwiseStringEvaluator","LLMPairwiseStringEvaluator","AgentTrajectoryEvaluator"],"sources":["../../src/evaluation/base.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { BaseCallbackConfig, Callbacks } from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {\n}\nexport type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nany, infer CallOptions> ? CallOptions : never;\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport declare const eqSet: (xs: Set<string>, ys: Set<string>) => boolean;\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n/**\n * Base llm chain class for evaluators.\n */\nexport declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * Base chain class for evaluators.\n */\nexport declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n    prediction: string;\n    reference?: string;\n    input?: string;\n}\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n    prediction: string;\n    predictionB: string;\n}\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n    input: string;\n    prediction: string;\n    predictionB: string;\n    reference?: string;\n}\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n    input: string;\n    prediction: string;\n    reference?: string;\n    agentTrajectory: AgentStep[];\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class StringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class PairwiseStringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Interface for evaluating agent trajectories.\n */\nexport declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n    requiresInput: boolean;\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n"],"mappings":";;;;;;;;;;;AAQiBQ,UAAAA,iBAAiB,CAAA,UAAWC,cAAX,GAA4BA,cAA5B,EAAA,UAAsDT,0BAAtD,GAAmFA,0BAAnF,CAAA,SAAuHO,aAAvH,CAAqIG,CAArI,EAAwIC,CAAxI,CAAA,CAAA;AAAWF,KAEjCG,qBAFiCH,CAAAA,sBAAAA,CAAAA,GAEeI,sBAFfJ,SAE8CT,0BAF9CS;;GAA+B,EAAA,KAAYT,YAAAA,CAAAA,GAI9Dc,WAJ8Dd,GAAAA,KAAAA;;;;;AAA8E;AAEtK;;;;;AAEqC,KAWzBS,cAAAA,GAAiBQ,MAXQ,CAAA,MAAA,EAAA,MAAA,GAAA,MAAA,GAAA,OAAA,CAAA;AAWrC;AAIA;;AAAqDR,uBAAvBS,YAAuBT,CAAAA,UAAAA,cAAAA,GAAiBA,cAAjBA,EAAAA,UAA2CT,0BAA3CS,GAAwET,0BAAxES,CAAAA,SAA4GH,QAA5GG,CAAqHC,CAArHD,EAAwHE,CAAxHF,CAAAA,CAAAA;EAAc,aAAGA,CAAAA,EAAAA,OAAAA;EAAc,iBAAYT,CAAAA,EAAAA,OAAAA;EAA0B,gBAAGA,CAAAA,EAAAA,MAAAA;EAA0B,oBAAmBU,CAAAA,EAAAA,MAAAA;EAAC;;AAAF;AAgBzK;;;EAAoE,mBAAGR,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;AAAkEG,uBAA3Gc,SAA2Gd,CAAAA,iBAAhFH,WAAgFG,GAAlEH,WAAkEG,EAAAA,kBAAnCH,WAAmCG,GAArBH,WAAqBG,CAAAA,SAAAA,SAAAA,CAAUe,QAAVf,EAAoBgB,SAApBhB,CAAAA,CAAAA;EAAS,aAAA,CAAA,EAAA,OAAA;EAkBjIiB,iBAAAA,CAAAA,EAAAA,OAAmB;EASnBC,gBAAAA,CAAAA,EAAAA,MAAAA;EAUAC,oBAAAA,CAAAA,EAAAA,MAAAA;EAaAC;AAUjB;;;;;EAAgI,mBAAGzB,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;;;AAeaE,UAzD/HoB,mBAAAA,CAyD+HpB;EAAW,UAAnByB,EAAAA,MAAAA;EAAO,SAWrHL,CAAAA,EAAAA,MAAAA;EAAmB,KAAGV,CAAAA,EAAAA,MAAAA;;;;;;AA1BmI,UAjClKW,2BAAAA,CAiCkK;EAgCrJK,UAAAA,EAAAA,MAAe;EAAA,WAAA,EAAA,MAAA;;;;;;;;AAwBuBzB,UA/EnDqB,8BAAAA,CA+EmDrB;EAAkB,KAAWD,EAAAA,MAAAA;EAAW,UAAnByB,EAAAA,MAAAA;EAAO,WAxB1CR,EAAAA,MAAAA;EAAS,SAAA,CAAA,EAAA,MAAA;AA6B/D;;;;;;;;AAkBoEf,UAzFnDqB,0BAAAA,CAyFmDrB;EAAS,KAAGD,EAAAA,MAAAA;EAAkB,UAAWD,EAAAA,MAAAA;EAAW,SAAnByB,CAAAA,EAAAA,MAAAA;EAAO,eAlB9CR,EAnEzClB,SAmEyCkB,EAAAA;AAAS;AAuBvE;;;;AAYmIf,uBAhGrGsB,kBAgGqGtB,CAAAA,UAhGxEK,cAgGwEL,GAhGvDK,cAgGuDL,EAAAA,UAhG7BJ,0BAgG6BI,GAhGAJ,0BAgGAI,CAAAA,SAhGoCc,YAgGpCd,CAhGiDM,CAgGjDN,EAhGoDO,CAgGpDP,CAAAA,CAAAA;EAAS;;;EAA+B,cAQ7IoB,CAAAA,EAAAA,MAAAA;EAA8B;;;;;;AApBiB;AAyB7E;;;EAasE,SAAgBZ,gBAAAA,CAAAA,IAAAA,EA3GlDU,mBA2GkDV,GA3G5BA,qBA2G4BA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EA3GiBR,SA2GjBQ,GA3G6BT,kBA2G7BS,CAAAA,EA3GkDe,OA2GlDf,CA3G0DV,WA2G1DU,CAAAA;EAAqB;;;;;;;;;;EAQsD,eArBlGM,CAAAA,IAAAA,EAnFrCI,mBAmFqCJ,GAnFfN,qBAmFeM,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAnF8Bd,SAmF9Bc,GAnF0Cf,kBAmF1Ce,CAAAA,EAnF+DS,OAmF/DT,CAnFuEhB,WAmFvEgB,CAAAA;AAAY;;;;;uBA7E7CU,eAAAA,SAAwBT,SAAAA;;;;;;;;;;;;;;kCAclBG,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;;;wBAUjFoB,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKnE2B,uBAAAA,SAAgCV,SAAAA;;;;;;;;;;;sCAWtBI,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;4BAOzFqB,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAK/E4B,0BAAAA,SAAmCZ,YAAAA;;;;;;;;;;;;sCAYzBM,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;4BAQ9IsB,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKpI6B,wBAAAA,SAAiCb,YAAAA;;;;;;;;;;;;;0CAanBO,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;gCAQ1IuB,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB"}

package/dist/evaluation/base.d.ts

@@ -0,0 +1,240 @@
+import { BaseChain } from "../chains/base.js";
+import { LLMChain, LLMChainInput } from "../chains/llm_chain.js";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
+import { ChainValues } from "@langchain/core/utils/types";
+import { AgentStep } from "@langchain/core/agents";
+
+//#region src/evaluation/base.d.ts
+/**
+ * Base input for evaluators.
+ */
+interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {}
+type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+any, infer CallOptions> ? CallOptions : never;
+/**
+ * Compare two sets for equality
+ *
+ * @param xs
+ * @param ys
+ */
+
+/**
+ * The type of the output of an evaluation evaluator.
+ */
+type EvalOutputType = Record<string, string | number | boolean>;
+/**
+ * Base llm chain class for evaluators.
+ */
+declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {
+  requiresInput?: boolean;
+  requiresReference?: boolean;
+  skipInputWarning?: string;
+  skipReferenceWarning?: string;
+  /**
+   * Check if the evaluation arguments are valid.
+   * @param reference  The reference label.
+   * @param input The input string.
+   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+   */
+  checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * Base chain class for evaluators.
+ */
+declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {
+  requiresInput?: boolean;
+  requiresReference?: boolean;
+  skipInputWarning?: string;
+  skipReferenceWarning?: string;
+  /**
+   * Check if the evaluation arguments are valid.
+   * @param reference  The reference label.
+   * @param input The input string.
+   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+   */
+  checkEvaluationArgs(reference?: string, input?: string): void;
+}
+/**
+ * @field prediction The output string from the  model.
+ * @field reference The expected output / reference string.
+ * @field input The input string.
+ */
+interface StringEvaluatorArgs {
+  prediction: string;
+  reference?: string;
+  input?: string;
+}
+/**
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ */
+interface PairwiseStringEvaluatorArgs {
+  prediction: string;
+  predictionB: string;
+}
+/**
+ * @field The input string.
+ * @field prediction The output string from the first model.
+ * @field predictionB The output string from the second model.
+ * @field reference The expected output / reference string.
+ */
+interface LLMPairwiseStringEvaluatorArgs {
+  input: string;
+  prediction: string;
+  predictionB: string;
+  reference?: string;
+}
+/**
+ * Args for AgentTrajectoryEvaluator
+ * @field input The input to the agent.
+ * @field prediction The final predicted response.
+ * @field reference The reference answer.
+ * @field agentTrajectory  The intermediate steps forming the agent trajectory.
+ */
+interface LLMTrajectoryEvaluatorArgs {
+  input: string;
+  prediction: string;
+  reference?: string;
+  agentTrajectory: AgentStep[];
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Grade, tag, or otherwise evaluate predictions relative to their inputs
+ * and/or reference labels
+ */
+declare abstract class StringEvaluator extends EvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param config
+   */
+  abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate Chain or LLM output, based on optional input and label.
+   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+   * - score: the score of the evaluation, if applicable.
+   * - value: the string value of the evaluation, if applicable.
+   * - reasoning: the reasoning for the evaluation, if applicable.
+   * @param args
+   * @param config
+   */
+  evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+declare abstract class PairwiseStringEvaluator extends EvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Compare the output of two models (or two outputs of the same model).
+ */
+declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param callOptions
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate the output string pairs.
+   * @param args
+   * @param callOptions
+   * @param config
+   * @return A dictionary containing the preference, scores, and/or other information.
+   */
+  evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+/**
+ * Interface for evaluating agent trajectories.
+ */
+declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {
+  requiresInput: boolean;
+  /**
+   * The name of the evaluation.
+   */
+  evaluationName?: string;
+  /**
+   * Evaluate a trajectory.
+   * @return The evaluation result.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+  /**
+   * Evaluate a trajectory.
+   * @return The evaluation result.
+   * @param args
+   * @param callOptions
+   * @param config
+   */
+  evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+}
+//#endregion
+export { AgentTrajectoryEvaluator, EvalOutputType, ExtractLLMCallOptions, LLMEvalChainInput, LLMPairwiseStringEvaluator, LLMPairwiseStringEvaluatorArgs, LLMStringEvaluator, LLMTrajectoryEvaluatorArgs, PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs };
+//# sourceMappingURL=base.d.ts.map

package/dist/evaluation/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","names":["BaseLanguageModelInterface","AgentStep","ChainValues","BaseCallbackConfig","Callbacks","BaseChain","LLMChain","LLMChainInput","LLMEvalChainInput","EvalOutputType","T","L","ExtractLLMCallOptions","LanguageModelInterface","CallOptions","eqSet","Set","Record","LLMEvalChain","EvalChain","RunInput","RunOutput","StringEvaluatorArgs","PairwiseStringEvaluatorArgs","LLMPairwiseStringEvaluatorArgs","LLMTrajectoryEvaluatorArgs","LLMStringEvaluator","Promise","StringEvaluator","PairwiseStringEvaluator","LLMPairwiseStringEvaluator","AgentTrajectoryEvaluator"],"sources":["../../src/evaluation/base.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { BaseCallbackConfig, Callbacks } from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChainInput<T, L> {\n}\nexport type ExtractLLMCallOptions<LanguageModelInterface> = LanguageModelInterface extends BaseLanguageModelInterface<\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nany, infer CallOptions> ? CallOptions : never;\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport declare const eqSet: (xs: Set<string>, ys: Set<string>) => boolean;\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n/**\n * Base llm chain class for evaluators.\n */\nexport declare abstract class LLMEvalChain<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMChain<T, L> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * Base chain class for evaluators.\n */\nexport declare abstract class EvalChain<RunInput extends ChainValues = ChainValues, RunOutput extends ChainValues = ChainValues> extends BaseChain<RunInput, RunOutput> {\n    requiresInput?: boolean;\n    requiresReference?: boolean;\n    skipInputWarning?: string;\n    skipReferenceWarning?: string;\n    /**\n     * Check if the evaluation arguments are valid.\n     * @param reference  The reference label.\n     * @param input The input string.\n     * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n     */\n    checkEvaluationArgs(reference?: string, input?: string): void;\n}\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n    prediction: string;\n    reference?: string;\n    input?: string;\n}\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n    prediction: string;\n    predictionB: string;\n}\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n    input: string;\n    prediction: string;\n    predictionB: string;\n    reference?: string;\n}\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n    input: string;\n    prediction: string;\n    reference?: string;\n    agentTrajectory: AgentStep[];\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class LLMStringEvaluator<T extends EvalOutputType = EvalOutputType, L extends BaseLanguageModelInterface = BaseLanguageModelInterface> extends LLMEvalChain<T, L> {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport declare abstract class StringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    abstract _evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate Chain or LLM output, based on optional input and label.\n     * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n     * - score: the score of the evaluation, if applicable.\n     * - value: the string value of the evaluation, if applicable.\n     * - reasoning: the reasoning for the evaluation, if applicable.\n     * @param args\n     * @param config\n     */\n    evaluateStrings(args: StringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class PairwiseStringEvaluator extends EvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport declare abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    abstract _evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate the output string pairs.\n     * @param args\n     * @param callOptions\n     * @param config\n     * @return A dictionary containing the preference, scores, and/or other information.\n     */\n    evaluateStringPairs(args: LLMPairwiseStringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n/**\n * Interface for evaluating agent trajectories.\n */\nexport declare abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n    requiresInput: boolean;\n    /**\n     * The name of the evaluation.\n     */\n    evaluationName?: string;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    abstract _evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    /**\n     * Evaluate a trajectory.\n     * @return The evaluation result.\n     * @param args\n     * @param callOptions\n     * @param config\n     */\n    evaluateAgentTrajectory(args: LLMTrajectoryEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this[\"llm\"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n}\n"],"mappings":";;;;;;;;;;;AAQiBQ,UAAAA,iBAAiB,CAAA,UAAWC,cAAX,GAA4BA,cAA5B,EAAA,UAAsDT,0BAAtD,GAAmFA,0BAAnF,CAAA,SAAuHO,aAAvH,CAAqIG,CAArI,EAAwIC,CAAxI,CAAA,CAAA;AAAWF,KAEjCG,qBAFiCH,CAAAA,sBAAAA,CAAAA,GAEeI,sBAFfJ,SAE8CT,0BAF9CS;;GAA+B,EAAA,KAAYT,YAAAA,CAAAA,GAI9Dc,WAJ8Dd,GAAAA,KAAAA;;;;;AAA8E;AAEtK;;;;;AAEqC,KAWzBS,cAAAA,GAAiBQ,MAXQ,CAAA,MAAA,EAAA,MAAA,GAAA,MAAA,GAAA,OAAA,CAAA;AAWrC;AAIA;;AAAqDR,uBAAvBS,YAAuBT,CAAAA,UAAAA,cAAAA,GAAiBA,cAAjBA,EAAAA,UAA2CT,0BAA3CS,GAAwET,0BAAxES,CAAAA,SAA4GH,QAA5GG,CAAqHC,CAArHD,EAAwHE,CAAxHF,CAAAA,CAAAA;EAAc,aAAGA,CAAAA,EAAAA,OAAAA;EAAc,iBAAYT,CAAAA,EAAAA,OAAAA;EAA0B,gBAAGA,CAAAA,EAAAA,MAAAA;EAA0B,oBAAmBU,CAAAA,EAAAA,MAAAA;EAAC;;AAAF;AAgBzK;;;EAAoE,mBAAGR,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;AAAkEG,uBAA3Gc,SAA2Gd,CAAAA,iBAAhFH,WAAgFG,GAAlEH,WAAkEG,EAAAA,kBAAnCH,WAAmCG,GAArBH,WAAqBG,CAAAA,SAAAA,SAAAA,CAAUe,QAAVf,EAAoBgB,SAApBhB,CAAAA,CAAAA;EAAS,aAAA,CAAA,EAAA,OAAA;EAkBjIiB,iBAAAA,CAAAA,EAAAA,OAAmB;EASnBC,gBAAAA,CAAAA,EAAAA,MAAAA;EAUAC,oBAAAA,CAAAA,EAAAA,MAAAA;EAaAC;AAUjB;;;;;EAAgI,mBAAGzB,CAAAA,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA,KAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAAAA,IAAAA;;;;;;;AAeaE,UAzD/HoB,mBAAAA,CAyD+HpB;EAAW,UAAnByB,EAAAA,MAAAA;EAAO,SAWrHL,CAAAA,EAAAA,MAAAA;EAAmB,KAAGV,CAAAA,EAAAA,MAAAA;;;;;;AA1BmI,UAjClKW,2BAAAA,CAiCkK;EAgCrJK,UAAAA,EAAAA,MAAe;EAAA,WAAA,EAAA,MAAA;;;;;;;;AAwBuBzB,UA/EnDqB,8BAAAA,CA+EmDrB;EAAkB,KAAWD,EAAAA,MAAAA;EAAW,UAAnByB,EAAAA,MAAAA;EAAO,WAxB1CR,EAAAA,MAAAA;EAAS,SAAA,CAAA,EAAA,MAAA;AA6B/D;;;;;;;;AAkBoEf,UAzFnDqB,0BAAAA,CAyFmDrB;EAAS,KAAGD,EAAAA,MAAAA;EAAkB,UAAWD,EAAAA,MAAAA;EAAW,SAAnByB,CAAAA,EAAAA,MAAAA;EAAO,eAlB9CR,EAnEzClB,SAmEyCkB,EAAAA;AAAS;AAuBvE;;;;AAYmIf,uBAhGrGsB,kBAgGqGtB,CAAAA,UAhGxEK,cAgGwEL,GAhGvDK,cAgGuDL,EAAAA,UAhG7BJ,0BAgG6BI,GAhGAJ,0BAgGAI,CAAAA,SAhGoCc,YAgGpCd,CAhGiDM,CAgGjDN,EAhGoDO,CAgGpDP,CAAAA,CAAAA;EAAS;;;EAA+B,cAQ7IoB,CAAAA,EAAAA,MAAAA;EAA8B;;;;;;AApBiB;AAyB7E;;;EAasE,SAAgBZ,gBAAAA,CAAAA,IAAAA,EA3GlDU,mBA2GkDV,GA3G5BA,qBA2G4BA,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EA3GiBR,SA2GjBQ,GA3G6BT,kBA2G7BS,CAAAA,EA3GkDe,OA2GlDf,CA3G0DV,WA2G1DU,CAAAA;EAAqB;;;;;;;;;;EAQsD,eArBlGM,CAAAA,IAAAA,EAnFrCI,mBAmFqCJ,GAnFfN,qBAmFeM,CAAAA,IAAAA,CAAAA,KAAAA,CAAAA,CAAAA,EAAAA,MAAAA,CAAAA,EAnF8Bd,SAmF9Bc,GAnF0Cf,kBAmF1Ce,CAAAA,EAnF+DS,OAmF/DT,CAnFuEhB,WAmFvEgB,CAAAA;AAAY;;;;;uBA7E7CU,eAAAA,SAAwBT,SAAAA;;;;;;;;;;;;;;kCAclBG,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;;;wBAUjFoB,8BAA8BlB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKnE2B,uBAAAA,SAAgCV,SAAAA;;;;;;;;;;;sCAWtBI,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;;;4BAOzFqB,sCAAsCnB,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAK/E4B,0BAAAA,SAAmCZ,YAAAA;;;;;;;;;;;;sCAYzBM,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;4BAQ9IsB,8CAA8CZ,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;uBAKpI6B,wBAAAA,SAAiCb,YAAAA;;;;;;;;;;;;;0CAanBO,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB;;;;;;;;gCAQ1IuB,0CAA0Cb,6CAA6CR,YAAYD,qBAAqBwB,QAAQzB"}

package/dist/evaluation/base.js

@@ -0,0 +1,164 @@
+import { BaseChain } from "../chains/base.js";
+import { LLMChain } from "../chains/llm_chain.js";
+import "../chains/index.js";
+
+//#region src/evaluation/base.ts
+/**
+* Compare two sets for equality
+*
+* @param xs
+* @param ys
+*/
+const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+/**
+* Base llm chain class for evaluators.
+*/
+var LLMEvalChain = class extends LLMChain {
+	requiresInput = false;
+	requiresReference = false;
+	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
+	/**
+	* Check if the evaluation arguments are valid.
+	* @param reference  The reference label.
+	* @param input The input string.
+	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+	*/
+	checkEvaluationArgs(reference, input) {
+		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
+		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
+		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
+		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
+	}
+};
+/**
+* Base chain class for evaluators.
+*/
+var EvalChain = class extends BaseChain {
+	requiresInput = false;
+	requiresReference = false;
+	skipInputWarning = `Ignoring input in ${this.constructor.name}, as it is not expected.`;
+	skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;
+	/**
+	* Check if the evaluation arguments are valid.
+	* @param reference  The reference label.
+	* @param input The input string.
+	* @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.
+	*/
+	checkEvaluationArgs(reference, input) {
+		if (this.requiresInput && input == null) throw new Error(`${this.constructor.name} requires an input string.`);
+		else if (input != null && !this.requiresInput) console.warn(this.skipInputWarning);
+		if (this.requiresReference && reference == null) throw new Error(`${this.constructor.name} requires a reference string.`);
+		else if (reference != null && !this.requiresReference) console.warn(this.skipReferenceWarning);
+	}
+};
+/**
+* Grade, tag, or otherwise evaluate predictions relative to their inputs
+* and/or reference labels
+*/
+var LLMStringEvaluator = class extends LLMEvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate Chain or LLM output, based on optional input and label.
+	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+	* - score: the score of the evaluation, if applicable.
+	* - value: the string value of the evaluation, if applicable.
+	* - reasoning: the reasoning for the evaluation, if applicable.
+	* @param args
+	* @param callOptions
+	* @param config
+	*/
+	evaluateStrings(args, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStrings(args, config);
+	}
+};
+/**
+* Grade, tag, or otherwise evaluate predictions relative to their inputs
+* and/or reference labels
+*/
+var StringEvaluator = class extends EvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate Chain or LLM output, based on optional input and label.
+	* @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
+	* - score: the score of the evaluation, if applicable.
+	* - value: the string value of the evaluation, if applicable.
+	* - reasoning: the reasoning for the evaluation, if applicable.
+	* @param args
+	* @param config
+	*/
+	evaluateStrings(args, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStrings(args, config);
+	}
+};
+/**
+* Compare the output of two models (or two outputs of the same model).
+*/
+var PairwiseStringEvaluator = class extends EvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate the output string pairs.
+	* @param args
+	* @param config
+	* @return A dictionary containing the preference, scores, and/or other information.
+	*/
+	evaluateStringPairs(args, config) {
+		return this._evaluateStringPairs(args, config);
+	}
+};
+/**
+* Compare the output of two models (or two outputs of the same model).
+*/
+var LLMPairwiseStringEvaluator = class extends LLMEvalChain {
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate the output string pairs.
+	* @param args
+	* @param callOptions
+	* @param config
+	* @return A dictionary containing the preference, scores, and/or other information.
+	*/
+	evaluateStringPairs(args, callOptions, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateStringPairs(args, callOptions, config);
+	}
+};
+/**
+* Interface for evaluating agent trajectories.
+*/
+var AgentTrajectoryEvaluator = class extends LLMEvalChain {
+	requiresInput = true;
+	/**
+	* The name of the evaluation.
+	*/
+	evaluationName = this.constructor.name;
+	/**
+	* Evaluate a trajectory.
+	* @return The evaluation result.
+	* @param args
+	* @param callOptions
+	* @param config
+	*/
+	evaluateAgentTrajectory(args, callOptions, config) {
+		this.checkEvaluationArgs(args.reference, args.input);
+		return this._evaluateAgentTrajectory(args, callOptions, config);
+	}
+};
+
+//#endregion
+export { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator, eqSet };
+//# sourceMappingURL=base.js.map

package/dist/evaluation/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","names":["xs: Set<string>","ys: Set<string>","reference?: string","input?: string","args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>","config?: Callbacks | BaseCallbackConfig","args: StringEvaluatorArgs","args: PairwiseStringEvaluatorArgs","args: LLMPairwiseStringEvaluatorArgs","callOptions?: ExtractLLMCallOptions<this[\"llm\"]>","args: LLMTrajectoryEvaluatorArgs"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,QAAQ,CAACA,IAAiBC,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBC,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;AAKD,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,CAAC,kBAAkB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;CAEhG,uBAAgC,CAAC,sBAAsB,EAAE,KAAK,YAAY,KAAK,wBAAwB,CAAC;;;;;;;CAQxG,oBAAoBD,WAAoBC,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,0BAA0B,CAAC;WAC3D,SAAS,QAAQ,CAAC,KAAK,eAChC,QAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,6BAA6B,CAAC;WAC9D,aAAa,QAAQ,CAAC,KAAK,mBACpC,QAAQ,KAAK,KAAK,qBAAqB;CAE1C;AACF;;;;;AAqDD,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACEC,MACAC,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;;AAMD,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACEC,MACAD,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;CAC3C;AACF;;;;AAKD,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACEE,MACAF,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;CAC/C;AACF;;;;AAKD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACEG,MACAC,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;CAC5D;AACF;;;;AAKD,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACEK,MACAD,aACAJ,QACsB;EACtB,KAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO;CAChE;AACF"}

package/dist/evaluation/comparison/index.cjs

@@ -0,0 +1 @@
+const require_pairwise = require('./pairwise.cjs');

package/dist/evaluation/comparison/index.js

@@ -0,0 +1 @@
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.js";