@langchain/classic 1.0.0-alpha.1

package/dist/evaluation/index.cjs

@@ -0,0 +1,54 @@
+const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_eval_chain = require('./qa/eval_chain.cjs');
+require('./qa/index.cjs');
+const require_criteria = require('./criteria/criteria.cjs');
+require('./criteria/index.cjs');
+const require_trajectory = require('./agents/trajectory.cjs');
+require('./agents/index.cjs');
+const require_base = require('./embedding_distance/base.cjs');
+require('./embedding_distance/index.cjs');
+const require_pairwise = require('./comparison/pairwise.cjs');
+require('./comparison/index.cjs');
+const require_loader = require('./loader.cjs');
+
+//#region src/evaluation/index.ts
+var evaluation_exports = {};
+require_rolldown_runtime.__export(evaluation_exports, {
+	CriteriaEvalChain: () => require_criteria.CriteriaEvalChain,
+	CriteriaResultOutputParser: () => require_criteria.CriteriaResultOutputParser,
+	EmbeddingDistanceEvalChain: () => require_base.EmbeddingDistanceEvalChain,
+	LabeledCriteriaEvalChain: () => require_criteria.LabeledCriteriaEvalChain,
+	LabeledPairwiseStringEvalChain: () => require_pairwise.LabeledPairwiseStringEvalChain,
+	PairwiseEmbeddingDistanceEvalChain: () => require_base.PairwiseEmbeddingDistanceEvalChain,
+	PairwiseStringEvalChain: () => require_pairwise.PairwiseStringEvalChain,
+	PairwiseStringResultOutputParser: () => require_pairwise.PairwiseStringResultOutputParser,
+	QAEvalChain: () => require_eval_chain.QAEvalChain,
+	TrajectoryEvalChain: () => require_trajectory.TrajectoryEvalChain,
+	TrajectoryOutputParser: () => require_trajectory.TrajectoryOutputParser,
+	computeEvaluationScore: () => require_base.computeEvaluationScore,
+	getDistanceCalculationFunction: () => require_base.getDistanceCalculationFunction,
+	loadEvaluator: () => require_loader.loadEvaluator
+});
+
+//#endregion
+exports.CriteriaEvalChain = require_criteria.CriteriaEvalChain;
+exports.CriteriaResultOutputParser = require_criteria.CriteriaResultOutputParser;
+exports.EmbeddingDistanceEvalChain = require_base.EmbeddingDistanceEvalChain;
+exports.LabeledCriteriaEvalChain = require_criteria.LabeledCriteriaEvalChain;
+exports.LabeledPairwiseStringEvalChain = require_pairwise.LabeledPairwiseStringEvalChain;
+exports.PairwiseEmbeddingDistanceEvalChain = require_base.PairwiseEmbeddingDistanceEvalChain;
+exports.PairwiseStringEvalChain = require_pairwise.PairwiseStringEvalChain;
+exports.PairwiseStringResultOutputParser = require_pairwise.PairwiseStringResultOutputParser;
+exports.QAEvalChain = require_eval_chain.QAEvalChain;
+exports.TrajectoryEvalChain = require_trajectory.TrajectoryEvalChain;
+exports.TrajectoryOutputParser = require_trajectory.TrajectoryOutputParser;
+exports.computeEvaluationScore = require_base.computeEvaluationScore;
+Object.defineProperty(exports, 'evaluation_exports', {
+  enumerable: true,
+  get: function () {
+    return evaluation_exports;
+  }
+});
+exports.getDistanceCalculationFunction = require_base.getDistanceCalculationFunction;
+exports.loadEvaluator = require_loader.loadEvaluator;
+//# sourceMappingURL=index.cjs.map

package/dist/evaluation/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs","names":[],"sources":["../../src/evaluation/index.ts"],"sourcesContent":["export * from \"./qa/index.js\";\nexport * from \"./criteria/index.js\";\nexport * from \"./agents/index.js\";\nexport * from \"./embedding_distance/index.js\";\nexport * from \"./comparison/index.js\";\nexport * from \"./types.js\";\nexport * from \"./loader.js\";\n"],"mappings":""}

package/dist/evaluation/index.d.cts

@@ -0,0 +1,8 @@
+import { QAEvalChain } from "./qa/eval_chain.cjs";
+import { Criteria, CriteriaEvalChain, CriteriaEvalInput, CriteriaLike, CriteriaResultOutputParser, LabeledCriteriaEvalChain } from "./criteria/criteria.cjs";
+import { TrajectoryEvalChain, TrajectoryOutputParser } from "./agents/trajectory.cjs";
+import { EmbeddingDistanceEvalChain, EmbeddingDistanceEvalChainInput, EmbeddingDistanceType, PairwiseEmbeddingDistanceEvalChain, computeEvaluationScore, getDistanceCalculationFunction } from "./embedding_distance/base.cjs";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./comparison/pairwise.cjs";
+import { EvaluatorType } from "./types.cjs";
+import { LoadEvaluatorOptions, loadEvaluator } from "./loader.cjs";
+export { Criteria, CriteriaEvalChain, CriteriaEvalInput, CriteriaLike, CriteriaResultOutputParser, EmbeddingDistanceEvalChain, EmbeddingDistanceEvalChainInput, EmbeddingDistanceType, EvaluatorType, LabeledCriteriaEvalChain, LabeledPairwiseStringEvalChain, LoadEvaluatorOptions, PairwiseEmbeddingDistanceEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser, QAEvalChain, TrajectoryEvalChain, TrajectoryOutputParser, computeEvaluationScore, getDistanceCalculationFunction, loadEvaluator };

package/dist/evaluation/index.d.ts

@@ -0,0 +1,8 @@
+import { QAEvalChain } from "./qa/eval_chain.js";
+import { Criteria, CriteriaEvalChain, CriteriaEvalInput, CriteriaLike, CriteriaResultOutputParser, LabeledCriteriaEvalChain } from "./criteria/criteria.js";
+import { TrajectoryEvalChain, TrajectoryOutputParser } from "./agents/trajectory.js";
+import { EmbeddingDistanceEvalChain, EmbeddingDistanceEvalChainInput, EmbeddingDistanceType, PairwiseEmbeddingDistanceEvalChain, computeEvaluationScore, getDistanceCalculationFunction } from "./embedding_distance/base.js";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./comparison/pairwise.js";
+import { EvaluatorType } from "./types.js";
+import { LoadEvaluatorOptions, loadEvaluator } from "./loader.js";
+export { Criteria, CriteriaEvalChain, CriteriaEvalInput, CriteriaLike, CriteriaResultOutputParser, EmbeddingDistanceEvalChain, EmbeddingDistanceEvalChainInput, EmbeddingDistanceType, EvaluatorType, LabeledCriteriaEvalChain, LabeledPairwiseStringEvalChain, LoadEvaluatorOptions, PairwiseEmbeddingDistanceEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser, QAEvalChain, TrajectoryEvalChain, TrajectoryOutputParser, computeEvaluationScore, getDistanceCalculationFunction, loadEvaluator };

package/dist/evaluation/index.js

@@ -0,0 +1,35 @@
+import { __export } from "../_virtual/rolldown_runtime.js";
+import { QAEvalChain } from "./qa/eval_chain.js";
+import "./qa/index.js";
+import { CriteriaEvalChain, CriteriaResultOutputParser, LabeledCriteriaEvalChain } from "./criteria/criteria.js";
+import "./criteria/index.js";
+import { TrajectoryEvalChain, TrajectoryOutputParser } from "./agents/trajectory.js";
+import "./agents/index.js";
+import { EmbeddingDistanceEvalChain, PairwiseEmbeddingDistanceEvalChain, computeEvaluationScore, getDistanceCalculationFunction } from "./embedding_distance/base.js";
+import "./embedding_distance/index.js";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./comparison/pairwise.js";
+import "./comparison/index.js";
+import { loadEvaluator } from "./loader.js";
+
+//#region src/evaluation/index.ts
+var evaluation_exports = {};
+__export(evaluation_exports, {
+	CriteriaEvalChain: () => CriteriaEvalChain,
+	CriteriaResultOutputParser: () => CriteriaResultOutputParser,
+	EmbeddingDistanceEvalChain: () => EmbeddingDistanceEvalChain,
+	LabeledCriteriaEvalChain: () => LabeledCriteriaEvalChain,
+	LabeledPairwiseStringEvalChain: () => LabeledPairwiseStringEvalChain,
+	PairwiseEmbeddingDistanceEvalChain: () => PairwiseEmbeddingDistanceEvalChain,
+	PairwiseStringEvalChain: () => PairwiseStringEvalChain,
+	PairwiseStringResultOutputParser: () => PairwiseStringResultOutputParser,
+	QAEvalChain: () => QAEvalChain,
+	TrajectoryEvalChain: () => TrajectoryEvalChain,
+	TrajectoryOutputParser: () => TrajectoryOutputParser,
+	computeEvaluationScore: () => computeEvaluationScore,
+	getDistanceCalculationFunction: () => getDistanceCalculationFunction,
+	loadEvaluator: () => loadEvaluator
+});
+
+//#endregion
+export { CriteriaEvalChain, CriteriaResultOutputParser, EmbeddingDistanceEvalChain, LabeledCriteriaEvalChain, LabeledPairwiseStringEvalChain, PairwiseEmbeddingDistanceEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser, QAEvalChain, TrajectoryEvalChain, TrajectoryOutputParser, computeEvaluationScore, evaluation_exports, getDistanceCalculationFunction, loadEvaluator };
+//# sourceMappingURL=index.js.map

package/dist/evaluation/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../../src/evaluation/index.ts"],"sourcesContent":["export * from \"./qa/index.js\";\nexport * from \"./criteria/index.js\";\nexport * from \"./agents/index.js\";\nexport * from \"./embedding_distance/index.js\";\nexport * from \"./comparison/index.js\";\nexport * from \"./types.js\";\nexport * from \"./loader.js\";\n"],"mappings":""}

package/dist/evaluation/loader.cjs

@@ -0,0 +1,57 @@
+const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_criteria = require('./criteria/criteria.cjs');
+require('./criteria/index.cjs');
+const require_trajectory = require('./agents/trajectory.cjs');
+require('./agents/index.cjs');
+const require_base = require('./embedding_distance/base.cjs');
+require('./embedding_distance/index.cjs');
+const require_pairwise = require('./comparison/pairwise.cjs');
+require('./comparison/index.cjs');
+const __langchain_core_language_models_chat_models = require_rolldown_runtime.__toESM(require("@langchain/core/language_models/chat_models"));
+
+//#region src/evaluation/loader.ts
+/**
+* Load the requested evaluation chain specified by a string
+* @param type The type of evaluator to load.
+* @param options
+*        - llm The language model to use for the evaluator.
+*        - criteria The criteria to use for the evaluator.
+*        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.
+*/
+async function loadEvaluator(type, options) {
+	const { llm, chainOptions, criteria, agentTools } = options;
+	let evaluator;
+	switch (type) {
+		case "criteria":
+			evaluator = await require_criteria.CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "labeled_criteria":
+			evaluator = await require_criteria.LabeledCriteriaEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "pairwise_string":
+			evaluator = await require_pairwise.PairwiseStringEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "labeled_pairwise_string":
+			evaluator = await require_pairwise.LabeledPairwiseStringEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "trajectory":
+			if (!(llm instanceof __langchain_core_language_models_chat_models.BaseChatModel)) throw new Error("LLM must be an instance of a base chat model.");
+			evaluator = await require_trajectory.TrajectoryEvalChain.fromLLM(llm, agentTools, chainOptions);
+			break;
+		case "embedding_distance":
+			evaluator = new require_base.EmbeddingDistanceEvalChain({
+				embedding: options?.embedding,
+				distanceMetric: options?.distanceMetric
+			});
+			break;
+		case "pairwise_embedding_distance":
+			evaluator = new require_base.PairwiseEmbeddingDistanceEvalChain({});
+			break;
+		default: throw new Error(`Unknown type: ${type}`);
+	}
+	return evaluator;
+}
+
+//#endregion
+exports.loadEvaluator = loadEvaluator;
+//# sourceMappingURL=loader.cjs.map

package/dist/evaluation/loader.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.cjs","names":["type: T","options: LoadEvaluatorOptions","evaluator: unknown","CriteriaEvalChain","LabeledCriteriaEvalChain","PairwiseStringEvalChain","LabeledPairwiseStringEvalChain","BaseChatModel","TrajectoryEvalChain","EmbeddingDistanceEvalChain","PairwiseEmbeddingDistanceEvalChain"],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpBA,MACAC,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,YAAY,GAAG;CAEpD,IAAIC;AACJ,SAAQ,MAAR;EACE,KAAK;GACH,YAAY,MAAMC,mCAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;GACH,YAAY,MAAMC,0CAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAMC,yCAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAMC,gDAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAeC,4DACnB,OAAM,IAAI,MAAM;GAElB,YAAY,MAAMC,uCAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;GACH,YAAY,IAAIC,wCAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;GAC1B;AACD;EACF,KAAK;GACH,YAAY,IAAIC,gDAAmC,CAAE;AACrD;EACF,QACE,OAAM,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM;CAC1C;AAED,QAAO;AACR"}

package/dist/evaluation/loader.d.cts

@@ -0,0 +1,38 @@
+import { LLMEvalChainInput } from "./base.cjs";
+import { CriteriaLike } from "./criteria/criteria.cjs";
+import { EmbeddingDistanceEvalChainInput } from "./embedding_distance/base.cjs";
+import { EvaluatorType } from "./types.cjs";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { StructuredToolInterface } from "@langchain/core/tools";
+
+//#region src/evaluation/loader.d.ts
+type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {
+  /**
+   * The language model to use for the evaluator.
+   */
+  llm: BaseLanguageModelInterface;
+  /**
+   * The options to pass to the evaluator chain.
+   */
+  chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>;
+  /**
+   * The criteria to use for the evaluator.
+   */
+  criteria?: CriteriaLike;
+  /**
+   * A list of tools available to the agent, for TrajectoryEvalChain.
+   */
+  agentTools?: StructuredToolInterface[];
+};
+/**
+ * Load the requested evaluation chain specified by a string
+ * @param type The type of evaluator to load.
+ * @param options
+ *        - llm The language model to use for the evaluator.
+ *        - criteria The criteria to use for the evaluator.
+ *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.
+ */
+declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;
+//#endregion
+export { LoadEvaluatorOptions, loadEvaluator };
+//# sourceMappingURL=loader.d.cts.map

package/dist/evaluation/loader.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.cts","names":["BaseLanguageModelInterface","StructuredToolInterface","CriteriaLike","EvaluatorType","LLMEvalChainInput","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","Omit","Partial","loadEvaluator","T","Promise"],"sources":["../../src/evaluation/loader.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { CriteriaLike } from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport { EmbeddingDistanceEvalChainInput } from \"./embedding_distance/index.js\";\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n    /**\n     * The language model to use for the evaluator.\n     */\n    llm: BaseLanguageModelInterface;\n    /**\n     * The options to pass to the evaluator chain.\n     */\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n    /**\n     * The criteria to use for the evaluator.\n     */\n    criteria?: CriteriaLike;\n    /**\n     * A list of tools available to the agent, for TrajectoryEvalChain.\n     */\n    agentTools?: StructuredToolInterface[];\n};\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;\n"],"mappings":";;;;;;;;KAMYM,oBAAAA,GAAuBD;;AAAnC;;EAAgC,GAAGA,EAI1BL,0BAJ0BK;EAA+B;;;EAQnC,YAAZG,CAAAA,EAAAA,OAAAA,CAAQD,IAARC,CAAaJ,iBAAbI,EAAAA,KAAAA,CAAAA,CAAAA;EAAO;;AAQc;EAUhBC,QAAAA,CAAAA,EAdTP,YAcsB;EAAA;;;EAAuC,UAAWI,CAAAA,EAVtEL,uBAUsEK,EAAAA;CAAoB;;;AAAU;;;;;;iBAA7FG,8BAA8BN,qBAAqBO,YAAYJ,uBAAuBK,QAAQR,cAAcO"}

package/dist/evaluation/loader.d.ts

@@ -0,0 +1,38 @@
+import { LLMEvalChainInput } from "./base.js";
+import { CriteriaLike } from "./criteria/criteria.js";
+import { EmbeddingDistanceEvalChainInput } from "./embedding_distance/base.js";
+import { EvaluatorType } from "./types.js";
+import { StructuredToolInterface } from "@langchain/core/tools";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+
+//#region src/evaluation/loader.d.ts
+type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {
+  /**
+   * The language model to use for the evaluator.
+   */
+  llm: BaseLanguageModelInterface;
+  /**
+   * The options to pass to the evaluator chain.
+   */
+  chainOptions?: Partial<Omit<LLMEvalChainInput, "llm">>;
+  /**
+   * The criteria to use for the evaluator.
+   */
+  criteria?: CriteriaLike;
+  /**
+   * A list of tools available to the agent, for TrajectoryEvalChain.
+   */
+  agentTools?: StructuredToolInterface[];
+};
+/**
+ * Load the requested evaluation chain specified by a string
+ * @param type The type of evaluator to load.
+ * @param options
+ *        - llm The language model to use for the evaluator.
+ *        - criteria The criteria to use for the evaluator.
+ *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.
+ */
+declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;
+//#endregion
+export { LoadEvaluatorOptions, loadEvaluator };
+//# sourceMappingURL=loader.d.ts.map

package/dist/evaluation/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","names":["BaseLanguageModelInterface","StructuredToolInterface","CriteriaLike","EvaluatorType","LLMEvalChainInput","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","Omit","Partial","loadEvaluator","T","Promise"],"sources":["../../src/evaluation/loader.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { CriteriaLike } from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport { EmbeddingDistanceEvalChainInput } from \"./embedding_distance/index.js\";\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n    /**\n     * The language model to use for the evaluator.\n     */\n    llm: BaseLanguageModelInterface;\n    /**\n     * The options to pass to the evaluator chain.\n     */\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n    /**\n     * The criteria to use for the evaluator.\n     */\n    criteria?: CriteriaLike;\n    /**\n     * A list of tools available to the agent, for TrajectoryEvalChain.\n     */\n    agentTools?: StructuredToolInterface[];\n};\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;\n"],"mappings":";;;;;;;;KAMYM,oBAAAA,GAAuBD;;AAAnC;;EAAgC,GAAGA,EAI1BL,0BAJ0BK;EAA+B;;;EAQnC,YAAZG,CAAAA,EAAAA,OAAAA,CAAQD,IAARC,CAAaJ,iBAAbI,EAAAA,KAAAA,CAAAA,CAAAA;EAAO;;AAQc;EAUhBC,QAAAA,CAAAA,EAdTP,YAcsB;EAAA;;;EAAuC,UAAWI,CAAAA,EAVtEL,uBAUsEK,EAAAA;CAAoB;;;AAAU;;;;;;iBAA7FG,8BAA8BN,qBAAqBO,YAAYJ,uBAAuBK,QAAQR,cAAcO"}

package/dist/evaluation/loader.js

@@ -0,0 +1,56 @@
+import { CriteriaEvalChain, LabeledCriteriaEvalChain } from "./criteria/criteria.js";
+import "./criteria/index.js";
+import { TrajectoryEvalChain } from "./agents/trajectory.js";
+import "./agents/index.js";
+import { EmbeddingDistanceEvalChain, PairwiseEmbeddingDistanceEvalChain } from "./embedding_distance/base.js";
+import "./embedding_distance/index.js";
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain } from "./comparison/pairwise.js";
+import "./comparison/index.js";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+
+//#region src/evaluation/loader.ts
+/**
+* Load the requested evaluation chain specified by a string
+* @param type The type of evaluator to load.
+* @param options
+*        - llm The language model to use for the evaluator.
+*        - criteria The criteria to use for the evaluator.
+*        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.
+*/
+async function loadEvaluator(type, options) {
+	const { llm, chainOptions, criteria, agentTools } = options;
+	let evaluator;
+	switch (type) {
+		case "criteria":
+			evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "labeled_criteria":
+			evaluator = await LabeledCriteriaEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "pairwise_string":
+			evaluator = await PairwiseStringEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "labeled_pairwise_string":
+			evaluator = await LabeledPairwiseStringEvalChain.fromLLM(llm, criteria, chainOptions);
+			break;
+		case "trajectory":
+			if (!(llm instanceof BaseChatModel)) throw new Error("LLM must be an instance of a base chat model.");
+			evaluator = await TrajectoryEvalChain.fromLLM(llm, agentTools, chainOptions);
+			break;
+		case "embedding_distance":
+			evaluator = new EmbeddingDistanceEvalChain({
+				embedding: options?.embedding,
+				distanceMetric: options?.distanceMetric
+			});
+			break;
+		case "pairwise_embedding_distance":
+			evaluator = new PairwiseEmbeddingDistanceEvalChain({});
+			break;
+		default: throw new Error(`Unknown type: ${type}`);
+	}
+	return evaluator;
+}
+
+//#endregion
+export { loadEvaluator };
+//# sourceMappingURL=loader.js.map

package/dist/evaluation/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","names":["type: T","options: LoadEvaluatorOptions","evaluator: unknown"],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpBA,MACAC,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,YAAY,GAAG;CAEpD,IAAIC;AACJ,SAAQ,MAAR;EACE,KAAK;GACH,YAAY,MAAM,kBAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;GACH,YAAY,MAAM,yBAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAM,wBAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAM,+BAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAe,eACnB,OAAM,IAAI,MAAM;GAElB,YAAY,MAAM,oBAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;GACH,YAAY,IAAI,2BAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;GAC1B;AACD;EACF,KAAK;GACH,YAAY,IAAI,mCAAmC,CAAE;AACrD;EACF,QACE,OAAM,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM;CAC1C;AAED,QAAO;AACR"}

package/dist/evaluation/qa/eval_chain.cjs

@@ -0,0 +1,41 @@
+const require_llm_chain = require('../../chains/llm_chain.cjs');
+const require_prompt = require('./prompt.cjs');
+
+//#region src/evaluation/qa/eval_chain.ts
+const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+var QAEvalChain = class extends require_llm_chain.LLMChain {
+	static lc_name() {
+		return "QAEvalChain";
+	}
+	static fromLlm(llm, options = {}) {
+		const prompt = options.prompt || require_prompt.QA_PROMPT;
+		const expectedInputVars = new Set([
+			"query",
+			"answer",
+			"result"
+		]);
+		const inputVarsSet = new Set(prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
+		return new this({
+			llm,
+			prompt,
+			...options.chainInput
+		});
+	}
+	async evaluate(examples, predictions, args = {
+		questionKey: "query",
+		answerKey: "answer",
+		predictionKey: "result"
+	}) {
+		const inputs = examples.map((example, i) => ({
+			query: example[args.questionKey],
+			answer: example[args.answerKey],
+			result: predictions[i][args.predictionKey]
+		}));
+		return await this.apply(inputs);
+	}
+};
+
+//#endregion
+exports.QAEvalChain = QAEvalChain;
+//# sourceMappingURL=eval_chain.cjs.map

package/dist/evaluation/qa/eval_chain.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_chain.cjs","names":["xs: Set<string>","ys: Set<string>","LLMChain","llm: BaseLanguageModelInterface","options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    }","QA_PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","examples: ChainValues","predictions: ChainValues","args: EvaluateArgs","example: ChainValues","i: number"],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,QAAQ,CAACA,IAAiBC,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiCC,2BAAS;CACxC,OAAO,UAAU;AACf,SAAO;CACR;CAED,OAAO,QACLC,KACAC,UAGI,CAAE,GACO;EACb,MAAM,SAAS,QAAQ,UAAUC;EACjC,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,OAAO;AAEjD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,OAAO,gBACP;AAGN,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;EAAY;CACvD;CAED,MAAM,SACJC,UACAC,aACAC,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;CAChB,GACqB;EACtB,MAAM,SAAS,SAAS,IAAI,CAACC,SAAsBC,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;EAC7B,GAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO;CAChC;AACF"}

package/dist/evaluation/qa/eval_chain.d.cts

@@ -0,0 +1,22 @@
+import { LLMChain, LLMChainInput } from "../../chains/llm_chain.cjs";
+import { ChainValues } from "@langchain/core/utils/types";
+import { PromptTemplate } from "@langchain/core/prompts";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+
+//#region src/evaluation/qa/eval_chain.d.ts
+interface EvaluateArgs {
+  questionKey: string;
+  answerKey: string;
+  predictionKey: string;
+}
+declare class QAEvalChain extends LLMChain {
+  static lc_name(): string;
+  static fromLlm(llm: BaseLanguageModelInterface, options?: {
+    prompt?: PromptTemplate;
+    chainInput?: Omit<LLMChainInput, "llm">;
+  }): QAEvalChain;
+  evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;
+}
+//#endregion
+export { QAEvalChain };
+//# sourceMappingURL=eval_chain.d.cts.map

package/dist/evaluation/qa/eval_chain.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_chain.d.cts","names":["BaseLanguageModelInterface","ChainValues","PromptTemplate","LLMChain","LLMChainInput","EvaluateArgs","QAEvalChain","Omit","Promise"],"sources":["../../../src/evaluation/qa/eval_chain.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\nexport interface EvaluateArgs {\n    questionKey: string;\n    answerKey: string;\n    predictionKey: string;\n}\nexport declare class QAEvalChain extends LLMChain {\n    static lc_name(): string;\n    static fromLlm(llm: BaseLanguageModelInterface, options?: {\n        prompt?: PromptTemplate;\n        chainInput?: Omit<LLMChainInput, \"llm\">;\n    }): QAEvalChain;\n    evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;\n}\n"],"mappings":";;;;;;UAIiBK,YAAAA;;EAAAA,SAAAA,EAAAA,MAAY;EAKRC,aAAAA,EAAW,MAAA;;AAERN,cAFHM,WAAAA,SAAoBH,QAAAA,CAEjBH;EAA0B,OACjCE,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAc,OACLE,OAAAA,CAAAA,GAAAA,EAFFJ,0BAEEI,EAAAA,OACP,CADOA,EAAAA;IAALG,MAAAA,CAAAA,EADJL,cACIK;IACbD,UAAAA,CAAAA,EADaC,IACbD,CADkBF,aAClBE,EAAAA,KAAAA,CAAAA;EAAW,CAAA,CAAA,EAAXA,WACeL;EAAW,QAAeA,CAAAA,QAAAA,EAA1BA,WAA0BA,EAAAA,WAAAA,EAAAA,WAAAA,EAAAA,IAAAA,CAAAA,EAAoBI,YAApBJ,CAAAA,EAAmCO,OAAnCP,CAA2CA,WAA3CA,CAAAA"}

package/dist/evaluation/qa/eval_chain.d.ts

@@ -0,0 +1,22 @@
+import { LLMChain, LLMChainInput } from "../../chains/llm_chain.js";
+import { PromptTemplate } from "@langchain/core/prompts";
+import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
+import { ChainValues } from "@langchain/core/utils/types";
+
+//#region src/evaluation/qa/eval_chain.d.ts
+interface EvaluateArgs {
+  questionKey: string;
+  answerKey: string;
+  predictionKey: string;
+}
+declare class QAEvalChain extends LLMChain {
+  static lc_name(): string;
+  static fromLlm(llm: BaseLanguageModelInterface, options?: {
+    prompt?: PromptTemplate;
+    chainInput?: Omit<LLMChainInput, "llm">;
+  }): QAEvalChain;
+  evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;
+}
+//#endregion
+export { QAEvalChain };
+//# sourceMappingURL=eval_chain.d.ts.map

package/dist/evaluation/qa/eval_chain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_chain.d.ts","names":["BaseLanguageModelInterface","ChainValues","PromptTemplate","LLMChain","LLMChainInput","EvaluateArgs","QAEvalChain","Omit","Promise"],"sources":["../../../src/evaluation/qa/eval_chain.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\nexport interface EvaluateArgs {\n    questionKey: string;\n    answerKey: string;\n    predictionKey: string;\n}\nexport declare class QAEvalChain extends LLMChain {\n    static lc_name(): string;\n    static fromLlm(llm: BaseLanguageModelInterface, options?: {\n        prompt?: PromptTemplate;\n        chainInput?: Omit<LLMChainInput, \"llm\">;\n    }): QAEvalChain;\n    evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;\n}\n"],"mappings":";;;;;;UAIiBK,YAAAA;;EAAAA,SAAAA,EAAAA,MAAY;EAKRC,aAAAA,EAAW,MAAA;;AAERN,cAFHM,WAAAA,SAAoBH,QAAAA,CAEjBH;EAA0B,OACjCE,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAAc,OACLE,OAAAA,CAAAA,GAAAA,EAFFJ,0BAEEI,EAAAA,OACP,CADOA,EAAAA;IAALG,MAAAA,CAAAA,EADJL,cACIK;IACbD,UAAAA,CAAAA,EADaC,IACbD,CADkBF,aAClBE,EAAAA,KAAAA,CAAAA;EAAW,CAAA,CAAA,EAAXA,WACeL;EAAW,QAAeA,CAAAA,QAAAA,EAA1BA,WAA0BA,EAAAA,WAAAA,EAAAA,WAAAA,EAAAA,IAAAA,CAAAA,EAAoBI,YAApBJ,CAAAA,EAAmCO,OAAnCP,CAA2CA,WAA3CA,CAAAA"}

package/dist/evaluation/qa/eval_chain.js

@@ -0,0 +1,41 @@
+import { LLMChain } from "../../chains/llm_chain.js";
+import { QA_PROMPT } from "./prompt.js";
+
+//#region src/evaluation/qa/eval_chain.ts
+const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
+var QAEvalChain = class extends LLMChain {
+	static lc_name() {
+		return "QAEvalChain";
+	}
+	static fromLlm(llm, options = {}) {
+		const prompt = options.prompt || QA_PROMPT;
+		const expectedInputVars = new Set([
+			"query",
+			"answer",
+			"result"
+		]);
+		const inputVarsSet = new Set(prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
+		return new this({
+			llm,
+			prompt,
+			...options.chainInput
+		});
+	}
+	async evaluate(examples, predictions, args = {
+		questionKey: "query",
+		answerKey: "answer",
+		predictionKey: "result"
+	}) {
+		const inputs = examples.map((example, i) => ({
+			query: example[args.questionKey],
+			answer: example[args.answerKey],
+			result: predictions[i][args.predictionKey]
+		}));
+		return await this.apply(inputs);
+	}
+};
+
+//#endregion
+export { QAEvalChain };
+//# sourceMappingURL=eval_chain.js.map

package/dist/evaluation/qa/eval_chain.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval_chain.js","names":["xs: Set<string>","ys: Set<string>","llm: BaseLanguageModelInterface","options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    }","expectedInputVars: Set<string>","inputVarsSet: Set<string>","examples: ChainValues","predictions: ChainValues","args: EvaluateArgs","example: ChainValues","i: number"],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,QAAQ,CAACA,IAAiBC,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiC,SAAS;CACxC,OAAO,UAAU;AACf,SAAO;CACR;CAED,OAAO,QACLC,KACAC,UAGI,CAAE,GACO;EACb,MAAM,SAAS,QAAQ,UAAU;EACjC,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,OAAO;AAEjD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,OAAO,gBACP;AAGN,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;EAAY;CACvD;CAED,MAAM,SACJC,UACAC,aACAC,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;CAChB,GACqB;EACtB,MAAM,SAAS,SAAS,IAAI,CAACC,SAAsBC,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;EAC7B,GAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO;CAChC;AACF"}

package/dist/evaluation/qa/index.cjs

@@ -0,0 +1 @@
+const require_eval_chain = require('./eval_chain.cjs');

package/dist/evaluation/qa/index.js

@@ -0,0 +1 @@
+import { QAEvalChain } from "./eval_chain.js";

package/dist/evaluation/qa/prompt.cjs

@@ -0,0 +1,31 @@
+const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
+const __langchain_core_prompts = require_rolldown_runtime.__toESM(require("@langchain/core/prompts"));
+
+//#region src/evaluation/qa/prompt.ts
+const QA_TEMPLATE = `You are a teacher grading a quiz.
+You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
+
+Example Format:
+QUESTION: question here
+STUDENT ANSWER: student's answer here
+TRUE ANSWER: true answer here
+GRADE: CORRECT or INCORRECT here
+
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+
+QUESTION: {query}
+STUDENT ANSWER: {result}
+TRUE ANSWER: {answer}
+GRADE:`;
+const QA_PROMPT = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
+	inputVariables: [
+		"query",
+		"result",
+		"answer"
+	],
+	template: QA_TEMPLATE
+});
+
+//#endregion
+exports.QA_PROMPT = QA_PROMPT;
+//# sourceMappingURL=prompt.cjs.map

package/dist/evaluation/qa/prompt.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.cjs","names":["PromptTemplate"],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;;AAEA,MAAM,cAAc,CAAC;;;;;;;;;;;;;;MAcf,CAAC;AACP,MAAa,4BAA4B,IAAIA,wCAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;CAAS;CAC7C,UAAU;AACX"}

package/dist/evaluation/qa/prompt.js

@@ -0,0 +1,30 @@
+import { PromptTemplate } from "@langchain/core/prompts";
+
+//#region src/evaluation/qa/prompt.ts
+const QA_TEMPLATE = `You are a teacher grading a quiz.
+You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.
+
+Example Format:
+QUESTION: question here
+STUDENT ANSWER: student's answer here
+TRUE ANSWER: true answer here
+GRADE: CORRECT or INCORRECT here
+
+Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! 
+
+QUESTION: {query}
+STUDENT ANSWER: {result}
+TRUE ANSWER: {answer}
+GRADE:`;
+const QA_PROMPT = /* @__PURE__ */ new PromptTemplate({
+	inputVariables: [
+		"query",
+		"result",
+		"answer"
+	],
+	template: QA_TEMPLATE
+});
+
+//#endregion
+export { QA_PROMPT };
+//# sourceMappingURL=prompt.js.map

package/dist/evaluation/qa/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;AAEA,MAAM,cAAc,CAAC;;;;;;;;;;;;;;MAcf,CAAC;AACP,MAAa,4BAA4B,IAAI,eAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;CAAS;CAC7C,UAAU;AACX"}