@langchain/classic 1.0.17 → 1.0.18

package/dist/evaluation/criteria/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/criteria/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst template = `You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n[BEGIN DATA]\n***\n[Input]: {input}\n***\n[Submission]: {output}\n***\n[Criteria]: {criteria}\n***\n[END DATA]\nDoes the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.`;\n\nexport const CRITERIA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"output\", \"criteria\"],\n  template,\n});\n\nconst referenceTemplate = `You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n[BEGIN DATA]\n***\n[Input]: {input}\n***\n[Submission]: {output}\n***\n[Criteria]: {criteria}\n***\n[Reference]: {reference}\n***\n[END DATA]\nDoes the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.`;\n\nexport const PROMPT_WITH_REFERENCES = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"output\", \"criteria\", \"reference\"],\n  template: referenceTemplate,\n});\n"],"mappings":";;;AAEA,MAAM,WAAW,CAAC;;;;;;;;;;4cAU0b,CAAC;AAE7c,MAAa,kCAAkC,IAAI,eAAe;CAChE,gBAAgB;EAAC;EAAS;EAAU;CAAW;CAC/C;AACD;AAED,MAAM,oBAAoB,CAAC;;;;;;;;;;;;4cAYib,CAAC;AAE7c,MAAa,yCAAyC,IAAI,eAAe;CACvE,gBAAgB;EAAC;EAAS;EAAU;EAAY;CAAY;CAC5D,UAAU;AACX"}
+{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/criteria/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst template = `You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n[BEGIN DATA]\n***\n[Input]: {input}\n***\n[Submission]: {output}\n***\n[Criteria]: {criteria}\n***\n[END DATA]\nDoes the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.`;\n\nexport const CRITERIA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"output\", \"criteria\"],\n  template,\n});\n\nconst referenceTemplate = `You are assessing a submitted answer on a given task or input based on a set of criteria. Here is the data:\n[BEGIN DATA]\n***\n[Input]: {input}\n***\n[Submission]: {output}\n***\n[Criteria]: {criteria}\n***\n[Reference]: {reference}\n***\n[END DATA]\nDoes the submission meet the Criteria? First, write out in a step by step manner your reasoning about each criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character \"Y\" or \"N\" (without quotes or punctuation) on its own line corresponding to the correct answer of whether the submission meets all criteria. At the end, repeat just the letter again by itself on a new line.`;\n\nexport const PROMPT_WITH_REFERENCES = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"input\", \"output\", \"criteria\", \"reference\"],\n  template: referenceTemplate,\n});\n"],"mappings":";;;AAEA,MAAM,WAAW;;;;;;;;;;;AAYjB,MAAa,kCAAkC,IAAI,eAAe;CAChE,gBAAgB;EAAC;EAAS;EAAU;EAAW;CAC/C;CACD,CAAC;AAEF,MAAM,oBAAoB;;;;;;;;;;;;;AAc1B,MAAa,yCAAyC,IAAI,eAAe;CACvE,gBAAgB;EAAC;EAAS;EAAU;EAAY;EAAY;CAC5D,UAAU;CACX,CAAC"}

package/dist/evaluation/embedding_distance/base.cjs

@@ -10,13 +10,12 @@ const require_euclidean = require('../../util/ml-distance-euclidean/euclidean.cj
 * @return The distance function.
 */
 function getDistanceCalculationFunction(distanceType) {
-	const distanceFunctions = {
+	return {
 		cosine: (X, Y) => 1 - require_similarities.cosine(X, Y),
 		euclidean: require_euclidean.euclidean,
 		manhattan: require_distances.manhattan,
 		chebyshev: require_distances.chebyshev
-	};
-	return distanceFunctions[distanceType];
+	}[distanceType];
 }
 /**
 * Compute the score based on the distance metric.
@@ -25,8 +24,7 @@ function getDistanceCalculationFunction(distanceType) {
 * @return The computed score.
 */
 function computeEvaluationScore(vectors, distanceMetric) {
-	const metricFunction = getDistanceCalculationFunction(distanceMetric);
-	return metricFunction(vectors[0], vectors[1]);
+	return getDistanceCalculationFunction(distanceMetric)(vectors[0], vectors[1]);
 }
 /**
 * Use embedding distances to score semantic difference between
@@ -59,8 +57,7 @@ var EmbeddingDistanceEvalChain = class extends require_base.StringEvaluator {
 	async _call(values, _runManager) {
 		const { prediction, reference } = values;
 		if (!this.embedding) throw new Error("Embedding is undefined");
-		const vectors = await this.embedding.embedDocuments([prediction, reference]);
-		const score = computeEvaluationScore(vectors, this.distanceMetric);
+		const score = computeEvaluationScore(await this.embedding.embedDocuments([prediction, reference]), this.distanceMetric);
 		return { [this.outputKey]: score };
 	}
 };
@@ -94,8 +91,7 @@ var PairwiseEmbeddingDistanceEvalChain = class extends require_base.PairwiseStri
 	async _call(values, _runManager) {
 		const { prediction, predictionB } = values;
 		if (!this.embedding) throw new Error("Embedding is undefined");
-		const vectors = await this.embedding.embedDocuments([prediction, predictionB]);
-		const score = computeEvaluationScore(vectors, this.distanceMetric);
+		const score = computeEvaluationScore(await this.embedding.embedDocuments([prediction, predictionB]), this.distanceMetric);
 		return { [this.outputKey]: score };
 	}
 };

package/dist/evaluation/embedding_distance/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["distanceType: EmbeddingDistanceType","distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction }","X: number[]","Y: number[]","cosine","vectors: number[][]","distanceMetric: EmbeddingDistanceType","StringEvaluator","fields: EmbeddingDistanceEvalChainInput","args: StringEvaluatorArgs","config: Callbacks | BaseCallbackConfig | undefined","values: ChainValues","_runManager: CallbackManagerForChainRun | undefined","PairwiseStringEvaluator","args: PairwiseStringEvaluatorArgs","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  CallbackManagerForChainRun,\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  PairwiseStringEvaluator,\n  PairwiseStringEvaluatorArgs,\n  StringEvaluator,\n  StringEvaluatorArgs,\n} from \"../base.js\";\nimport { cosine } from \"../../util/ml-distance/similarities.js\";\nimport { chebyshev, manhattan } from \"../../util/ml-distance/distances.js\";\nimport { euclidean } from \"../../util/ml-distance-euclidean/euclidean.js\";\n\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType =\n  | \"cosine\"\n  | \"euclidean\"\n  | \"manhattan\"\n  | \"chebyshev\";\n\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n  /**\n   * The embedding objects to vectorize the outputs.\n   */\n  embedding?: EmbeddingsInterface;\n\n  /**\n   * The distance metric to use\n   * for comparing the embeddings.\n   */\n  distanceMetric?: EmbeddingDistanceType;\n}\n\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport function getDistanceCalculationFunction(\n  distanceType: EmbeddingDistanceType\n): VectorFunction {\n  const distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction } =\n    {\n      cosine: (X: number[], Y: number[]) => 1.0 - cosine(X, Y),\n      euclidean,\n      manhattan,\n      chebyshev,\n    };\n\n  return distanceFunctions[distanceType];\n}\n\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport function computeEvaluationScore(\n  vectors: number[][],\n  distanceMetric: EmbeddingDistanceType\n): number {\n  const metricFunction = getDistanceCalculationFunction(distanceMetric);\n  return metricFunction(vectors[0], vectors[1]);\n}\n\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport class EmbeddingDistanceEvalChain\n  extends StringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = true;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config: Callbacks | BaseCallbackConfig | undefined\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"reference\", \"prediction\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, reference } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      reference,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport class PairwiseEmbeddingDistanceEvalChain\n  extends PairwiseStringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = false;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `pairwise_embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"prediction\", \"predictionB\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, predictionB } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      predictionB,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwDA,SAAgB,+BACdA,cACgB;CAChB,MAAMC,oBACJ;EACE,QAAQ,CAACC,GAAaC,MAAgB,IAAMC,4BAAO,GAAG,EAAE;EACxD;EACA;EACA;CACD;AAEH,QAAO,kBAAkB;AAC1B;;;;;;;AAQD,SAAgB,uBACdC,SACAC,gBACQ;CACR,MAAM,iBAAiB,+BAA+B,eAAe;AACrE,QAAO,eAAe,QAAQ,IAAI,QAAQ,GAAG;AAC9C;;;;;AAMD,IAAa,6BAAb,cACUC,6BAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAYC,QAAyC;EACnD,OAAO;EACP,KAAK,YAAY,QAAQ;EACzB,KAAK,iBAAiB,QAAQ,kBAAkB;CACjD;CAED,aAAa;AACX,SAAO,CAAC,UAAU,EAAE,KAAK,eAAe,SAAS,CAAC;CACnD;CAED,MAAM,iBACJC,MACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,WAAY;CACpD;CAED,IAAI,YAAsB;AACxB,SAAO,CAAC,aAAa,YAAa;CACnC;CAED,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,SAAU;CACxB;CAED,MAAM,MACJC,QACAC,aACsB;EACtB,MAAM,EAAE,YAAY,WAAW,GAAG;AAElC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM;EAErC,MAAM,UAAU,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,SACD,EAAC;EAEF,MAAM,QAAQ,uBAAuB,SAAS,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,MAAO;CACnC;AACF;;;;AAKD,IAAa,qCAAb,cACUC,qCAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAYL,QAAyC;EACnD,OAAO;EACP,KAAK,YAAY,QAAQ;EACzB,KAAK,iBAAiB,QAAQ,kBAAkB;CACjD;CAED,aAAa;AACX,SAAO,CAAC,mBAAmB,EAAE,KAAK,eAAe,SAAS,CAAC;CAC5D;CAED,MAAM,qBACJM,MACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,WAAY;CACpD;CAED,IAAI,YAAsB;AACxB,SAAO,CAAC,cAAc,aAAc;CACrC;CAED,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,SAAU;CACxB;CAED,MAAM,MACJJ,QACAC,aACsB;EACtB,MAAM,EAAE,YAAY,aAAa,GAAG;AAEpC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM;EAErC,MAAM,UAAU,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,WACD,EAAC;EAEF,MAAM,QAAQ,uBAAuB,SAAS,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,MAAO;CACnC;AACF"}
+{"version":3,"file":"base.cjs","names":["cosine","StringEvaluator","PairwiseStringEvaluator"],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  CallbackManagerForChainRun,\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  PairwiseStringEvaluator,\n  PairwiseStringEvaluatorArgs,\n  StringEvaluator,\n  StringEvaluatorArgs,\n} from \"../base.js\";\nimport { cosine } from \"../../util/ml-distance/similarities.js\";\nimport { chebyshev, manhattan } from \"../../util/ml-distance/distances.js\";\nimport { euclidean } from \"../../util/ml-distance-euclidean/euclidean.js\";\n\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType =\n  | \"cosine\"\n  | \"euclidean\"\n  | \"manhattan\"\n  | \"chebyshev\";\n\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n  /**\n   * The embedding objects to vectorize the outputs.\n   */\n  embedding?: EmbeddingsInterface;\n\n  /**\n   * The distance metric to use\n   * for comparing the embeddings.\n   */\n  distanceMetric?: EmbeddingDistanceType;\n}\n\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport function getDistanceCalculationFunction(\n  distanceType: EmbeddingDistanceType\n): VectorFunction {\n  const distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction } =\n    {\n      cosine: (X: number[], Y: number[]) => 1.0 - cosine(X, Y),\n      euclidean,\n      manhattan,\n      chebyshev,\n    };\n\n  return distanceFunctions[distanceType];\n}\n\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport function computeEvaluationScore(\n  vectors: number[][],\n  distanceMetric: EmbeddingDistanceType\n): number {\n  const metricFunction = getDistanceCalculationFunction(distanceMetric);\n  return metricFunction(vectors[0], vectors[1]);\n}\n\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport class EmbeddingDistanceEvalChain\n  extends StringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = true;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config: Callbacks | BaseCallbackConfig | undefined\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"reference\", \"prediction\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, reference } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      reference,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport class PairwiseEmbeddingDistanceEvalChain\n  extends PairwiseStringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = false;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `pairwise_embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"prediction\", \"predictionB\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, predictionB } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      predictionB,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwDA,SAAgB,+BACd,cACgB;AAShB,QAPE;EACE,SAAS,GAAa,MAAgB,IAAMA,4BAAO,GAAG,EAAE;EACxD;EACA;EACA;EACD,CAEsB;;;;;;;;AAS3B,SAAgB,uBACd,SACA,gBACQ;AAER,QADuB,+BAA+B,eAAe,CAC/C,QAAQ,IAAI,QAAQ,GAAG;;;;;;AAO/C,IAAa,6BAAb,cACUC,6BAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAY,QAAyC;AACnD,SAAO;AACP,OAAK,YAAY,QAAQ;AACzB,OAAK,iBAAiB,QAAQ,kBAAkB;;CAGlD,aAAa;AACX,SAAO,aAAa,KAAK,eAAe;;CAG1C,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,YAAY;;CAGrD,IAAI,YAAsB;AACxB,SAAO,CAAC,aAAa,aAAa;;CAGpC,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,UAAU;;CAGzB,MAAM,MACJ,QACA,aACsB;EACtB,MAAM,EAAE,YAAY,cAAc;AAElC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM,yBAAyB;EAO9D,MAAM,QAAQ,uBALE,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,UACD,CAAC,EAE4C,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,OAAO;;;;;;AAOtC,IAAa,qCAAb,cACUC,qCAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAY,QAAyC;AACnD,SAAO;AACP,OAAK,YAAY,QAAQ;AACzB,OAAK,iBAAiB,QAAQ,kBAAkB;;CAGlD,aAAa;AACX,SAAO,sBAAsB,KAAK,eAAe;;CAGnD,MAAM,qBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,YAAY;;CAGrD,IAAI,YAAsB;AACxB,SAAO,CAAC,cAAc,cAAc;;CAGtC,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,UAAU;;CAGzB,MAAM,MACJ,QACA,aACsB;EACtB,MAAM,EAAE,YAAY,gBAAgB;AAEpC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM,yBAAyB;EAO9D,MAAM,QAAQ,uBALE,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,YACD,CAAC,EAE4C,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,OAAO"}

package/dist/evaluation/embedding_distance/base.d.cts

@@ -4,7 +4,6 @@ import { BaseCallbackConfig, CallbackManagerForChainRun, Callbacks } from "@lang
 import { EmbeddingsInterface } from "@langchain/core/embeddings";
 
 //#region src/evaluation/embedding_distance/base.d.ts
-
 /**
  *
  * Embedding Distance Metric.

package/dist/evaluation/embedding_distance/base.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.cts","names":["EmbeddingsInterface","ChainValues","CallbackManagerForChainRun","Callbacks","BaseCallbackConfig","PairwiseStringEvaluator","PairwiseStringEvaluatorArgs","StringEvaluator","StringEvaluatorArgs","EmbeddingDistanceType","EmbeddingDistanceEvalChainInput","VectorFunction","getDistanceCalculationFunction","computeEvaluationScore","EmbeddingDistanceEvalChain","Promise","PairwiseEmbeddingDistanceEvalChain"],"sources":["../../../src/evaluation/embedding_distance/base.d.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { CallbackManagerForChainRun, Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs } from \"../base.js\";\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType = \"cosine\" | \"euclidean\" | \"manhattan\" | \"chebyshev\";\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n    /**\n     * The embedding objects to vectorize the outputs.\n     */\n    embedding?: EmbeddingsInterface;\n    /**\n     * The distance metric to use\n     * for comparing the embeddings.\n     */\n    distanceMetric?: EmbeddingDistanceType;\n}\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport declare function getDistanceCalculationFunction(distanceType: EmbeddingDistanceType): VectorFunction;\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport declare function computeEvaluationScore(vectors: number[][], distanceMetric: EmbeddingDistanceType): number;\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport declare class EmbeddingDistanceEvalChain extends StringEvaluator implements EmbeddingDistanceEvalChainInput {\n    requiresReference: boolean;\n    requiresInput: boolean;\n    outputKey: string;\n    embedding?: EmbeddingsInterface;\n    distanceMetric: EmbeddingDistanceType;\n    constructor(fields: EmbeddingDistanceEvalChainInput);\n    _chainType(): \"embedding_chebyshev_distance\" | \"embedding_cosine_distance\" | \"embedding_euclidean_distance\" | \"embedding_manhattan_distance\";\n    _evaluateStrings(args: StringEvaluatorArgs, config: Callbacks | BaseCallbackConfig | undefined): Promise<ChainValues>;\n    get inputKeys(): string[];\n    get outputKeys(): string[];\n    _call(values: ChainValues, _runManager: CallbackManagerForChainRun | undefined): Promise<ChainValues>;\n}\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport declare class PairwiseEmbeddingDistanceEvalChain extends PairwiseStringEvaluator implements EmbeddingDistanceEvalChainInput {\n    requiresReference: boolean;\n    requiresInput: boolean;\n    outputKey: string;\n    embedding?: EmbeddingsInterface;\n    distanceMetric: EmbeddingDistanceType;\n    constructor(fields: EmbeddingDistanceEvalChainInput);\n    _chainType(): \"pairwise_embedding_chebyshev_distance\" | \"pairwise_embedding_cosine_distance\" | \"pairwise_embedding_euclidean_distance\" | \"pairwise_embedding_manhattan_distance\";\n    _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    get inputKeys(): string[];\n    get outputKeys(): string[];\n    _call(values: ChainValues, _runManager: CallbackManagerForChainRun | undefined): Promise<ChainValues>;\n}\nexport {};\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;;;AAcA;AAIA;AAUC;AAOD;AAOA;AAKA;;;AAMwBU,KAvCZD,qBAAAA,GAuCYC,QAAAA,GAAAA,WAAAA,GAAAA,WAAAA,GAAAA,WAAAA;;;;AAEqFT,UArC5FS,+BAAAA,CAqC4FT;EAARc;;;EAGRd,SAAAA,CAAAA,EApC7ED,mBAoC6EC;EAARc;;;AAX6B;EAgB7FC,cAAAA,CAAAA,EApCAP,qBAoCkC;;KAlClDE,cAAAA,GAuCeF,CAAAA,OAAAA,EAAAA,MAAAA,EAAAA,EAAAA,OAAAA,EAAAA,MAAAA,EAAAA,EAAAA,GAAAA,MAAAA;;;;;;AAGkFM,iBApC9EH,8BAAAA,CAoC8EG,YAAAA,EApCjCN,qBAoCiCM,CAAAA,EApCTJ,cAoCSI;;;;;;;AAR4B,iBArB1GF,sBAAAA,CAqB0G,OAAA,EAAA,MAAA,EAAA,EAAA,EAAA,cAAA,EArB9CJ,qBAqB8C,CAAA,EAAA,MAAA;;;;;cAhB7GK,0BAAAA,SAAmCP,eAAAA,YAA2BG;;;;cAInEV;kBACIS;sBACIC;;yBAEGF,6BAA6BL,YAAYC,iCAAiCW,QAAQd;;;gBAG3FA,0BAA0BC,yCAAyCa,QAAQd;;;;;cAKxEe,kCAAAA,SAA2CX,uBAAAA,YAAmCK;;;;cAInFV;kBACIS;sBACIC;;6BAEOJ,sCAAsCH,YAAYC,qBAAqBW,QAAQd;;;gBAG5FA,0BAA0BC,yCAAyCa,QAAQd"}
+{"version":3,"file":"base.d.cts","names":[],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"mappings":";;;;;;;;AA2BA;;;;;AASA;;;KATY,qBAAA;;;;UASK,+BAAA;EAUuB;;AACvC;EAPC,SAAA,GAAY,mBAAA;;;;AAgBd;EAVE,cAAA,GAAiB,qBAAA;AAAA;AAAA,KAGd,cAAA,IAAkB,OAAA,YAAmB,OAAA;;;;;;iBAO1B,8BAAA,CACd,YAAA,EAAc,qBAAA,GACb,cAAA;;;;;;;iBAkBa,sBAAA,CACd,OAAA,cACA,cAAA,EAAgB,qBAAA;;AAUlB;;;cAAa,0BAAA,SACH,eAAA,YACG,+BAAA;EAEX,iBAAA;EAEA,aAAA;EAEA,SAAA;EAEA,SAAA,GAAY,mBAAA;EAEZ,cAAA,EAAgB,qBAAA;EAEhB,WAAA,CAAY,MAAA,EAAQ,+BAAA;EAMpB,UAAA,CAAA;EAIM,gBAAA,CACJ,IAAA,EAAM,mBAAA,EACN,MAAA,EAAQ,SAAA,GAAY,kBAAA,eACnB,OAAA,CAAQ,WAAA;EAAA,IAMP,SAAA,CAAA;EAAA,IAIA,UAAA,CAAA;EAIE,KAAA,CACJ,MAAA,EAAQ,WAAA,EACR,WAAA,EAAa,0BAAA,eACZ,OAAA,CAAQ,WAAA;AAAA;;;;cAmBA,kCAAA,SACH,uBAAA,YACG,+BAAA;EAEX,iBAAA;EAEA,aAAA;EAEA,SAAA;EAEA,SAAA,GAAY,mBAAA;EAEZ,cAAA,EAAgB,qBAAA;EAEhB,WAAA,CAAY,MAAA,EAAQ,+BAAA;EAMpB,UAAA,CAAA;EAIM,oBAAA,CACJ,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAAA,IAMP,SAAA,CAAA;EAAA,IAIA,UAAA,CAAA;EAIE,KAAA,CACJ,MAAA,EAAQ,WAAA,EACR,WAAA,EAAa,0BAAA,eACZ,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/embedding_distance/base.d.ts

@@ -4,7 +4,6 @@ import { ChainValues } from "@langchain/core/utils/types";
 import { EmbeddingsInterface } from "@langchain/core/embeddings";
 
 //#region src/evaluation/embedding_distance/base.d.ts
-
 /**
  *
  * Embedding Distance Metric.

package/dist/evaluation/embedding_distance/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","names":["EmbeddingsInterface","ChainValues","CallbackManagerForChainRun","Callbacks","BaseCallbackConfig","PairwiseStringEvaluator","PairwiseStringEvaluatorArgs","StringEvaluator","StringEvaluatorArgs","EmbeddingDistanceType","EmbeddingDistanceEvalChainInput","VectorFunction","getDistanceCalculationFunction","computeEvaluationScore","EmbeddingDistanceEvalChain","Promise","PairwiseEmbeddingDistanceEvalChain"],"sources":["../../../src/evaluation/embedding_distance/base.d.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { CallbackManagerForChainRun, Callbacks, BaseCallbackConfig } from \"@langchain/core/callbacks/manager\";\nimport { PairwiseStringEvaluator, PairwiseStringEvaluatorArgs, StringEvaluator, StringEvaluatorArgs } from \"../base.js\";\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType = \"cosine\" | \"euclidean\" | \"manhattan\" | \"chebyshev\";\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n    /**\n     * The embedding objects to vectorize the outputs.\n     */\n    embedding?: EmbeddingsInterface;\n    /**\n     * The distance metric to use\n     * for comparing the embeddings.\n     */\n    distanceMetric?: EmbeddingDistanceType;\n}\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport declare function getDistanceCalculationFunction(distanceType: EmbeddingDistanceType): VectorFunction;\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport declare function computeEvaluationScore(vectors: number[][], distanceMetric: EmbeddingDistanceType): number;\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport declare class EmbeddingDistanceEvalChain extends StringEvaluator implements EmbeddingDistanceEvalChainInput {\n    requiresReference: boolean;\n    requiresInput: boolean;\n    outputKey: string;\n    embedding?: EmbeddingsInterface;\n    distanceMetric: EmbeddingDistanceType;\n    constructor(fields: EmbeddingDistanceEvalChainInput);\n    _chainType(): \"embedding_chebyshev_distance\" | \"embedding_cosine_distance\" | \"embedding_euclidean_distance\" | \"embedding_manhattan_distance\";\n    _evaluateStrings(args: StringEvaluatorArgs, config: Callbacks | BaseCallbackConfig | undefined): Promise<ChainValues>;\n    get inputKeys(): string[];\n    get outputKeys(): string[];\n    _call(values: ChainValues, _runManager: CallbackManagerForChainRun | undefined): Promise<ChainValues>;\n}\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport declare class PairwiseEmbeddingDistanceEvalChain extends PairwiseStringEvaluator implements EmbeddingDistanceEvalChainInput {\n    requiresReference: boolean;\n    requiresInput: boolean;\n    outputKey: string;\n    embedding?: EmbeddingsInterface;\n    distanceMetric: EmbeddingDistanceType;\n    constructor(fields: EmbeddingDistanceEvalChainInput);\n    _chainType(): \"pairwise_embedding_chebyshev_distance\" | \"pairwise_embedding_cosine_distance\" | \"pairwise_embedding_euclidean_distance\" | \"pairwise_embedding_manhattan_distance\";\n    _evaluateStringPairs(args: PairwiseStringEvaluatorArgs, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;\n    get inputKeys(): string[];\n    get outputKeys(): string[];\n    _call(values: ChainValues, _runManager: CallbackManagerForChainRun | undefined): Promise<ChainValues>;\n}\nexport {};\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;;;AAcA;AAIA;AAUC;AAOD;AAOA;AAKA;;;AAMwBU,KAvCZD,qBAAAA,GAuCYC,QAAAA,GAAAA,WAAAA,GAAAA,WAAAA,GAAAA,WAAAA;;;;AAEqFT,UArC5FS,+BAAAA,CAqC4FT;EAARc;;;EAGRd,SAAAA,CAAAA,EApC7ED,mBAoC6EC;EAARc;;;AAX6B;EAgB7FC,cAAAA,CAAAA,EApCAP,qBAoCkC;;KAlClDE,cAAAA,GAuCeF,CAAAA,OAAAA,EAAAA,MAAAA,EAAAA,EAAAA,OAAAA,EAAAA,MAAAA,EAAAA,EAAAA,GAAAA,MAAAA;;;;;;AAGkFM,iBApC9EH,8BAAAA,CAoC8EG,YAAAA,EApCjCN,qBAoCiCM,CAAAA,EApCTJ,cAoCSI;;;;;;;AAR4B,iBArB1GF,sBAAAA,CAqB0G,OAAA,EAAA,MAAA,EAAA,EAAA,EAAA,cAAA,EArB9CJ,qBAqB8C,CAAA,EAAA,MAAA;;;;;cAhB7GK,0BAAAA,SAAmCP,eAAAA,YAA2BG;;;;cAInEV;kBACIS;sBACIC;;yBAEGF,6BAA6BL,YAAYC,iCAAiCW,QAAQd;;;gBAG3FA,0BAA0BC,yCAAyCa,QAAQd;;;;;cAKxEe,kCAAAA,SAA2CX,uBAAAA,YAAmCK;;;;cAInFV;kBACIS;sBACIC;;6BAEOJ,sCAAsCH,YAAYC,qBAAqBW,QAAQd;;;gBAG5FA,0BAA0BC,yCAAyCa,QAAQd"}
+{"version":3,"file":"base.d.ts","names":[],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"mappings":";;;;;;;;AA2BA;;;;;AASA;;;KATY,qBAAA;;;;UASK,+BAAA;EAUuB;;AACvC;EAPC,SAAA,GAAY,mBAAA;;;;AAgBd;EAVE,cAAA,GAAiB,qBAAA;AAAA;AAAA,KAGd,cAAA,IAAkB,OAAA,YAAmB,OAAA;;;;;;iBAO1B,8BAAA,CACd,YAAA,EAAc,qBAAA,GACb,cAAA;;;;;;;iBAkBa,sBAAA,CACd,OAAA,cACA,cAAA,EAAgB,qBAAA;;AAUlB;;;cAAa,0BAAA,SACH,eAAA,YACG,+BAAA;EAEX,iBAAA;EAEA,aAAA;EAEA,SAAA;EAEA,SAAA,GAAY,mBAAA;EAEZ,cAAA,EAAgB,qBAAA;EAEhB,WAAA,CAAY,MAAA,EAAQ,+BAAA;EAMpB,UAAA,CAAA;EAIM,gBAAA,CACJ,IAAA,EAAM,mBAAA,EACN,MAAA,EAAQ,SAAA,GAAY,kBAAA,eACnB,OAAA,CAAQ,WAAA;EAAA,IAMP,SAAA,CAAA;EAAA,IAIA,UAAA,CAAA;EAIE,KAAA,CACJ,MAAA,EAAQ,WAAA,EACR,WAAA,EAAa,0BAAA,eACZ,OAAA,CAAQ,WAAA;AAAA;;;;cAmBA,kCAAA,SACH,uBAAA,YACG,+BAAA;EAEX,iBAAA;EAEA,aAAA;EAEA,SAAA;EAEA,SAAA,GAAY,mBAAA;EAEZ,cAAA,EAAgB,qBAAA;EAEhB,WAAA,CAAY,MAAA,EAAQ,+BAAA;EAMpB,UAAA,CAAA;EAIM,oBAAA,CACJ,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAAA,IAMP,SAAA,CAAA;EAAA,IAIA,UAAA,CAAA;EAIE,KAAA,CACJ,MAAA,EAAQ,WAAA,EACR,WAAA,EAAa,0BAAA,eACZ,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/embedding_distance/base.js

@@ -10,13 +10,12 @@ import { euclidean } from "../../util/ml-distance-euclidean/euclidean.js";
 * @return The distance function.
 */
 function getDistanceCalculationFunction(distanceType) {
-	const distanceFunctions = {
+	return {
 		cosine: (X, Y) => 1 - cosine(X, Y),
 		euclidean,
 		manhattan,
 		chebyshev
-	};
-	return distanceFunctions[distanceType];
+	}[distanceType];
 }
 /**
 * Compute the score based on the distance metric.
@@ -25,8 +24,7 @@ function getDistanceCalculationFunction(distanceType) {
 * @return The computed score.
 */
 function computeEvaluationScore(vectors, distanceMetric) {
-	const metricFunction = getDistanceCalculationFunction(distanceMetric);
-	return metricFunction(vectors[0], vectors[1]);
+	return getDistanceCalculationFunction(distanceMetric)(vectors[0], vectors[1]);
 }
 /**
 * Use embedding distances to score semantic difference between
@@ -59,8 +57,7 @@ var EmbeddingDistanceEvalChain = class extends StringEvaluator {
 	async _call(values, _runManager) {
 		const { prediction, reference } = values;
 		if (!this.embedding) throw new Error("Embedding is undefined");
-		const vectors = await this.embedding.embedDocuments([prediction, reference]);
-		const score = computeEvaluationScore(vectors, this.distanceMetric);
+		const score = computeEvaluationScore(await this.embedding.embedDocuments([prediction, reference]), this.distanceMetric);
 		return { [this.outputKey]: score };
 	}
 };
@@ -94,8 +91,7 @@ var PairwiseEmbeddingDistanceEvalChain = class extends PairwiseStringEvaluator {
 	async _call(values, _runManager) {
 		const { prediction, predictionB } = values;
 		if (!this.embedding) throw new Error("Embedding is undefined");
-		const vectors = await this.embedding.embedDocuments([prediction, predictionB]);
-		const score = computeEvaluationScore(vectors, this.distanceMetric);
+		const score = computeEvaluationScore(await this.embedding.embedDocuments([prediction, predictionB]), this.distanceMetric);
 		return { [this.outputKey]: score };
 	}
 };

package/dist/evaluation/embedding_distance/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":["distanceType: EmbeddingDistanceType","distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction }","X: number[]","Y: number[]","vectors: number[][]","distanceMetric: EmbeddingDistanceType","fields: EmbeddingDistanceEvalChainInput","args: StringEvaluatorArgs","config: Callbacks | BaseCallbackConfig | undefined","values: ChainValues","_runManager: CallbackManagerForChainRun | undefined","args: PairwiseStringEvaluatorArgs","config?: Callbacks | BaseCallbackConfig"],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  CallbackManagerForChainRun,\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  PairwiseStringEvaluator,\n  PairwiseStringEvaluatorArgs,\n  StringEvaluator,\n  StringEvaluatorArgs,\n} from \"../base.js\";\nimport { cosine } from \"../../util/ml-distance/similarities.js\";\nimport { chebyshev, manhattan } from \"../../util/ml-distance/distances.js\";\nimport { euclidean } from \"../../util/ml-distance-euclidean/euclidean.js\";\n\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType =\n  | \"cosine\"\n  | \"euclidean\"\n  | \"manhattan\"\n  | \"chebyshev\";\n\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n  /**\n   * The embedding objects to vectorize the outputs.\n   */\n  embedding?: EmbeddingsInterface;\n\n  /**\n   * The distance metric to use\n   * for comparing the embeddings.\n   */\n  distanceMetric?: EmbeddingDistanceType;\n}\n\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport function getDistanceCalculationFunction(\n  distanceType: EmbeddingDistanceType\n): VectorFunction {\n  const distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction } =\n    {\n      cosine: (X: number[], Y: number[]) => 1.0 - cosine(X, Y),\n      euclidean,\n      manhattan,\n      chebyshev,\n    };\n\n  return distanceFunctions[distanceType];\n}\n\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport function computeEvaluationScore(\n  vectors: number[][],\n  distanceMetric: EmbeddingDistanceType\n): number {\n  const metricFunction = getDistanceCalculationFunction(distanceMetric);\n  return metricFunction(vectors[0], vectors[1]);\n}\n\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport class EmbeddingDistanceEvalChain\n  extends StringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = true;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config: Callbacks | BaseCallbackConfig | undefined\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"reference\", \"prediction\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, reference } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      reference,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport class PairwiseEmbeddingDistanceEvalChain\n  extends PairwiseStringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = false;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `pairwise_embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"prediction\", \"predictionB\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, predictionB } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      predictionB,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwDA,SAAgB,+BACdA,cACgB;CAChB,MAAMC,oBACJ;EACE,QAAQ,CAACC,GAAaC,MAAgB,IAAM,OAAO,GAAG,EAAE;EACxD;EACA;EACA;CACD;AAEH,QAAO,kBAAkB;AAC1B;;;;;;;AAQD,SAAgB,uBACdC,SACAC,gBACQ;CACR,MAAM,iBAAiB,+BAA+B,eAAe;AACrE,QAAO,eAAe,QAAQ,IAAI,QAAQ,GAAG;AAC9C;;;;;AAMD,IAAa,6BAAb,cACU,gBAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAYC,QAAyC;EACnD,OAAO;EACP,KAAK,YAAY,QAAQ;EACzB,KAAK,iBAAiB,QAAQ,kBAAkB;CACjD;CAED,aAAa;AACX,SAAO,CAAC,UAAU,EAAE,KAAK,eAAe,SAAS,CAAC;CACnD;CAED,MAAM,iBACJC,MACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,WAAY;CACpD;CAED,IAAI,YAAsB;AACxB,SAAO,CAAC,aAAa,YAAa;CACnC;CAED,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,SAAU;CACxB;CAED,MAAM,MACJC,QACAC,aACsB;EACtB,MAAM,EAAE,YAAY,WAAW,GAAG;AAElC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM;EAErC,MAAM,UAAU,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,SACD,EAAC;EAEF,MAAM,QAAQ,uBAAuB,SAAS,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,MAAO;CACnC;AACF;;;;AAKD,IAAa,qCAAb,cACU,wBAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAYJ,QAAyC;EACnD,OAAO;EACP,KAAK,YAAY,QAAQ;EACzB,KAAK,iBAAiB,QAAQ,kBAAkB;CACjD;CAED,aAAa;AACX,SAAO,CAAC,mBAAmB,EAAE,KAAK,eAAe,SAAS,CAAC;CAC5D;CAED,MAAM,qBACJK,MACAC,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,WAAY;CACpD;CAED,IAAI,YAAsB;AACxB,SAAO,CAAC,cAAc,aAAc;CACrC;CAED,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,SAAU;CACxB;CAED,MAAM,MACJH,QACAC,aACsB;EACtB,MAAM,EAAE,YAAY,aAAa,GAAG;AAEpC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM;EAErC,MAAM,UAAU,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,WACD,EAAC;EAEF,MAAM,QAAQ,uBAAuB,SAAS,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,MAAO;CACnC;AACF"}
+{"version":3,"file":"base.js","names":[],"sources":["../../../src/evaluation/embedding_distance/base.ts"],"sourcesContent":["import type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  CallbackManagerForChainRun,\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  PairwiseStringEvaluator,\n  PairwiseStringEvaluatorArgs,\n  StringEvaluator,\n  StringEvaluatorArgs,\n} from \"../base.js\";\nimport { cosine } from \"../../util/ml-distance/similarities.js\";\nimport { chebyshev, manhattan } from \"../../util/ml-distance/distances.js\";\nimport { euclidean } from \"../../util/ml-distance-euclidean/euclidean.js\";\n\n/**\n *\n * Embedding Distance Metric.\n *\n * COSINE: Cosine distance metric.\n * EUCLIDEAN: Euclidean distance metric.\n * MANHATTAN: Manhattan distance metric.\n * CHEBYSHEV: Chebyshev distance metric.\n * HAMMING: Hamming distance metric.\n */\nexport type EmbeddingDistanceType =\n  | \"cosine\"\n  | \"euclidean\"\n  | \"manhattan\"\n  | \"chebyshev\";\n\n/**\n * Embedding Distance Evaluation Chain Input.\n */\nexport interface EmbeddingDistanceEvalChainInput {\n  /**\n   * The embedding objects to vectorize the outputs.\n   */\n  embedding?: EmbeddingsInterface;\n\n  /**\n   * The distance metric to use\n   * for comparing the embeddings.\n   */\n  distanceMetric?: EmbeddingDistanceType;\n}\n\ntype VectorFunction = (xVector: number[], yVector: number[]) => number;\n\n/**\n * Get the distance function for the given distance type.\n * @param distance The distance type.\n * @return The distance function.\n */\nexport function getDistanceCalculationFunction(\n  distanceType: EmbeddingDistanceType\n): VectorFunction {\n  const distanceFunctions: { [key in EmbeddingDistanceType]: VectorFunction } =\n    {\n      cosine: (X: number[], Y: number[]) => 1.0 - cosine(X, Y),\n      euclidean,\n      manhattan,\n      chebyshev,\n    };\n\n  return distanceFunctions[distanceType];\n}\n\n/**\n * Compute the score based on the distance metric.\n * @param vectors The input vectors.\n * @param distanceMetric The distance metric.\n * @return The computed score.\n */\nexport function computeEvaluationScore(\n  vectors: number[][],\n  distanceMetric: EmbeddingDistanceType\n): number {\n  const metricFunction = getDistanceCalculationFunction(distanceMetric);\n  return metricFunction(vectors[0], vectors[1]);\n}\n\n/**\n * Use embedding distances to score semantic difference between\n * a prediction and reference.\n */\nexport class EmbeddingDistanceEvalChain\n  extends StringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = true;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config: Callbacks | BaseCallbackConfig | undefined\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"reference\", \"prediction\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, reference } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      reference,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n\n/**\n * Use embedding distances to score semantic difference between two predictions.\n */\nexport class PairwiseEmbeddingDistanceEvalChain\n  extends PairwiseStringEvaluator\n  implements EmbeddingDistanceEvalChainInput\n{\n  requiresReference = false;\n\n  requiresInput = false;\n\n  outputKey = \"score\";\n\n  embedding?: EmbeddingsInterface;\n\n  distanceMetric: EmbeddingDistanceType = \"cosine\";\n\n  constructor(fields: EmbeddingDistanceEvalChainInput) {\n    super();\n    this.embedding = fields?.embedding;\n    this.distanceMetric = fields?.distanceMetric || \"cosine\";\n  }\n\n  _chainType() {\n    return `pairwise_embedding_${this.distanceMetric}_distance` as const;\n  }\n\n  async _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call(args, config);\n\n    return { [this.outputKey]: result[this.outputKey] };\n  }\n\n  get inputKeys(): string[] {\n    return [\"prediction\", \"predictionB\"];\n  }\n\n  get outputKeys(): string[] {\n    return [this.outputKey];\n  }\n\n  async _call(\n    values: ChainValues,\n    _runManager: CallbackManagerForChainRun | undefined\n  ): Promise<ChainValues> {\n    const { prediction, predictionB } = values;\n\n    if (!this.embedding) throw new Error(\"Embedding is undefined\");\n\n    const vectors = await this.embedding.embedDocuments([\n      prediction,\n      predictionB,\n    ]);\n\n    const score = computeEvaluationScore(vectors, this.distanceMetric);\n\n    return { [this.outputKey]: score };\n  }\n}\n"],"mappings":";;;;;;;;;;;AAwDA,SAAgB,+BACd,cACgB;AAShB,QAPE;EACE,SAAS,GAAa,MAAgB,IAAM,OAAO,GAAG,EAAE;EACxD;EACA;EACA;EACD,CAEsB;;;;;;;;AAS3B,SAAgB,uBACd,SACA,gBACQ;AAER,QADuB,+BAA+B,eAAe,CAC/C,QAAQ,IAAI,QAAQ,GAAG;;;;;;AAO/C,IAAa,6BAAb,cACU,gBAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAY,QAAyC;AACnD,SAAO;AACP,OAAK,YAAY,QAAQ;AACzB,OAAK,iBAAiB,QAAQ,kBAAkB;;CAGlD,aAAa;AACX,SAAO,aAAa,KAAK,eAAe;;CAG1C,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,YAAY;;CAGrD,IAAI,YAAsB;AACxB,SAAO,CAAC,aAAa,aAAa;;CAGpC,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,UAAU;;CAGzB,MAAM,MACJ,QACA,aACsB;EACtB,MAAM,EAAE,YAAY,cAAc;AAElC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM,yBAAyB;EAO9D,MAAM,QAAQ,uBALE,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,UACD,CAAC,EAE4C,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,OAAO;;;;;;AAOtC,IAAa,qCAAb,cACU,wBAEV;CACE,oBAAoB;CAEpB,gBAAgB;CAEhB,YAAY;CAEZ;CAEA,iBAAwC;CAExC,YAAY,QAAyC;AACnD,SAAO;AACP,OAAK,YAAY,QAAQ;AACzB,OAAK,iBAAiB,QAAQ,kBAAkB;;CAGlD,aAAa;AACX,SAAO,sBAAsB,KAAK,eAAe;;CAGnD,MAAM,qBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,MAAM,OAAO;AAE5C,SAAO,GAAG,KAAK,YAAY,OAAO,KAAK,YAAY;;CAGrD,IAAI,YAAsB;AACxB,SAAO,CAAC,cAAc,cAAc;;CAGtC,IAAI,aAAuB;AACzB,SAAO,CAAC,KAAK,UAAU;;CAGzB,MAAM,MACJ,QACA,aACsB;EACtB,MAAM,EAAE,YAAY,gBAAgB;AAEpC,MAAI,CAAC,KAAK,UAAW,OAAM,IAAI,MAAM,yBAAyB;EAO9D,MAAM,QAAQ,uBALE,MAAM,KAAK,UAAU,eAAe,CAClD,YACA,YACD,CAAC,EAE4C,KAAK,eAAe;AAElE,SAAO,GAAG,KAAK,YAAY,OAAO"}

package/dist/evaluation/embedding_distance/index.js

@@ -1 +1,3 @@
 import { EmbeddingDistanceEvalChain, PairwiseEmbeddingDistanceEvalChain, computeEvaluationScore, getDistanceCalculationFunction } from "./base.js";
+
+export {  };

package/dist/evaluation/index.cjs

@@ -1,4 +1,5 @@
-const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
+const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
 const require_eval_chain = require('./qa/eval_chain.cjs');
 require('./qa/index.cjs');
 const require_criteria = require('./criteria/criteria.cjs');
@@ -12,8 +13,7 @@ require('./comparison/index.cjs');
 const require_loader = require('./loader.cjs');
 
 //#region src/evaluation/index.ts
-var evaluation_exports = {};
-require_rolldown_runtime.__export(evaluation_exports, {
+var evaluation_exports = /* @__PURE__ */ require_runtime.__exportAll({
 	CriteriaEvalChain: () => require_criteria.CriteriaEvalChain,
 	CriteriaResultOutputParser: () => require_criteria.CriteriaResultOutputParser,
 	EmbeddingDistanceEvalChain: () => require_base.EmbeddingDistanceEvalChain,

package/dist/evaluation/index.js

@@ -1,4 +1,4 @@
-import { __export } from "../_virtual/rolldown_runtime.js";
+import { __exportAll } from "../_virtual/_rolldown/runtime.js";
 import { QAEvalChain } from "./qa/eval_chain.js";
 import "./qa/index.js";
 import { CriteriaEvalChain, CriteriaResultOutputParser, LabeledCriteriaEvalChain } from "./criteria/criteria.js";
@@ -12,8 +12,7 @@ import "./comparison/index.js";
 import { loadEvaluator } from "./loader.js";
 
 //#region src/evaluation/index.ts
-var evaluation_exports = {};
-__export(evaluation_exports, {
+var evaluation_exports = /* @__PURE__ */ __exportAll({
 	CriteriaEvalChain: () => CriteriaEvalChain,
 	CriteriaResultOutputParser: () => CriteriaResultOutputParser,
 	EmbeddingDistanceEvalChain: () => EmbeddingDistanceEvalChain,

package/dist/evaluation/loader.cjs

@@ -1,4 +1,4 @@
-const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
 const require_criteria = require('./criteria/criteria.cjs');
 require('./criteria/index.cjs');
 const require_trajectory = require('./agents/trajectory.cjs');
@@ -7,7 +7,7 @@ const require_base = require('./embedding_distance/base.cjs');
 require('./embedding_distance/index.cjs');
 const require_pairwise = require('./comparison/pairwise.cjs');
 require('./comparison/index.cjs');
-const __langchain_core_language_models_chat_models = require_rolldown_runtime.__toESM(require("@langchain/core/language_models/chat_models"));
+let _langchain_core_language_models_chat_models = require("@langchain/core/language_models/chat_models");
 
 //#region src/evaluation/loader.ts
 /**
@@ -35,7 +35,7 @@ async function loadEvaluator(type, options) {
 			evaluator = await require_pairwise.LabeledPairwiseStringEvalChain.fromLLM(llm, criteria, chainOptions);
 			break;
 		case "trajectory":
-			if (!(llm instanceof __langchain_core_language_models_chat_models.BaseChatModel)) throw new Error("LLM must be an instance of a base chat model.");
+			if (!(llm instanceof _langchain_core_language_models_chat_models.BaseChatModel)) throw new Error("LLM must be an instance of a base chat model.");
 			evaluator = await require_trajectory.TrajectoryEvalChain.fromLLM(llm, agentTools, chainOptions);
 			break;
 		case "embedding_distance":

package/dist/evaluation/loader.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"loader.cjs","names":["type: T","options: LoadEvaluatorOptions","evaluator: unknown","CriteriaEvalChain","LabeledCriteriaEvalChain","PairwiseStringEvalChain","LabeledPairwiseStringEvalChain","BaseChatModel","TrajectoryEvalChain","EmbeddingDistanceEvalChain","PairwiseEmbeddingDistanceEvalChain"],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpBA,MACAC,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,YAAY,GAAG;CAEpD,IAAIC;AACJ,SAAQ,MAAR;EACE,KAAK;GACH,YAAY,MAAMC,mCAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;GACH,YAAY,MAAMC,0CAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAMC,yCAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAMC,gDAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAeC,4DACnB,OAAM,IAAI,MAAM;GAElB,YAAY,MAAMC,uCAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;GACH,YAAY,IAAIC,wCAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;GAC1B;AACD;EACF,KAAK;GACH,YAAY,IAAIC,gDAAmC,CAAE;AACrD;EACF,QACE,OAAM,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM;CAC1C;AAED,QAAO;AACR"}
+{"version":3,"file":"loader.cjs","names":["CriteriaEvalChain","LabeledCriteriaEvalChain","PairwiseStringEvalChain","LabeledPairwiseStringEvalChain","BaseChatModel","TrajectoryEvalChain","EmbeddingDistanceEvalChain","PairwiseEmbeddingDistanceEvalChain"],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpB,MACA,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,eAAe;CAEpD,IAAI;AACJ,SAAQ,MAAR;EACE,KAAK;AACH,eAAY,MAAMA,mCAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;AACH,eAAY,MAAMC,0CAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;AACH,eAAY,MAAMC,yCAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;AACH,eAAY,MAAMC,gDAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAeC,2DACnB,OAAM,IAAI,MAAM,gDAAgD;AAElE,eAAY,MAAMC,uCAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;AACH,eAAY,IAAIC,wCAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;IAC1B,CAAC;AACF;EACF,KAAK;AACH,eAAY,IAAIC,gDAAmC,EAAE,CAAC;AACtD;EACF,QACE,OAAM,IAAI,MAAM,iBAAiB,OAAO;;AAG5C,QAAO"}

package/dist/evaluation/loader.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"loader.d.cts","names":["BaseLanguageModelInterface","StructuredToolInterface","CriteriaLike","EvaluatorType","LLMEvalChainInput","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","Omit","Partial","loadEvaluator","T","Promise"],"sources":["../../src/evaluation/loader.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { CriteriaLike } from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport { EmbeddingDistanceEvalChainInput } from \"./embedding_distance/index.js\";\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n    /**\n     * The language model to use for the evaluator.\n     */\n    llm: BaseLanguageModelInterface;\n    /**\n     * The options to pass to the evaluator chain.\n     */\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n    /**\n     * The criteria to use for the evaluator.\n     */\n    criteria?: CriteriaLike;\n    /**\n     * A list of tools available to the agent, for TrajectoryEvalChain.\n     */\n    agentTools?: StructuredToolInterface[];\n};\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;\n//# sourceMappingURL=loader.d.ts.map"],"mappings":";;;;;;;;KAMYM,oBAAAA,GAAuBD;;AAAnC;;EAISL,GAAAA,EAAAA,0BAAAA;EAIuBI;;;EAIjBF,YAAAA,CAAAA,EAJIM,OAIJN,CAJYK,IAIZL,CAJiBE,iBAIjBF,EAAAA,KAAAA,CAAAA,CAAAA;EAIED;AAAuB;AAUxC;EAAsDE,QAAAA,CAAAA,EAdvCD,YAcuCC;EAAqBO;;;EAAyDA,UAAAA,CAAAA,EAVnHT,uBAUmHS,EAAAA;CAAtBC;AAAO;;;;;;;;iBAA7FF,8BAA8BN,qBAAqBO,YAAYJ,uBAAuBK,QAAQR,cAAcO"}
+{"version":3,"file":"loader.d.cts","names":[],"sources":["../../src/evaluation/loader.ts"],"mappings":";;;;;;;;KAqBY,oBAAA,GAAuB,+BAAA;;AAAnC;;EAIE,GAAA,EAAK,0BAAA;EAJ4B;;;EAQjC,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA;EAAb;;;EAIf,QAAA,GAAW,YAAA;EAKyB;;;EAApC,UAAA,GAAa,uBAAA;AAAA;;;;;;;;;iBAWO,aAAA,iBAA8B,aAAA,CAAA,CAClD,IAAA,EAAM,CAAA,EACN,OAAA,EAAS,oBAAA,GACR,OAAA,CAAQ,aAAA,CAAc,CAAA"}

package/dist/evaluation/loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loader.d.ts","names":["BaseLanguageModelInterface","StructuredToolInterface","CriteriaLike","EvaluatorType","LLMEvalChainInput","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","Omit","Partial","loadEvaluator","T","Promise"],"sources":["../../src/evaluation/loader.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { CriteriaLike } from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport { EmbeddingDistanceEvalChainInput } from \"./embedding_distance/index.js\";\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n    /**\n     * The language model to use for the evaluator.\n     */\n    llm: BaseLanguageModelInterface;\n    /**\n     * The options to pass to the evaluator chain.\n     */\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n    /**\n     * The criteria to use for the evaluator.\n     */\n    criteria?: CriteriaLike;\n    /**\n     * A list of tools available to the agent, for TrajectoryEvalChain.\n     */\n    agentTools?: StructuredToolInterface[];\n};\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport declare function loadEvaluator<T extends keyof EvaluatorType>(type: T, options: LoadEvaluatorOptions): Promise<EvaluatorType[T]>;\n//# sourceMappingURL=loader.d.ts.map"],"mappings":";;;;;;;;;;KAMYM,oBAAAA,GAAuBD;;;;EAAvBC,GAAAA,EAIHN,0BAJuB;EAAGK;;;EAQRE,YAAAA,CAAAA,EAARC,OAAQD,CAAAA,IAAAA,CAAKH,iBAALG,EAAAA,KAAAA,CAAAA,CAAAA;EAARC;;;EAQqB,QAAA,CAAA,EAJzBN,YAIyB;EAUhBO;;;EAA+DH,UAAAA,CAAAA,EAVtEL,uBAUsEK,EAAAA;CAA+BH;;;AAAD;;;;;;iBAA7FM,8BAA8BN,qBAAqBO,YAAYJ,uBAAuBK,QAAQR,cAAcO"}
+{"version":3,"file":"loader.d.ts","names":[],"sources":["../../src/evaluation/loader.ts"],"mappings":";;;;;;;;;;KAqBY,oBAAA,GAAuB,+BAAA;;;;EAIjC,GAAA,EAAK,0BAAA;EAJyB;;;EAQ9B,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA;EAAA;;;EAI5B,QAAA,GAAW,YAAA;EAKE;;;EAAb,UAAA,GAAa,uBAAA;AAAA;;;;;;;;;iBAWO,aAAA,iBAA8B,aAAA,CAAA,CAClD,IAAA,EAAM,CAAA,EACN,OAAA,EAAS,oBAAA,GACR,OAAA,CAAQ,aAAA,CAAc,CAAA"}

package/dist/evaluation/loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"loader.js","names":["type: T","options: LoadEvaluatorOptions","evaluator: unknown"],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpBA,MACAC,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,YAAY,GAAG;CAEpD,IAAIC;AACJ,SAAQ,MAAR;EACE,KAAK;GACH,YAAY,MAAM,kBAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;GACH,YAAY,MAAM,yBAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAM,wBAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;GACH,YAAY,MAAM,+BAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAe,eACnB,OAAM,IAAI,MAAM;GAElB,YAAY,MAAM,oBAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;GACH,YAAY,IAAI,2BAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;GAC1B;AACD;EACF,KAAK;GACH,YAAY,IAAI,mCAAmC,CAAE;AACrD;EACF,QACE,OAAM,IAAI,MAAM,CAAC,cAAc,EAAE,MAAM;CAC1C;AAED,QAAO;AACR"}
+{"version":3,"file":"loader.js","names":[],"sources":["../../src/evaluation/loader.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  CriteriaLike,\n  CriteriaEvalChain,\n  LabeledCriteriaEvalChain,\n} from \"./criteria/index.js\";\nimport type { EvaluatorType } from \"./types.js\";\nimport { LLMEvalChainInput } from \"./base.js\";\nimport {\n  LabeledPairwiseStringEvalChain,\n  PairwiseStringEvalChain,\n} from \"./comparison/index.js\";\nimport {\n  EmbeddingDistanceEvalChain,\n  EmbeddingDistanceEvalChainInput,\n  PairwiseEmbeddingDistanceEvalChain,\n} from \"./embedding_distance/index.js\";\nimport { TrajectoryEvalChain } from \"./agents/index.js\";\n\nexport type LoadEvaluatorOptions = EmbeddingDistanceEvalChainInput & {\n  /**\n   * The language model to use for the evaluator.\n   */\n  llm: BaseLanguageModelInterface;\n  /**\n   * The options to pass to the evaluator chain.\n   */\n  chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>;\n  /**\n   * The criteria to use for the evaluator.\n   */\n  criteria?: CriteriaLike;\n\n  /**\n   * A list of tools available to the agent, for TrajectoryEvalChain.\n   */\n  agentTools?: StructuredToolInterface[];\n};\n\n/**\n * Load the requested evaluation chain specified by a string\n * @param type The type of evaluator to load.\n * @param options\n *        - llm The language model to use for the evaluator.\n *        - criteria The criteria to use for the evaluator.\n *        - agentTools A list of tools available to the agent,for TrajectoryEvalChain.\n */\nexport async function loadEvaluator<T extends keyof EvaluatorType>(\n  type: T,\n  options: LoadEvaluatorOptions\n): Promise<EvaluatorType[T]> {\n  const { llm, chainOptions, criteria, agentTools } = options;\n\n  let evaluator: unknown;\n  switch (type) {\n    case \"criteria\":\n      evaluator = await CriteriaEvalChain.fromLLM(llm, criteria, chainOptions);\n      break;\n    case \"labeled_criteria\":\n      evaluator = await LabeledCriteriaEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"pairwise_string\":\n      evaluator = await PairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"labeled_pairwise_string\":\n      evaluator = await LabeledPairwiseStringEvalChain.fromLLM(\n        llm,\n        criteria,\n        chainOptions\n      );\n      break;\n    case \"trajectory\":\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      if (!(llm instanceof BaseChatModel)) {\n        throw new Error(\"LLM must be an instance of a base chat model.\");\n      }\n      evaluator = await TrajectoryEvalChain.fromLLM(\n        llm,\n        agentTools,\n        chainOptions\n      );\n      break;\n    case \"embedding_distance\":\n      evaluator = new EmbeddingDistanceEvalChain({\n        embedding: options?.embedding,\n        distanceMetric: options?.distanceMetric,\n      });\n      break;\n    case \"pairwise_embedding_distance\":\n      evaluator = new PairwiseEmbeddingDistanceEvalChain({});\n      break;\n    default:\n      throw new Error(`Unknown type: ${type}`);\n  }\n\n  return evaluator as EvaluatorType[T];\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,cACpB,MACA,SAC2B;CAC3B,MAAM,EAAE,KAAK,cAAc,UAAU,eAAe;CAEpD,IAAI;AACJ,SAAQ,MAAR;EACE,KAAK;AACH,eAAY,MAAM,kBAAkB,QAAQ,KAAK,UAAU,aAAa;AACxE;EACF,KAAK;AACH,eAAY,MAAM,yBAAyB,QACzC,KACA,UACA,aACD;AACD;EACF,KAAK;AACH,eAAY,MAAM,wBAAwB,QACxC,KACA,UACA,aACD;AACD;EACF,KAAK;AACH,eAAY,MAAM,+BAA+B,QAC/C,KACA,UACA,aACD;AACD;EACF,KAAK;AAEH,OAAI,EAAE,eAAe,eACnB,OAAM,IAAI,MAAM,gDAAgD;AAElE,eAAY,MAAM,oBAAoB,QACpC,KACA,YACA,aACD;AACD;EACF,KAAK;AACH,eAAY,IAAI,2BAA2B;IACzC,WAAW,SAAS;IACpB,gBAAgB,SAAS;IAC1B,CAAC;AACF;EACF,KAAK;AACH,eAAY,IAAI,mCAAmC,EAAE,CAAC;AACtD;EACF,QACE,OAAM,IAAI,MAAM,iBAAiB,OAAO;;AAG5C,QAAO"}

package/dist/evaluation/qa/eval_chain.cjs

@@ -14,8 +14,7 @@ var QAEvalChain = class extends require_llm_chain.LLMChain {
 			"answer",
 			"result"
 		]);
-		const inputVarsSet = new Set(prompt.inputVariables);
-		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
+		if (!eqSet(expectedInputVars, new Set(prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
 		return new this({
 			llm,
 			prompt,

package/dist/evaluation/qa/eval_chain.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"eval_chain.cjs","names":["xs: Set<string>","ys: Set<string>","LLMChain","llm: BaseLanguageModelInterface","options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    }","QA_PROMPT","expectedInputVars: Set<string>","inputVarsSet: Set<string>","examples: ChainValues","predictions: ChainValues","args: EvaluateArgs","example: ChainValues","i: number"],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,QAAQ,CAACA,IAAiBC,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiCC,2BAAS;CACxC,OAAO,UAAU;AACf,SAAO;CACR;CAED,OAAO,QACLC,KACAC,UAGI,CAAE,GACO;EACb,MAAM,SAAS,QAAQ,UAAUC;EACjC,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,OAAO;AAEjD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,OAAO,gBACP;AAGN,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;EAAY;CACvD;CAED,MAAM,SACJC,UACAC,aACAC,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;CAChB,GACqB;EACtB,MAAM,SAAS,SAAS,IAAI,CAACC,SAAsBC,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;EAC7B,GAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO;CAChC;AACF"}
+{"version":3,"file":"eval_chain.cjs","names":["LLMChain","QA_PROMPT"],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,SAAS,IAAiB,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiCA,2BAAS;CACxC,OAAO,UAAU;AACf,SAAO;;CAGT,OAAO,QACL,KACA,UAGI,EAAE,EACO;EACb,MAAM,SAAS,QAAQ,UAAUC;EACjC,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,OAAO,eAAe,CAErB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,OAAO,iBAEV;AAEH,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;GAAY,CAAC;;CAGzD,MAAM,SACJ,UACA,aACA,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;EAChB,EACqB;EACtB,MAAM,SAAS,SAAS,KAAK,SAAsB,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;GAC7B,EAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO"}

package/dist/evaluation/qa/eval_chain.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"eval_chain.d.cts","names":["BaseLanguageModelInterface","ChainValues","PromptTemplate","LLMChain","LLMChainInput","EvaluateArgs","QAEvalChain","Omit","Promise"],"sources":["../../../src/evaluation/qa/eval_chain.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\nexport interface EvaluateArgs {\n    questionKey: string;\n    answerKey: string;\n    predictionKey: string;\n}\nexport declare class QAEvalChain extends LLMChain {\n    static lc_name(): string;\n    static fromLlm(llm: BaseLanguageModelInterface, options?: {\n        prompt?: PromptTemplate;\n        chainInput?: Omit<LLMChainInput, \"llm\">;\n    }): QAEvalChain;\n    evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;\n}\n//# sourceMappingURL=eval_chain.d.ts.map"],"mappings":";;;;;;UAIiBK,YAAAA;;EAAAA,SAAAA,EAAAA,MAAY;EAKRC,aAAAA,EAAW,MAAA;;AAGfJ,cAHII,WAAAA,SAAoBH,QAAAA,CAGxBD;EACSE,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAALG,OAAAA,OAAAA,CAAAA,GAAAA,EAFGP,0BAEHO,EAAAA,OAE4BN,CAF5BM,EAAAA;IACbD,MAAAA,CAAAA,EAFSJ,cAETI;IACeL,UAAAA,CAAAA,EAFFM,IAEEN,CAFGG,aAEHH,EAAAA,KAAAA,CAAAA;EAA0BA,CAAAA,CAAAA,EADzCK,WACyCL;EAAoBI,QAAAA,CAAAA,QAAAA,EAA9CJ,WAA8CI,EAAAA,WAAAA,EAApBJ,WAAoBI,EAAAA,IAAAA,CAAAA,EAAAA,YAAAA,CAAAA,EAAeG,OAAfH,CAAuBJ,WAAvBI,CAAAA"}
+{"version":3,"file":"eval_chain.d.cts","names":[],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"mappings":";;;;;;UAMiB,YAAA;EACf,WAAA;EACA,SAAA;EACA,aAAA;AAAA;AAAA,cAMW,WAAA,SAAoB,QAAA;EAAA,OACxB,OAAA,CAAA;EAAA,OAIA,OAAA,CACL,GAAA,EAAK,0BAAA,EACL,OAAA;IACE,MAAA,GAAS,cAAA;IACT,UAAA,GAAa,IAAA,CAAK,aAAA;EAAA,IAEnB,WAAA;EAoBG,QAAA,CACJ,QAAA,EAAU,WAAA,EACV,WAAA,EAAa,WAAA,EACb,IAAA,GAAM,YAAA,GAKL,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/qa/eval_chain.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval_chain.d.ts","names":["BaseLanguageModelInterface","ChainValues","PromptTemplate","LLMChain","LLMChainInput","EvaluateArgs","QAEvalChain","Omit","Promise"],"sources":["../../../src/evaluation/qa/eval_chain.d.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\nexport interface EvaluateArgs {\n    questionKey: string;\n    answerKey: string;\n    predictionKey: string;\n}\nexport declare class QAEvalChain extends LLMChain {\n    static lc_name(): string;\n    static fromLlm(llm: BaseLanguageModelInterface, options?: {\n        prompt?: PromptTemplate;\n        chainInput?: Omit<LLMChainInput, \"llm\">;\n    }): QAEvalChain;\n    evaluate(examples: ChainValues, predictions: ChainValues, args?: EvaluateArgs): Promise<ChainValues>;\n}\n//# sourceMappingURL=eval_chain.d.ts.map"],"mappings":";;;;;;UAIiBK,YAAAA;;EAAAA,SAAAA,EAAAA,MAAY;EAKRC,aAAAA,EAAW,MAAA;;AAGfJ,cAHII,WAAAA,SAAoBH,QAAAA,CAGxBD;EACSE,OAAAA,OAAAA,CAAAA,CAAAA,EAAAA,MAAAA;EAALG,OAAAA,OAAAA,CAAAA,GAAAA,EAFGP,0BAEHO,EAAAA,OAE4BN,CAF5BM,EAAAA;IACbD,MAAAA,CAAAA,EAFSJ,cAETI;IACeL,UAAAA,CAAAA,EAFFM,IAEEN,CAFGG,aAEHH,EAAAA,KAAAA,CAAAA;EAA0BA,CAAAA,CAAAA,EADzCK,WACyCL;EAAoBI,QAAAA,CAAAA,QAAAA,EAA9CJ,WAA8CI,EAAAA,WAAAA,EAApBJ,WAAoBI,EAAAA,IAAAA,CAAAA,EAAAA,YAAAA,CAAAA,EAAeG,OAAfH,CAAuBJ,WAAvBI,CAAAA"}
+{"version":3,"file":"eval_chain.d.ts","names":[],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"mappings":";;;;;;UAMiB,YAAA;EACf,WAAA;EACA,SAAA;EACA,aAAA;AAAA;AAAA,cAMW,WAAA,SAAoB,QAAA;EAAA,OACxB,OAAA,CAAA;EAAA,OAIA,OAAA,CACL,GAAA,EAAK,0BAAA,EACL,OAAA;IACE,MAAA,GAAS,cAAA;IACT,UAAA,GAAa,IAAA,CAAK,aAAA;EAAA,IAEnB,WAAA;EAoBG,QAAA,CACJ,QAAA,EAAU,WAAA,EACV,WAAA,EAAa,WAAA,EACb,IAAA,GAAM,YAAA,GAKL,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/qa/eval_chain.js

@@ -14,8 +14,7 @@ var QAEvalChain = class extends LLMChain {
 			"answer",
 			"result"
 		]);
-		const inputVarsSet = new Set(prompt.inputVariables);
-		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
+		if (!eqSet(expectedInputVars, new Set(prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${prompt.inputVariables}`);
 		return new this({
 			llm,
 			prompt,

package/dist/evaluation/qa/eval_chain.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval_chain.js","names":["xs: Set<string>","ys: Set<string>","llm: BaseLanguageModelInterface","options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    }","expectedInputVars: Set<string>","inputVarsSet: Set<string>","examples: ChainValues","predictions: ChainValues","args: EvaluateArgs","example: ChainValues","i: number"],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,QAAQ,CAACA,IAAiBC,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,EAAG,EAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiC,SAAS;CACxC,OAAO,UAAU;AACf,SAAO;CACR;CAED,OAAO,QACLC,KACAC,UAGI,CAAE,GACO;EACb,MAAM,SAAS,QAAQ,UAAU;EACjC,MAAMC,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACD;EAED,MAAMC,eAA4B,IAAI,IAAI,OAAO;AAEjD,MAAI,CAAC,MAAM,mBAAmB,aAAa,CACzC,OAAM,IAAI,MACR,CAAC,0BAA0B,EAAE,CAAC,GAAG,iBAAkB,EAAC,UAAU,EAC5D,OAAO,gBACP;AAGN,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;EAAY;CACvD;CAED,MAAM,SACJC,UACAC,aACAC,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;CAChB,GACqB;EACtB,MAAM,SAAS,SAAS,IAAI,CAACC,SAAsBC,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;EAC7B,GAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO;CAChC;AACF"}
+{"version":3,"file":"eval_chain.js","names":[],"sources":["../../../src/evaluation/qa/eval_chain.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { QA_PROMPT } from \"./prompt.js\";\nimport { LLMChain, LLMChainInput } from \"../../chains/llm_chain.js\";\n\nexport interface EvaluateArgs {\n  questionKey: string;\n  answerKey: string;\n  predictionKey: string;\n}\n\nconst eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\nexport class QAEvalChain extends LLMChain {\n  static lc_name() {\n    return \"QAEvalChain\";\n  }\n\n  static fromLlm(\n    llm: BaseLanguageModelInterface,\n    options: {\n      prompt?: PromptTemplate;\n      chainInput?: Omit<LLMChainInput, \"llm\">;\n    } = {}\n  ): QAEvalChain {\n    const prompt = options.prompt || QA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"query\",\n      \"answer\",\n      \"result\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          prompt.inputVariables\n        }`\n      );\n    }\n    return new this({ llm, prompt, ...options.chainInput });\n  }\n\n  async evaluate(\n    examples: ChainValues,\n    predictions: ChainValues,\n    args: EvaluateArgs = {\n      questionKey: \"query\",\n      answerKey: \"answer\",\n      predictionKey: \"result\",\n    }\n  ): Promise<ChainValues> {\n    const inputs = examples.map((example: ChainValues, i: number) => ({\n      query: example[args.questionKey],\n      answer: example[args.answerKey],\n      result: predictions[i][args.predictionKey],\n    }));\n\n    return await this.apply(inputs);\n  }\n}\n"],"mappings":";;;;AAYA,MAAM,SAAS,IAAiB,OAC9B,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;AAExD,IAAa,cAAb,cAAiC,SAAS;CACxC,OAAO,UAAU;AACf,SAAO;;CAGT,OAAO,QACL,KACA,UAGI,EAAE,EACO;EACb,MAAM,SAAS,QAAQ,UAAU;EACjC,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,OAAO,eAAe,CAErB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,OAAO,iBAEV;AAEH,SAAO,IAAI,KAAK;GAAE;GAAK;GAAQ,GAAG,QAAQ;GAAY,CAAC;;CAGzD,MAAM,SACJ,UACA,aACA,OAAqB;EACnB,aAAa;EACb,WAAW;EACX,eAAe;EAChB,EACqB;EACtB,MAAM,SAAS,SAAS,KAAK,SAAsB,OAAe;GAChE,OAAO,QAAQ,KAAK;GACpB,QAAQ,QAAQ,KAAK;GACrB,QAAQ,YAAY,GAAG,KAAK;GAC7B,EAAE;AAEH,SAAO,MAAM,KAAK,MAAM,OAAO"}

package/dist/evaluation/qa/index.js

@@ -1 +1,3 @@
 import { QAEvalChain } from "./eval_chain.js";
+
+export {  };

package/dist/evaluation/qa/prompt.cjs

@@ -1,5 +1,5 @@
-const require_rolldown_runtime = require('../../_virtual/rolldown_runtime.cjs');
-const __langchain_core_prompts = require_rolldown_runtime.__toESM(require("@langchain/core/prompts"));
+const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
+let _langchain_core_prompts = require("@langchain/core/prompts");
 
 //#region src/evaluation/qa/prompt.ts
 const QA_TEMPLATE = `You are a teacher grading a quiz.
@@ -17,7 +17,7 @@ QUESTION: {query}
 STUDENT ANSWER: {result}
 TRUE ANSWER: {answer}
 GRADE:`;
-const QA_PROMPT = /* @__PURE__ */ new __langchain_core_prompts.PromptTemplate({
+const QA_PROMPT = /* @__PURE__ */ new _langchain_core_prompts.PromptTemplate({
 	inputVariables: [
 		"query",
 		"result",

package/dist/evaluation/qa/prompt.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.cjs","names":["PromptTemplate"],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;;AAEA,MAAM,cAAc,CAAC;;;;;;;;;;;;;;MAcf,CAAC;AACP,MAAa,4BAA4B,IAAIA,wCAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;CAAS;CAC7C,UAAU;AACX"}
+{"version":3,"file":"prompt.cjs","names":["PromptTemplate"],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;;AAEA,MAAM,cAAc;;;;;;;;;;;;;;;AAepB,MAAa,4BAA4B,IAAIA,uCAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;EAAS;CAC7C,UAAU;CACX,CAAC"}

package/dist/evaluation/qa/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;AAEA,MAAM,cAAc,CAAC;;;;;;;;;;;;;;MAcf,CAAC;AACP,MAAa,4BAA4B,IAAI,eAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;CAAS;CAC7C,UAAU;AACX"}
+{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/qa/prompt.ts"],"sourcesContent":["import { PromptTemplate } from \"@langchain/core/prompts\";\n\nconst QA_TEMPLATE = `You are a teacher grading a quiz.\nYou are given a question, the student's answer, and the true answer, and are asked to score the student answer as either CORRECT or INCORRECT.\n\nExample Format:\nQUESTION: question here\nSTUDENT ANSWER: student's answer here\nTRUE ANSWER: true answer here\nGRADE: CORRECT or INCORRECT here\n\nGrade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n\nQUESTION: {query}\nSTUDENT ANSWER: {result}\nTRUE ANSWER: {answer}\nGRADE:`;\nexport const QA_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"result\", \"answer\"],\n  template: QA_TEMPLATE,\n});\n\nconst SQL_TEMPLATE = `You are comparing a submitted answer to an expert answer on a given SQL coding question. Here is the data:\n[BEGIN DATA]\n***\n[Question]: {query}\n***\n[Expert]: {answer}\n***\n[Submission]: {result}\n***\n[END DATA]\nCompare the content and correctness of the submitted SQL with the expert answer. Ignore any differences in whitespace, style, or output column names. The submitted answer may either be correct or incorrect. Determine which case applies. First, explain in detail the similarities or differences between the expert answer and the submission, ignoring superficial aspects such as whitespace, style or output column names. Do not state the final answer in your initial explanation. Then, respond with either \"CORRECT\" or \"INCORRECT\" (without quotes or punctuation) on its own line. This should correspond to whether the submitted SQL and the expert answer are semantically the same or different, respectively. Then, repeat your final answer on a new line.`;\n\nexport const SQL_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"query\", \"answer\", \"result\"],\n  template: SQL_TEMPLATE,\n});\n"],"mappings":";;;AAEA,MAAM,cAAc;;;;;;;;;;;;;;;AAepB,MAAa,4BAA4B,IAAI,eAAe;CAC1D,gBAAgB;EAAC;EAAS;EAAU;EAAS;CAC7C,UAAU;CACX,CAAC"}

package/dist/evaluation/types.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.cts","names":["AgentTrajectoryEvaluator","LLMPairwiseStringEvaluator","LLMStringEvaluator","PairwiseStringEvaluator","StringEvaluator","EvaluatorType"],"sources":["../../src/evaluation/types.d.ts"],"sourcesContent":["import { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator } from \"./base.js\";\nexport interface EvaluatorType {\n    /**\n     * The criteria evaluator, which evaluates a model based on a\n     * custom set of criteria without any reference labels.\n     * */\n    criteria: LLMStringEvaluator;\n    /**\n     * The labeled criteria evaluator, which evaluates a model based on a\n     * custom set of criteria, with a reference label.\n     * */\n    labeled_criteria: LLMStringEvaluator;\n    /**\n     *  The pairwise string evaluator, which predicts the preferred prediction from\n     *  between two models.\n     */\n    pairwise_string: LLMPairwiseStringEvaluator;\n    /**\n     * The labeled pairwise string evaluator, which predicts the preferred prediction\n     * from between two models based on a ground truth reference label.\n     * */\n    labeled_pairwise_string: LLMPairwiseStringEvaluator;\n    /**\n     * The agent trajectory evaluator, which grades the agent's intermediate steps.\n     */\n    trajectory: AgentTrajectoryEvaluator;\n    /**\n     * Compare a prediction to a reference label using embedding distance.\n     * */\n    embedding_distance: StringEvaluator;\n    /**\n     * Compare two predictions using embedding distance.\n     * */\n    pairwise_embedding_distance: PairwiseStringEvaluator;\n}\n//# sourceMappingURL=types.d.ts.map"],"mappings":";;;UACiBK,aAAAA;;AAAjB;;;EAeqBJ,QAAAA,EAVPC,kBAUOD;EAKQA;;;;EAY2B,gBAAA,EAtBlCC,kBAsBkC;;;;;mBAjBnCD;;;;;2BAKQA;;;;cAIbD;;;;sBAIQI;;;;+BAISD"}
+{"version":3,"file":"types.d.cts","names":[],"sources":["../../src/evaluation/types.ts"],"mappings":";;;UAQiB,aAAA;;AAAjB;;;EAKE,QAAA,EAAU,kBAAA;EAOQ;;;;EAAlB,gBAAA,EAAkB,kBAAA;EA2BW;;;;EArB7B,eAAA,EAAiB,0BAAA;EANjB;;;;EAYA,uBAAA,EAAyB,0BAAA;EAAA;;;EAKzB,UAAA,EAAY,wBAAA;EAKQ;;;EAApB,kBAAA,EAAoB,eAAA;EAKgC;;;EAApD,2BAAA,EAA6B,uBAAA;AAAA"}

package/dist/evaluation/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":["AgentTrajectoryEvaluator","LLMPairwiseStringEvaluator","LLMStringEvaluator","PairwiseStringEvaluator","StringEvaluator","EvaluatorType"],"sources":["../../src/evaluation/types.d.ts"],"sourcesContent":["import { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator } from \"./base.js\";\nexport interface EvaluatorType {\n    /**\n     * The criteria evaluator, which evaluates a model based on a\n     * custom set of criteria without any reference labels.\n     * */\n    criteria: LLMStringEvaluator;\n    /**\n     * The labeled criteria evaluator, which evaluates a model based on a\n     * custom set of criteria, with a reference label.\n     * */\n    labeled_criteria: LLMStringEvaluator;\n    /**\n     *  The pairwise string evaluator, which predicts the preferred prediction from\n     *  between two models.\n     */\n    pairwise_string: LLMPairwiseStringEvaluator;\n    /**\n     * The labeled pairwise string evaluator, which predicts the preferred prediction\n     * from between two models based on a ground truth reference label.\n     * */\n    labeled_pairwise_string: LLMPairwiseStringEvaluator;\n    /**\n     * The agent trajectory evaluator, which grades the agent's intermediate steps.\n     */\n    trajectory: AgentTrajectoryEvaluator;\n    /**\n     * Compare a prediction to a reference label using embedding distance.\n     * */\n    embedding_distance: StringEvaluator;\n    /**\n     * Compare two predictions using embedding distance.\n     * */\n    pairwise_embedding_distance: PairwiseStringEvaluator;\n}\n//# sourceMappingURL=types.d.ts.map"],"mappings":";;;UACiBK,aAAAA;;AAAjB;;;EAeqBJ,QAAAA,EAVPC,kBAUOD;EAKQA;;;;EAY2B,gBAAA,EAtBlCC,kBAsBkC;;;;;mBAjBnCD;;;;;2BAKQA;;;;cAIbD;;;;sBAIQI;;;;+BAISD"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../../src/evaluation/types.ts"],"mappings":";;;UAQiB,aAAA;;AAAjB;;;EAKE,QAAA,EAAU,kBAAA;EAOQ;;;;EAAlB,gBAAA,EAAkB,kBAAA;EA2BW;;;;EArB7B,eAAA,EAAiB,0BAAA;EANjB;;;;EAYA,uBAAA,EAAyB,0BAAA;EAAA;;;EAKzB,UAAA,EAAY,wBAAA;EAKQ;;;EAApB,kBAAA,EAAoB,eAAA;EAKgC;;;EAApD,2BAAA,EAA6B,uBAAA;AAAA"}