@langchain/classic 1.0.45 → 1.0.46

package/dist/embeddings/cache_backed.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"cache_backed.cjs","names":["Embeddings","EncoderBackedStore"],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2CA,2BAAAA,WAAW;CACpD;CAEA;CAEA,YAAY,QAAqC;EAC/C,MAAM,MAAM;EACZ,KAAK,uBAAuB,OAAO;EACnC,KAAK,yBAAyB,OAAO;CACvC;;;;;;;;;;;;;;;;CAiBA,MAAM,WAAW,UAAqC;EACpD,OAAO,KAAK,qBAAqB,WAAW,QAAQ;CACtD;;;;;;;;;;;CAYA,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,SAAS;EAChE,MAAM,kBAAkB,CAAC;EACzB,MAAM,mBAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GACvC,IAAI,QAAQ,OAAO,KAAA,GAAW;GAC5B,gBAAgB,KAAK,CAAC;GACtB,iBAAiB,KAAK,UAAU,EAAE;EACpC;EAEF,IAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,gBAAgB;GACjE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,EAAE,CAC/C;GACA,MAAM,KAAK,uBAAuB,KAAK,aAAa;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,GAC/C,QAAQ,gBAAgB,MAAM,eAAe;EAEjD;EACA,OAAO;CACT;;;;;;;;;CAUA,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,YAAY;EAChC,MAAM,UAAU,IAAI,YAAY;EAChC,MAAM,qBAAqB,IAAIC,+BAAAA,mBAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,OAAA,GAAA,2BAAA,OAAA,CAAa,GAAG;GAC5D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,KAAK,CAAC;GAChE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,eAAe,CAAC;EAC9C,CAAC;EACD,OAAO,IAAI,KAAK;GACd;GACA,wBAAwB;EAC1B,CAAC;CACH;AACF"}
+{"version":3,"file":"cache_backed.cjs","names":["Embeddings","EncoderBackedStore","sha256"],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2CA,2BAAAA,WAAW;CACpD;CAEA;CAEA,YAAY,QAAqC;EAC/C,MAAM,MAAM;EACZ,KAAK,uBAAuB,OAAO;EACnC,KAAK,yBAAyB,OAAO;CACvC;;;;;;;;;;;;;;;;CAiBA,MAAM,WAAW,UAAqC;EACpD,OAAO,KAAK,qBAAqB,WAAW,QAAQ;CACtD;;;;;;;;;;;CAYA,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,SAAS;EAChE,MAAM,kBAAkB,CAAC;EACzB,MAAM,mBAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GACvC,IAAI,QAAQ,OAAO,KAAA,GAAW;GAC5B,gBAAgB,KAAK,CAAC;GACtB,iBAAiB,KAAK,UAAU,EAAE;EACpC;EAEF,IAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,gBAAgB;GACjE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,EAAE,CAC/C;GACA,MAAM,KAAK,uBAAuB,KAAK,aAAa;GACpD,KAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,GAC/C,QAAQ,gBAAgB,MAAM,eAAe;EAEjD;EACA,OAAO;CACT;;;;;;;;;CAUA,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,YAAY;EAChC,MAAM,UAAU,IAAI,YAAY;EAChC,MAAM,qBAAqB,IAAIC,+BAAAA,mBAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,OAAA,GAAMC,2BAAAA,OAAAA,CAAO,GAAG;GAC5D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,KAAK,CAAC;GAChE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,eAAe,CAAC;EAC9C,CAAC;EACD,OAAO,IAAI,KAAK;GACd;GACA,wBAAwB;EAC1B,CAAC;CACH;AACF"}

package/dist/evaluation/comparison/pairwise.cjs

@@ -105,7 +105,8 @@ To use references, use the LabeledPairwiseStringEvalChain instead.`;
 			"input",
 			"criteria"
 		]);
-		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 	/**
@@ -159,7 +160,8 @@ var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
 			"reference",
 			"criteria"
 		]);
-		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 };

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAKA,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAY;CAEvD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI;EACJ,IAAI;EAEJ,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,UAAU,QAAQ,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,MAAM,EAAE;EACrD,IAAI,CAAC;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,SAAS,OAAO,GACnC,MAAM,IAAI,MACR,oBAAoB,QAAQ,2CAE9B;EAGF,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACL,EAAE;EAEF,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MAAM,8CAA8C;EAGhE,OAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACF,CAAC;CACH;AACF;;;;;AAMA,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,iCAAiC;CAEpD,OAAO,wBACL,UACwB;EACxB,IAAI,aAAa,KAAA,GAQf,OAAO;GANL;GACA;GACA;GACA;EAGmB,CAAC,CAAC,QACpB,aAAqC,iBAAiB;GACrD,YAAY,gBAAgB,mBAAmB;GAC/C,OAAO;EACT,GACA,CAAC,CACH;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoBC,iCAAAA,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;CAEA,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACC,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,MAAM;EAE5D,MAAM,YAAY,KAAK,wBAAwB,QAAQ;EACvD,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EACZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;CAEA,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAY,GAAG,MAAM;EAElE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;;;AAOA,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACF,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}
+{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAKA,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAY;CAEvD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI;EACJ,IAAI;EAEJ,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,UAAU,QAAQ,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,MAAM,EAAE;EACrD,IAAI,CAAC;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,SAAS,OAAO,GACnC,MAAM,IAAI,MACR,oBAAoB,QAAQ,2CAE9B;EAGF,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACL,EAAE;EAEF,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MAAM,8CAA8C;EAGhE,OAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACF,CAAC;CACH;AACF;;;;;AAMA,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,iCAAiC;CAEpD,OAAO,wBACL,UACwB;EACxB,IAAI,aAAa,KAAA,GAQf,OAAO;GANL;GACA;GACA;GACA;EAGmB,CAAC,CAAC,QACpB,aAAqC,iBAAiB;GACrD,YAAY,gBAAgB,mBAAmB;GAC/C,OAAO;EACT,GACA,CAAC,CACH;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoBC,iCAAAA,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;CAEA,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAACC,aAAAA,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,MAAM;EAE5D,MAAM,YAAY,KAAK,wBAAwB,QAAQ;EACvD,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EACZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;CAEA,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAY,GAAG,MAAM;EAElE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;;;AAOA,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAACF,aAAAA,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}

package/dist/evaluation/comparison/pairwise.js

@@ -105,7 +105,8 @@ To use references, use the LabeledPairwiseStringEvalChain instead.`;
 			"input",
 			"criteria"
 		]);
-		if (!eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 	/**
@@ -159,7 +160,8 @@ var LabeledPairwiseStringEvalChain = class extends PairwiseStringEvalChain {
 			"reference",
 			"criteria"
 		]);
-		if (!eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 };

package/dist/evaluation/comparison/pairwise.js.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.js","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAKA,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAY;CAEvD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI;EACJ,IAAI;EAEJ,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,UAAU,QAAQ,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,MAAM,EAAE;EACrD,IAAI,CAAC;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,SAAS,OAAO,GACnC,MAAM,IAAI,MACR,oBAAoB,QAAQ,2CAE9B;EAGF,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACL,EAAE;EAEF,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MAAM,8CAA8C;EAGhE,OAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACF,CAAC;CACH;AACF;;;;;AAMA,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,iCAAiC;CAEpD,OAAO,wBACL,UACwB;EACxB,IAAI,aAAa,KAAA,GAQf,OAAO;GANL;GACA;GACA;GACA;EAGmB,CAAC,CAAC,QACpB,aAAqC,iBAAiB;GACrD,YAAY,gBAAgB,mBAAmB;GAC/C,OAAO;EACT,GACA,CAAC,CACH;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;CAEA,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAAC,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,MAAM;EAE5D,MAAM,YAAY,KAAK,wBAAwB,QAAQ;EACvD,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EACZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;EAE3B,OAAO;CACT;CAEA,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAY,GAAG,MAAM;EAElE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;;;AAOA,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAAC,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}
+{"version":3,"file":"pairwise.js","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAKA,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,eAAe;EAAC;EAAa;EAAc;CAAY;CAEvD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI;EACJ,IAAI;EAEJ,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GAEL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,UAAU,QAAQ,QAAQ,OAAO,EAAE,CAAC,CAAC,QAAQ,MAAM,EAAE;EACrD,IAAI,CAAC;GAAC;GAAK;GAAK;EAAG,CAAC,CAAC,SAAS,OAAO,GACnC,MAAM,IAAI,MACR,oBAAoB,QAAQ,2CAE9B;EAGF,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;EACL,EAAE;EAEF,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,MAAM,8CAA8C;EAGhE,OAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;EACF,CAAC;CACH;AACF;;;;;AAMA,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,iCAAiC;CAEpD,OAAO,wBACL,UACwB;EACxB,IAAI,aAAa,KAAA,GAQf,OAAO;GANL;GACA;GACA;GACA;EAGmB,CAAC,CAAC,QACpB,aAAqC,iBAAiB;GACrD,YAAY,gBAAgB,mBAAmB;GAC/C,OAAO;EACT,GACA,CAAC,CACH;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;CAEA,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAAC,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,MAAM;EAE5D,MAAM,YAAY,KAAK,wBAAwB,QAAQ;EACvD,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EACZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;EAE3B,OAAO;CACT;CAEA,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;EAAY,GAAG,MAAM;EAElE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;;;AAOA,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAAC,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}

package/dist/evaluation/criteria/criteria.cjs

@@ -87,7 +87,8 @@ var CriteriaEvalChain = class extends require_base.LLMStringEvaluator {
 			"output",
 			"criteria"
 		]);
-		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 	/**
@@ -148,7 +149,8 @@ var LabeledCriteriaEvalChain = class extends CriteriaEvalChain {
 			"criteria",
 			"reference"
 		]);
-		if (!require_base.eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!require_base.eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 };

package/dist/evaluation/criteria/criteria.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"criteria.cjs","names":["BaseLLMOutputParser","LLMStringEvaluator","ConstitutionalPrinciple","CRITERIA_PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAUA,IAAa,6BAAb,cAAgDA,+BAAAA,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI,YAAY;EAChB,IAAI,UAAU;EAEd,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GACL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,IAAI,QAAQ;EAEZ,IAAI,QAAQ,YAAY,MAAM,KAC5B,QAAQ;OACH,IAAI,QAAQ,YAAY,MAAM,KACnC,QAAQ;EAGV,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;EACF,CAAC;CACH;AACF;AAQA,IAAa,oBAAb,cAAuCC,aAAAA,mBAAmB;CACxD,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,2BAA2B;;;;;;;;;;CAWjC,OAAO,gBAAgB,UAAiD;EACtE,IAAI,aAAa,KAAA,GACf,OAAO,EACL,aAAa,mBAAmB,YAClC;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoBC,iCAAAA,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;;;;;CAMA,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACC,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,KAAK,SAAS,uBAAuB,aAAa,eACpD,MAAM,IAAI,MACR,gLAIF;EAGF,IAAI,SAAS,KAAK,cAAc,cAAc,MAAM;EAEpD,MAAM,YAAY,KAAK,gBAAgB,QAAQ;EAC/C,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EAEZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;EACV;EACA,IAAI,KAAK,mBACP,UAAU,YAAY;EAExB,OAAO;CACT;;;;;CAMA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;CAEA,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,IAAI,EAAE,GAAG,MAAM;EAErE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;AAKA,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;EACvB,OAAO;CACT;CAGA,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAACF,aAAAA,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}
+{"version":3,"file":"criteria.cjs","names":["BaseLLMOutputParser","LLMStringEvaluator","ConstitutionalPrinciple","CRITERIA_PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAUA,IAAa,6BAAb,cAAgDA,+BAAAA,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI,YAAY;EAChB,IAAI,UAAU;EAEd,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GACL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,IAAI,QAAQ;EAEZ,IAAI,QAAQ,YAAY,MAAM,KAC5B,QAAQ;OACH,IAAI,QAAQ,YAAY,MAAM,KACnC,QAAQ;EAGV,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;EACF,CAAC;CACH;AACF;AAQA,IAAa,oBAAb,cAAuCC,aAAAA,mBAAmB;CACxD,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,2BAA2B;;;;;;;;;;CAWjC,OAAO,gBAAgB,UAAiD;EACtE,IAAI,aAAa,KAAA,GACf,OAAO,EACL,aAAa,mBAAmB,YAClC;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoBC,iCAAAA,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;;;;;CAMA,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAACC,aAAAA,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,KAAK,SAAS,uBAAuB,aAAa,eACpD,MAAM,IAAI,MACR,gLAIF;EAGF,IAAI,SAAS,KAAK,cAAc,cAAc,MAAM;EAEpD,MAAM,YAAY,KAAK,gBAAgB,QAAQ;EAC/C,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EAEZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;EACV;EACA,IAAI,KAAK,mBACP,UAAU,YAAY;EAExB,OAAO;CACT;;;;;CAMA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,UAC9B,OAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;EAE3B,OAAO;CACT;CAEA,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,IAAI,EAAE,GAAG,MAAM;EAErE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;AAKA,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;EACvB,OAAO;CACT;CAGA,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAACF,aAAAA,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}

package/dist/evaluation/criteria/criteria.js

@@ -87,7 +87,8 @@ var CriteriaEvalChain = class extends LLMStringEvaluator {
 			"output",
 			"criteria"
 		]);
-		if (!eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 	/**
@@ -148,7 +149,8 @@ var LabeledCriteriaEvalChain = class extends CriteriaEvalChain {
 			"criteria",
 			"reference"
 		]);
-		if (!eqSet(expectedInputVars, new Set(_prompt.inputVariables))) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
+		const inputVarsSet = new Set(_prompt.inputVariables);
+		if (!eqSet(expectedInputVars, inputVarsSet)) throw new Error(`Input variables should be ${[...expectedInputVars]}, but got ${_prompt.inputVariables}`);
 		return _prompt;
 	}
 };

package/dist/evaluation/criteria/criteria.js.map

@@ -1 +1 @@
-{"version":3,"file":"criteria.js","names":[],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAUA,IAAa,6BAAb,cAAgD,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI,YAAY;EAChB,IAAI,UAAU;EAEd,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GACL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,IAAI,QAAQ;EAEZ,IAAI,QAAQ,YAAY,MAAM,KAC5B,QAAQ;OACH,IAAI,QAAQ,YAAY,MAAM,KACnC,QAAQ;EAGV,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;EACF,CAAC;CACH;AACF;AAQA,IAAa,oBAAb,cAAuC,mBAAmB;CACxD,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,2BAA2B;;;;;;;;;;CAWjC,OAAO,gBAAgB,UAAiD;EACtE,IAAI,aAAa,KAAA,GACf,OAAO,EACL,aAAa,mBAAmB,YAClC;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;;;;;CAMA,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACF,CAAC;EAID,IAAI,CAAC,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,KAAK,SAAS,uBAAuB,aAAa,eACpD,MAAM,IAAI,MACR,gLAIF;EAGF,IAAI,SAAS,KAAK,cAAc,cAAc,MAAM;EAEpD,MAAM,YAAY,KAAK,gBAAgB,QAAQ;EAC/C,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EAEZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;EACV;EACA,IAAI,KAAK,mBACP,UAAU,YAAY;EAExB,OAAO;CACT;;;;;CAMA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;EAE3B,OAAO;CACT;CAEA,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,IAAI,EAAE,GAAG,MAAM;EAErE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;AAKA,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;EACvB,OAAO;CACT;CAGA,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAID,IAAI,CAAC,MAAM,mBAAmB,IAFQ,IAAI,QAAQ,cAET,CAAC,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}
+{"version":3,"file":"criteria.js","names":[],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;AACV;;;;AAUA,IAAa,6BAAb,cAAgD,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,KAAK,CAAC,CAAC,MAAM,IAAI;EACrC,IAAI,YAAY;EAChB,IAAI,UAAU;EAEd,IAAI,OAAO,WAAW,GACpB,CAAC,WAAW;OACP;GACL,YAAY,OAAO,MAAM,GAAG,OAAO,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE;GACtD,UAAU,OAAO,OAAO,SAAS;EACnC;EAEA,IAAI,QAAQ;EAEZ,IAAI,QAAQ,YAAY,MAAM,KAC5B,QAAQ;OACH,IAAI,QAAQ,YAAY,MAAM,KACnC,QAAQ;EAGV,OAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;EACF,CAAC;CACH;AACF;AAQA,IAAa,oBAAb,cAAuC,mBAAmB;CACxD,OAAO,UAAkB;EACvB,OAAO;CACT;CAEA;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,2BAA2B;;;;;;;;;;CAWjC,OAAO,gBAAgB,UAAiD;EACtE,IAAI,aAAa,KAAA,GACf,OAAO,EACL,aAAa,mBAAmB,YAClC;EAGF,IAAI,YAAuC,CAAC;EAE5C,IAAI,OAAO,aAAa,UAClB;OAAA,YAAY,oBACd,YAAY,GAAG,WAAW,mBAAmB,UAAU;EAAA,OAGpD,IAAI,oBAAoB,yBAC7B,YAAY,GAAG,SAAS,OAAO,SAAS,gBAAgB;OACnD;GACL,IAAI,CAAC,UACH,MAAM,IAAI,MACR,kHAGF;GAEF,YAAY,EAAE,GAAG,SAAS;EAC5B;EACA,OAAO;CACT;;;;;CAMA,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAAC,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;;;;;;;CAQA,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,KAAK,SAAS,uBAAuB,aAAa,eACpD,MAAM,IAAI,MACR,gLAIF;EAGF,IAAI,SAAS,KAAK,cAAc,cAAc,MAAM;EAEpD,MAAM,YAAY,KAAK,gBAAgB,QAAQ;EAC/C,MAAM,cAAc,OAAO,QAAQ,SAAS,CAAC,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,GAAG,CAAC,CAC7B,KAAK,IAAI;EAEZ,SAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,YAAY,CAAC;EAEvD,MAAM,UAAU;EAChB,IAAI,SAEF,OAAO,QAAQ;EAGjB,OAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;EACL,CAAC;CACH;CAEA,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;EACV;EACA,IAAI,KAAK,mBACP,UAAU,YAAY;EAExB,OAAO;CACT;;;;;CAMA,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;EAC3B,IAAI,WAAW,UAAU,OAAO,UAC9B,OAAO,WAAW,OAAO;EAE3B,OAAO;CACT;CAEA,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,IAAI,EAAE,GAAG,MAAM;EAErE,OAAO,KAAK,eAAe,MAAM;CACnC;AACF;;;;AAKA,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;EACvB,OAAO;CACT;CAGA,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAU;EAC1B,MAAM,oCAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;EACF,CAAC;EAED,MAAM,eAA4B,IAAI,IAAI,QAAQ,cAAc;EAEhE,IAAI,CAAC,MAAM,mBAAmB,YAAY,GACxC,MAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,iBAAiB,EAAE,YAClD,QAAQ,gBAEZ;EAEF,OAAO;CACT;AACF"}

package/dist/experimental/autogpt/agent.cjs

@@ -71,15 +71,16 @@ var AutoGPT = class AutoGPT {
 	* @returns A new instance of the AutoGPT class.
 	*/
 	static fromLLMAndTools(llm, tools, { aiName, aiRole, memory, maxIterations = 100, outputParser = new require_output_parser.AutoGPTOutputParser() }) {
+		const prompt = new require_prompt.AutoGPTPrompt({
+			aiName,
+			aiRole,
+			tools,
+			tokenCounter: llm.getNumTokens.bind(llm),
+			sendTokenLimit: (0, _langchain_core_language_models_base.getModelContextSize)("modelName" in llm ? llm.modelName : "gpt2")
+		});
 		const chain = new require_llm_chain.LLMChain({
 			llm,
-			prompt: new require_prompt.AutoGPTPrompt({
-				aiName,
-				aiRole,
-				tools,
-				tokenCounter: llm.getNumTokens.bind(llm),
-				sendTokenLimit: (0, _langchain_core_language_models_base.getModelContextSize)("modelName" in llm ? llm.modelName : "gpt2")
-			})
+			prompt
 		});
 		return new AutoGPT({
 			aiName,

package/dist/experimental/autogpt/agent.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent.cjs","names":["AutoGPTOutputParser","LLMChain","AutoGPTPrompt","HumanMessage","AIMessage","SystemMessage"],"sources":["../../../src/experimental/autogpt/agent.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport {\n  AIMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  getEmbeddingContextSize,\n  getModelContextSize,\n} from \"@langchain/core/language_models/base\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\n\nimport { AutoGPTOutputParser } from \"./output_parser.js\";\nimport { AutoGPTPrompt } from \"./prompt.js\";\n// import { HumanInputRun } from \"./tools/human/tool\"; // TODO\nimport { ObjectTool, FINISH_NAME } from \"./schema.js\";\nimport { TokenTextSplitter } from \"../../text_splitter.js\";\n\n/**\n * Interface for the input parameters of the AutoGPT class.\n */\nexport interface AutoGPTInput {\n  aiName: string;\n  aiRole: string;\n  memory: VectorStoreRetrieverInterface;\n  humanInTheLoop?: boolean;\n  outputParser?: AutoGPTOutputParser;\n  maxIterations?: number;\n}\n\n/**\n * Class representing the AutoGPT concept with LangChain primitives. It is\n * designed to be used with a set of tools such as a search tool,\n * write-file tool, and a read-file tool.\n * @example\n * ```typescript\n * const autogpt = AutoGPT.fromLLMAndTools(\n *   new ChatOpenAI({ model: \"gpt-4o-mini\", temperature: 0 }),\n *   [\n *     new ReadFileTool({ store: new InMemoryFileStore() }),\n *     new WriteFileTool({ store: new InMemoryFileStore() }),\n *     new SerpAPI(\"YOUR_SERPAPI_API_KEY\", {\n *       location: \"San Francisco,California,United States\",\n *       hl: \"en\",\n *       gl: \"us\",\n *     }),\n *   ],\n *   {\n *     memory: new MemoryVectorStore(new OpenAIEmbeddings()).asRetriever(),\n *     aiName: \"Tom\",\n *     aiRole: \"Assistant\",\n *   },\n * );\n * const result = await autogpt.run([\"write a weather report for SF today\"]);\n * ```\n */\nexport class AutoGPT {\n  aiName: string;\n\n  memory: VectorStoreRetrieverInterface;\n\n  fullMessageHistory: BaseMessage[];\n\n  nextActionCount: number;\n\n  chain: LLMChain;\n\n  outputParser: AutoGPTOutputParser;\n\n  tools: ObjectTool[];\n\n  feedbackTool?: Tool;\n\n  maxIterations: number;\n\n  // Currently not generic enough to support any text splitter.\n  textSplitter: TokenTextSplitter;\n\n  constructor({\n    aiName,\n    memory,\n    chain,\n    outputParser,\n    tools,\n    feedbackTool,\n    maxIterations,\n  }: Omit<Required<AutoGPTInput>, \"aiRole\" | \"humanInTheLoop\"> & {\n    chain: LLMChain;\n    tools: ObjectTool[];\n    feedbackTool?: Tool;\n  }) {\n    this.aiName = aiName;\n    this.memory = memory;\n    this.fullMessageHistory = [];\n    this.nextActionCount = 0;\n    this.chain = chain;\n    this.outputParser = outputParser;\n    this.tools = tools;\n    this.feedbackTool = feedbackTool;\n    this.maxIterations = maxIterations;\n    const chunkSize = getEmbeddingContextSize(\n      \"modelName\" in memory.vectorStore.embeddings\n        ? (memory.vectorStore.embeddings.modelName as string)\n        : undefined\n    );\n    this.textSplitter = new TokenTextSplitter({\n      chunkSize,\n      chunkOverlap: Math.round(chunkSize / 10),\n    });\n  }\n\n  /**\n   * Creates a new AutoGPT instance from a given LLM and a set of tools.\n   * @param llm A BaseChatModel object.\n   * @param tools An array of ObjectTool objects.\n   * @param options.aiName The name of the AI.\n   * @param options.aiRole The role of the AI.\n   * @param options.memory A VectorStoreRetriever object that represents the memory of the AI.\n   * @param options.maxIterations The maximum number of iterations the AI can perform.\n   * @param options.outputParser An AutoGPTOutputParser object that parses the output of the AI.\n   * @returns A new instance of the AutoGPT class.\n   */\n  static fromLLMAndTools(\n    llm: BaseChatModel,\n    tools: ObjectTool[],\n    {\n      aiName,\n      aiRole,\n      memory,\n      maxIterations = 100,\n      // humanInTheLoop = false,\n      outputParser = new AutoGPTOutputParser(),\n    }: AutoGPTInput\n  ): AutoGPT {\n    const prompt = new AutoGPTPrompt({\n      aiName,\n      aiRole,\n      tools,\n      tokenCounter: llm.getNumTokens.bind(llm),\n      sendTokenLimit: getModelContextSize(\n        \"modelName\" in llm ? (llm.modelName as string) : \"gpt2\"\n      ),\n    });\n    // const feedbackTool = humanInTheLoop ? new HumanInputRun() : null;\n    const chain = new LLMChain({ llm, prompt });\n    return new AutoGPT({\n      aiName,\n      memory,\n      chain,\n      outputParser,\n      tools,\n      // feedbackTool,\n      maxIterations,\n    });\n  }\n\n  /**\n   * Runs the AI with a given set of goals.\n   * @param goals An array of strings representing the goals.\n   * @returns A string representing the result of the run or undefined if the maximum number of iterations is reached without a result.\n   */\n  async run(goals: string[]): Promise<string | undefined> {\n    const user_input =\n      \"Determine which next command to use, and respond using the format specified above:\";\n    let loopCount = 0;\n    while (loopCount < this.maxIterations) {\n      loopCount += 1;\n\n      const { text: assistantReply } = await this.chain.call({\n        goals,\n        user_input,\n        memory: this.memory,\n        messages: this.fullMessageHistory,\n      });\n\n      // Print the assistant reply\n      console.log(assistantReply);\n      this.fullMessageHistory.push(new HumanMessage(user_input));\n      this.fullMessageHistory.push(new AIMessage(assistantReply));\n\n      const action = await this.outputParser.parse(assistantReply);\n      const tools = this.tools.reduce(\n        (acc, tool) => ({ ...acc, [tool.name]: tool }),\n        {} as { [key: string]: ObjectTool }\n      );\n      if (action.name === FINISH_NAME) {\n        return action.args.response;\n      }\n      let result: string;\n      if (action.name in tools) {\n        const tool = tools[action.name];\n        let observation;\n        try {\n          observation = await tool.call(action.args);\n        } catch (e) {\n          observation = `Error in args: ${e}`;\n        }\n        result = `Command ${tool.name} returned: ${observation}`;\n      } else if (action.name === \"ERROR\") {\n        result = `Error: ${action.args}. `;\n      } else {\n        result = `Unknown command '${action.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format.`;\n      }\n\n      let memoryToAdd = `Assistant Reply: ${assistantReply}\\nResult: ${result} `;\n      if (this.feedbackTool) {\n        const feedback = `\\n${await this.feedbackTool.call(\"Input: \")}`;\n        if (feedback === \"q\" || feedback === \"stop\") {\n          console.log(\"EXITING\");\n          return \"EXITING\";\n        }\n        memoryToAdd += feedback;\n      }\n\n      const documents = await this.textSplitter.createDocuments([memoryToAdd]);\n      await this.memory.addDocuments(documents);\n      this.fullMessageHistory.push(new SystemMessage(result));\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,IAAa,UAAb,MAAa,QAAQ;CACnB;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA,YAAY,EACV,QACA,QACA,OACA,cACA,OACA,cACA,iBAKC;EACD,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,qBAAqB,CAAC;EAC3B,KAAK,kBAAkB;EACvB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,gBAAgB;EACrB,MAAM,aAAA,GAAA,qCAAA,wBAAA,CACJ,eAAe,OAAO,YAAY,aAC7B,OAAO,YAAY,WAAW,YAC/B,KAAA,CACN;EACA,KAAK,eAAe,IAAA,sBAAA,sBAAI,kBAAkB;GACxC;GACA,cAAc,KAAK,MAAM,YAAY,EAAE;EACzC,CAAC;CACH;;;;;;;;;;;;CAaA,OAAO,gBACL,KACA,OACA,EACE,QACA,QACA,QACA,gBAAgB,KAEhB,eAAe,IAAIA,sBAAAA,oBAAoB,KAEhC;EAWT,MAAM,QAAQ,IAAIC,kBAAAA,SAAS;GAAE;GAAK,QAAA,IAVfC,eAAAA,cAAc;IAC/B;IACA;IACA;IACA,cAAc,IAAI,aAAa,KAAK,GAAG;IACvC,iBAAA,GAAA,qCAAA,oBAAA,CACE,eAAe,MAAO,IAAI,YAAuB,MACnD;GACF,CAEuC;EAAE,CAAC;EAC1C,OAAO,IAAI,QAAQ;GACjB;GACA;GACA;GACA;GACA;GAEA;EACF,CAAC;CACH;;;;;;CAOA,MAAM,IAAI,OAA8C;EACtD,MAAM,aACJ;EACF,IAAI,YAAY;EAChB,OAAO,YAAY,KAAK,eAAe;GACrC,aAAa;GAEb,MAAM,EAAE,MAAM,mBAAmB,MAAM,KAAK,MAAM,KAAK;IACrD;IACA;IACA,QAAQ,KAAK;IACb,UAAU,KAAK;GACjB,CAAC;GAGD,QAAQ,IAAI,cAAc;GAC1B,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,aAAa,UAAU,CAAC;GACzD,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,UAAU,cAAc,CAAC;GAE1D,MAAM,SAAS,MAAM,KAAK,aAAa,MAAM,cAAc;GAC3D,MAAM,QAAQ,KAAK,MAAM,QACtB,KAAK,UAAU;IAAE,GAAG;KAAM,KAAK,OAAO;GAAK,IAC5C,CAAC,CACH;GACA,IAAI,OAAO,SAAA,UACT,OAAO,OAAO,KAAK;GAErB,IAAI;GACJ,IAAI,OAAO,QAAQ,OAAO;IACxB,MAAM,OAAO,MAAM,OAAO;IAC1B,IAAI;IACJ,IAAI;KACF,cAAc,MAAM,KAAK,KAAK,OAAO,IAAI;IAC3C,SAAS,GAAG;KACV,cAAc,kBAAkB;IAClC;IACA,SAAS,WAAW,KAAK,KAAK,aAAa;GAC7C,OAAO,IAAI,OAAO,SAAS,SACzB,SAAS,UAAU,OAAO,KAAK;QAE/B,SAAS,oBAAoB,OAAO,KAAK;GAG3C,IAAI,cAAc,oBAAoB,eAAe,YAAY,OAAO;GACxE,IAAI,KAAK,cAAc;IACrB,MAAM,WAAW,KAAK,MAAM,KAAK,aAAa,KAAK,SAAS;IAC5D,IAAI,aAAa,OAAO,aAAa,QAAQ;KAC3C,QAAQ,IAAI,SAAS;KACrB,OAAO;IACT;IACA,eAAe;GACjB;GAEA,MAAM,YAAY,MAAM,KAAK,aAAa,gBAAgB,CAAC,WAAW,CAAC;GACvE,MAAM,KAAK,OAAO,aAAa,SAAS;GACxC,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,cAAc,MAAM,CAAC;EACxD;CAGF;AACF"}
+{"version":3,"file":"agent.cjs","names":["getEmbeddingContextSize","AutoGPTOutputParser","AutoGPTPrompt","getModelContextSize","LLMChain","HumanMessage","AIMessage","SystemMessage"],"sources":["../../../src/experimental/autogpt/agent.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport {\n  AIMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  getEmbeddingContextSize,\n  getModelContextSize,\n} from \"@langchain/core/language_models/base\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\n\nimport { AutoGPTOutputParser } from \"./output_parser.js\";\nimport { AutoGPTPrompt } from \"./prompt.js\";\n// import { HumanInputRun } from \"./tools/human/tool\"; // TODO\nimport { ObjectTool, FINISH_NAME } from \"./schema.js\";\nimport { TokenTextSplitter } from \"../../text_splitter.js\";\n\n/**\n * Interface for the input parameters of the AutoGPT class.\n */\nexport interface AutoGPTInput {\n  aiName: string;\n  aiRole: string;\n  memory: VectorStoreRetrieverInterface;\n  humanInTheLoop?: boolean;\n  outputParser?: AutoGPTOutputParser;\n  maxIterations?: number;\n}\n\n/**\n * Class representing the AutoGPT concept with LangChain primitives. It is\n * designed to be used with a set of tools such as a search tool,\n * write-file tool, and a read-file tool.\n * @example\n * ```typescript\n * const autogpt = AutoGPT.fromLLMAndTools(\n *   new ChatOpenAI({ model: \"gpt-4o-mini\", temperature: 0 }),\n *   [\n *     new ReadFileTool({ store: new InMemoryFileStore() }),\n *     new WriteFileTool({ store: new InMemoryFileStore() }),\n *     new SerpAPI(\"YOUR_SERPAPI_API_KEY\", {\n *       location: \"San Francisco,California,United States\",\n *       hl: \"en\",\n *       gl: \"us\",\n *     }),\n *   ],\n *   {\n *     memory: new MemoryVectorStore(new OpenAIEmbeddings()).asRetriever(),\n *     aiName: \"Tom\",\n *     aiRole: \"Assistant\",\n *   },\n * );\n * const result = await autogpt.run([\"write a weather report for SF today\"]);\n * ```\n */\nexport class AutoGPT {\n  aiName: string;\n\n  memory: VectorStoreRetrieverInterface;\n\n  fullMessageHistory: BaseMessage[];\n\n  nextActionCount: number;\n\n  chain: LLMChain;\n\n  outputParser: AutoGPTOutputParser;\n\n  tools: ObjectTool[];\n\n  feedbackTool?: Tool;\n\n  maxIterations: number;\n\n  // Currently not generic enough to support any text splitter.\n  textSplitter: TokenTextSplitter;\n\n  constructor({\n    aiName,\n    memory,\n    chain,\n    outputParser,\n    tools,\n    feedbackTool,\n    maxIterations,\n  }: Omit<Required<AutoGPTInput>, \"aiRole\" | \"humanInTheLoop\"> & {\n    chain: LLMChain;\n    tools: ObjectTool[];\n    feedbackTool?: Tool;\n  }) {\n    this.aiName = aiName;\n    this.memory = memory;\n    this.fullMessageHistory = [];\n    this.nextActionCount = 0;\n    this.chain = chain;\n    this.outputParser = outputParser;\n    this.tools = tools;\n    this.feedbackTool = feedbackTool;\n    this.maxIterations = maxIterations;\n    const chunkSize = getEmbeddingContextSize(\n      \"modelName\" in memory.vectorStore.embeddings\n        ? (memory.vectorStore.embeddings.modelName as string)\n        : undefined\n    );\n    this.textSplitter = new TokenTextSplitter({\n      chunkSize,\n      chunkOverlap: Math.round(chunkSize / 10),\n    });\n  }\n\n  /**\n   * Creates a new AutoGPT instance from a given LLM and a set of tools.\n   * @param llm A BaseChatModel object.\n   * @param tools An array of ObjectTool objects.\n   * @param options.aiName The name of the AI.\n   * @param options.aiRole The role of the AI.\n   * @param options.memory A VectorStoreRetriever object that represents the memory of the AI.\n   * @param options.maxIterations The maximum number of iterations the AI can perform.\n   * @param options.outputParser An AutoGPTOutputParser object that parses the output of the AI.\n   * @returns A new instance of the AutoGPT class.\n   */\n  static fromLLMAndTools(\n    llm: BaseChatModel,\n    tools: ObjectTool[],\n    {\n      aiName,\n      aiRole,\n      memory,\n      maxIterations = 100,\n      // humanInTheLoop = false,\n      outputParser = new AutoGPTOutputParser(),\n    }: AutoGPTInput\n  ): AutoGPT {\n    const prompt = new AutoGPTPrompt({\n      aiName,\n      aiRole,\n      tools,\n      tokenCounter: llm.getNumTokens.bind(llm),\n      sendTokenLimit: getModelContextSize(\n        \"modelName\" in llm ? (llm.modelName as string) : \"gpt2\"\n      ),\n    });\n    // const feedbackTool = humanInTheLoop ? new HumanInputRun() : null;\n    const chain = new LLMChain({ llm, prompt });\n    return new AutoGPT({\n      aiName,\n      memory,\n      chain,\n      outputParser,\n      tools,\n      // feedbackTool,\n      maxIterations,\n    });\n  }\n\n  /**\n   * Runs the AI with a given set of goals.\n   * @param goals An array of strings representing the goals.\n   * @returns A string representing the result of the run or undefined if the maximum number of iterations is reached without a result.\n   */\n  async run(goals: string[]): Promise<string | undefined> {\n    const user_input =\n      \"Determine which next command to use, and respond using the format specified above:\";\n    let loopCount = 0;\n    while (loopCount < this.maxIterations) {\n      loopCount += 1;\n\n      const { text: assistantReply } = await this.chain.call({\n        goals,\n        user_input,\n        memory: this.memory,\n        messages: this.fullMessageHistory,\n      });\n\n      // Print the assistant reply\n      console.log(assistantReply);\n      this.fullMessageHistory.push(new HumanMessage(user_input));\n      this.fullMessageHistory.push(new AIMessage(assistantReply));\n\n      const action = await this.outputParser.parse(assistantReply);\n      const tools = this.tools.reduce(\n        (acc, tool) => ({ ...acc, [tool.name]: tool }),\n        {} as { [key: string]: ObjectTool }\n      );\n      if (action.name === FINISH_NAME) {\n        return action.args.response;\n      }\n      let result: string;\n      if (action.name in tools) {\n        const tool = tools[action.name];\n        let observation;\n        try {\n          observation = await tool.call(action.args);\n        } catch (e) {\n          observation = `Error in args: ${e}`;\n        }\n        result = `Command ${tool.name} returned: ${observation}`;\n      } else if (action.name === \"ERROR\") {\n        result = `Error: ${action.args}. `;\n      } else {\n        result = `Unknown command '${action.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format.`;\n      }\n\n      let memoryToAdd = `Assistant Reply: ${assistantReply}\\nResult: ${result} `;\n      if (this.feedbackTool) {\n        const feedback = `\\n${await this.feedbackTool.call(\"Input: \")}`;\n        if (feedback === \"q\" || feedback === \"stop\") {\n          console.log(\"EXITING\");\n          return \"EXITING\";\n        }\n        memoryToAdd += feedback;\n      }\n\n      const documents = await this.textSplitter.createDocuments([memoryToAdd]);\n      await this.memory.addDocuments(documents);\n      this.fullMessageHistory.push(new SystemMessage(result));\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,IAAa,UAAb,MAAa,QAAQ;CACnB;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA,YAAY,EACV,QACA,QACA,OACA,cACA,OACA,cACA,iBAKC;EACD,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,qBAAqB,CAAC;EAC3B,KAAK,kBAAkB;EACvB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,gBAAgB;EACrB,MAAM,aAAA,GAAYA,qCAAAA,wBAAAA,CAChB,eAAe,OAAO,YAAY,aAC7B,OAAO,YAAY,WAAW,YAC/B,KAAA,CACN;EACA,KAAK,eAAe,IAAA,sBAAA,sBAAI,kBAAkB;GACxC;GACA,cAAc,KAAK,MAAM,YAAY,EAAE;EACzC,CAAC;CACH;;;;;;;;;;;;CAaA,OAAO,gBACL,KACA,OACA,EACE,QACA,QACA,QACA,gBAAgB,KAEhB,eAAe,IAAIC,sBAAAA,oBAAoB,KAEhC;EACT,MAAM,SAAS,IAAIC,eAAAA,cAAc;GAC/B;GACA;GACA;GACA,cAAc,IAAI,aAAa,KAAK,GAAG;GACvC,iBAAA,GAAgBC,qCAAAA,oBAAAA,CACd,eAAe,MAAO,IAAI,YAAuB,MACnD;EACF,CAAC;EAED,MAAM,QAAQ,IAAIC,kBAAAA,SAAS;GAAE;GAAK;EAAO,CAAC;EAC1C,OAAO,IAAI,QAAQ;GACjB;GACA;GACA;GACA;GACA;GAEA;EACF,CAAC;CACH;;;;;;CAOA,MAAM,IAAI,OAA8C;EACtD,MAAM,aACJ;EACF,IAAI,YAAY;EAChB,OAAO,YAAY,KAAK,eAAe;GACrC,aAAa;GAEb,MAAM,EAAE,MAAM,mBAAmB,MAAM,KAAK,MAAM,KAAK;IACrD;IACA;IACA,QAAQ,KAAK;IACb,UAAU,KAAK;GACjB,CAAC;GAGD,QAAQ,IAAI,cAAc;GAC1B,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,aAAa,UAAU,CAAC;GACzD,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,UAAU,cAAc,CAAC;GAE1D,MAAM,SAAS,MAAM,KAAK,aAAa,MAAM,cAAc;GAC3D,MAAM,QAAQ,KAAK,MAAM,QACtB,KAAK,UAAU;IAAE,GAAG;KAAM,KAAK,OAAO;GAAK,IAC5C,CAAC,CACH;GACA,IAAI,OAAO,SAAA,UACT,OAAO,OAAO,KAAK;GAErB,IAAI;GACJ,IAAI,OAAO,QAAQ,OAAO;IACxB,MAAM,OAAO,MAAM,OAAO;IAC1B,IAAI;IACJ,IAAI;KACF,cAAc,MAAM,KAAK,KAAK,OAAO,IAAI;IAC3C,SAAS,GAAG;KACV,cAAc,kBAAkB;IAClC;IACA,SAAS,WAAW,KAAK,KAAK,aAAa;GAC7C,OAAO,IAAI,OAAO,SAAS,SACzB,SAAS,UAAU,OAAO,KAAK;QAE/B,SAAS,oBAAoB,OAAO,KAAK;GAG3C,IAAI,cAAc,oBAAoB,eAAe,YAAY,OAAO;GACxE,IAAI,KAAK,cAAc;IACrB,MAAM,WAAW,KAAK,MAAM,KAAK,aAAa,KAAK,SAAS;IAC5D,IAAI,aAAa,OAAO,aAAa,QAAQ;KAC3C,QAAQ,IAAI,SAAS;KACrB,OAAO;IACT;IACA,eAAe;GACjB;GAEA,MAAM,YAAY,MAAM,KAAK,aAAa,gBAAgB,CAAC,WAAW,CAAC;GACvE,MAAM,KAAK,OAAO,aAAa,SAAS;GACxC,KAAK,mBAAmB,KAAK,IAAIC,yBAAAA,cAAc,MAAM,CAAC;EACxD;CAGF;AACF"}

package/dist/experimental/autogpt/agent.js

@@ -71,15 +71,16 @@ var AutoGPT = class AutoGPT {
 	* @returns A new instance of the AutoGPT class.
 	*/
 	static fromLLMAndTools(llm, tools, { aiName, aiRole, memory, maxIterations = 100, outputParser = new AutoGPTOutputParser() }) {
+		const prompt = new AutoGPTPrompt({
+			aiName,
+			aiRole,
+			tools,
+			tokenCounter: llm.getNumTokens.bind(llm),
+			sendTokenLimit: getModelContextSize("modelName" in llm ? llm.modelName : "gpt2")
+		});
 		const chain = new LLMChain({
 			llm,
-			prompt: new AutoGPTPrompt({
-				aiName,
-				aiRole,
-				tools,
-				tokenCounter: llm.getNumTokens.bind(llm),
-				sendTokenLimit: getModelContextSize("modelName" in llm ? llm.modelName : "gpt2")
-			})
+			prompt
 		});
 		return new AutoGPT({
 			aiName,

package/dist/experimental/autogpt/agent.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent.js","names":["TokenTextSplitter"],"sources":["../../../src/experimental/autogpt/agent.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport {\n  AIMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  getEmbeddingContextSize,\n  getModelContextSize,\n} from \"@langchain/core/language_models/base\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\n\nimport { AutoGPTOutputParser } from \"./output_parser.js\";\nimport { AutoGPTPrompt } from \"./prompt.js\";\n// import { HumanInputRun } from \"./tools/human/tool\"; // TODO\nimport { ObjectTool, FINISH_NAME } from \"./schema.js\";\nimport { TokenTextSplitter } from \"../../text_splitter.js\";\n\n/**\n * Interface for the input parameters of the AutoGPT class.\n */\nexport interface AutoGPTInput {\n  aiName: string;\n  aiRole: string;\n  memory: VectorStoreRetrieverInterface;\n  humanInTheLoop?: boolean;\n  outputParser?: AutoGPTOutputParser;\n  maxIterations?: number;\n}\n\n/**\n * Class representing the AutoGPT concept with LangChain primitives. It is\n * designed to be used with a set of tools such as a search tool,\n * write-file tool, and a read-file tool.\n * @example\n * ```typescript\n * const autogpt = AutoGPT.fromLLMAndTools(\n *   new ChatOpenAI({ model: \"gpt-4o-mini\", temperature: 0 }),\n *   [\n *     new ReadFileTool({ store: new InMemoryFileStore() }),\n *     new WriteFileTool({ store: new InMemoryFileStore() }),\n *     new SerpAPI(\"YOUR_SERPAPI_API_KEY\", {\n *       location: \"San Francisco,California,United States\",\n *       hl: \"en\",\n *       gl: \"us\",\n *     }),\n *   ],\n *   {\n *     memory: new MemoryVectorStore(new OpenAIEmbeddings()).asRetriever(),\n *     aiName: \"Tom\",\n *     aiRole: \"Assistant\",\n *   },\n * );\n * const result = await autogpt.run([\"write a weather report for SF today\"]);\n * ```\n */\nexport class AutoGPT {\n  aiName: string;\n\n  memory: VectorStoreRetrieverInterface;\n\n  fullMessageHistory: BaseMessage[];\n\n  nextActionCount: number;\n\n  chain: LLMChain;\n\n  outputParser: AutoGPTOutputParser;\n\n  tools: ObjectTool[];\n\n  feedbackTool?: Tool;\n\n  maxIterations: number;\n\n  // Currently not generic enough to support any text splitter.\n  textSplitter: TokenTextSplitter;\n\n  constructor({\n    aiName,\n    memory,\n    chain,\n    outputParser,\n    tools,\n    feedbackTool,\n    maxIterations,\n  }: Omit<Required<AutoGPTInput>, \"aiRole\" | \"humanInTheLoop\"> & {\n    chain: LLMChain;\n    tools: ObjectTool[];\n    feedbackTool?: Tool;\n  }) {\n    this.aiName = aiName;\n    this.memory = memory;\n    this.fullMessageHistory = [];\n    this.nextActionCount = 0;\n    this.chain = chain;\n    this.outputParser = outputParser;\n    this.tools = tools;\n    this.feedbackTool = feedbackTool;\n    this.maxIterations = maxIterations;\n    const chunkSize = getEmbeddingContextSize(\n      \"modelName\" in memory.vectorStore.embeddings\n        ? (memory.vectorStore.embeddings.modelName as string)\n        : undefined\n    );\n    this.textSplitter = new TokenTextSplitter({\n      chunkSize,\n      chunkOverlap: Math.round(chunkSize / 10),\n    });\n  }\n\n  /**\n   * Creates a new AutoGPT instance from a given LLM and a set of tools.\n   * @param llm A BaseChatModel object.\n   * @param tools An array of ObjectTool objects.\n   * @param options.aiName The name of the AI.\n   * @param options.aiRole The role of the AI.\n   * @param options.memory A VectorStoreRetriever object that represents the memory of the AI.\n   * @param options.maxIterations The maximum number of iterations the AI can perform.\n   * @param options.outputParser An AutoGPTOutputParser object that parses the output of the AI.\n   * @returns A new instance of the AutoGPT class.\n   */\n  static fromLLMAndTools(\n    llm: BaseChatModel,\n    tools: ObjectTool[],\n    {\n      aiName,\n      aiRole,\n      memory,\n      maxIterations = 100,\n      // humanInTheLoop = false,\n      outputParser = new AutoGPTOutputParser(),\n    }: AutoGPTInput\n  ): AutoGPT {\n    const prompt = new AutoGPTPrompt({\n      aiName,\n      aiRole,\n      tools,\n      tokenCounter: llm.getNumTokens.bind(llm),\n      sendTokenLimit: getModelContextSize(\n        \"modelName\" in llm ? (llm.modelName as string) : \"gpt2\"\n      ),\n    });\n    // const feedbackTool = humanInTheLoop ? new HumanInputRun() : null;\n    const chain = new LLMChain({ llm, prompt });\n    return new AutoGPT({\n      aiName,\n      memory,\n      chain,\n      outputParser,\n      tools,\n      // feedbackTool,\n      maxIterations,\n    });\n  }\n\n  /**\n   * Runs the AI with a given set of goals.\n   * @param goals An array of strings representing the goals.\n   * @returns A string representing the result of the run or undefined if the maximum number of iterations is reached without a result.\n   */\n  async run(goals: string[]): Promise<string | undefined> {\n    const user_input =\n      \"Determine which next command to use, and respond using the format specified above:\";\n    let loopCount = 0;\n    while (loopCount < this.maxIterations) {\n      loopCount += 1;\n\n      const { text: assistantReply } = await this.chain.call({\n        goals,\n        user_input,\n        memory: this.memory,\n        messages: this.fullMessageHistory,\n      });\n\n      // Print the assistant reply\n      console.log(assistantReply);\n      this.fullMessageHistory.push(new HumanMessage(user_input));\n      this.fullMessageHistory.push(new AIMessage(assistantReply));\n\n      const action = await this.outputParser.parse(assistantReply);\n      const tools = this.tools.reduce(\n        (acc, tool) => ({ ...acc, [tool.name]: tool }),\n        {} as { [key: string]: ObjectTool }\n      );\n      if (action.name === FINISH_NAME) {\n        return action.args.response;\n      }\n      let result: string;\n      if (action.name in tools) {\n        const tool = tools[action.name];\n        let observation;\n        try {\n          observation = await tool.call(action.args);\n        } catch (e) {\n          observation = `Error in args: ${e}`;\n        }\n        result = `Command ${tool.name} returned: ${observation}`;\n      } else if (action.name === \"ERROR\") {\n        result = `Error: ${action.args}. `;\n      } else {\n        result = `Unknown command '${action.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format.`;\n      }\n\n      let memoryToAdd = `Assistant Reply: ${assistantReply}\\nResult: ${result} `;\n      if (this.feedbackTool) {\n        const feedback = `\\n${await this.feedbackTool.call(\"Input: \")}`;\n        if (feedback === \"q\" || feedback === \"stop\") {\n          console.log(\"EXITING\");\n          return \"EXITING\";\n        }\n        memoryToAdd += feedback;\n      }\n\n      const documents = await this.textSplitter.createDocuments([memoryToAdd]);\n      await this.memory.addDocuments(documents);\n      this.fullMessageHistory.push(new SystemMessage(result));\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,IAAa,UAAb,MAAa,QAAQ;CACnB;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA,YAAY,EACV,QACA,QACA,OACA,cACA,OACA,cACA,iBAKC;EACD,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,qBAAqB,CAAC;EAC3B,KAAK,kBAAkB;EACvB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,gBAAgB;EACrB,MAAM,YAAY,wBAChB,eAAe,OAAO,YAAY,aAC7B,OAAO,YAAY,WAAW,YAC/B,KAAA,CACN;EACA,KAAK,eAAe,IAAIA,sBAAAA,kBAAkB;GACxC;GACA,cAAc,KAAK,MAAM,YAAY,EAAE;EACzC,CAAC;CACH;;;;;;;;;;;;CAaA,OAAO,gBACL,KACA,OACA,EACE,QACA,QACA,QACA,gBAAgB,KAEhB,eAAe,IAAI,oBAAoB,KAEhC;EAWT,MAAM,QAAQ,IAAI,SAAS;GAAE;GAAK,QAAA,IAVf,cAAc;IAC/B;IACA;IACA;IACA,cAAc,IAAI,aAAa,KAAK,GAAG;IACvC,gBAAgB,oBACd,eAAe,MAAO,IAAI,YAAuB,MACnD;GACF,CAEuC;EAAE,CAAC;EAC1C,OAAO,IAAI,QAAQ;GACjB;GACA;GACA;GACA;GACA;GAEA;EACF,CAAC;CACH;;;;;;CAOA,MAAM,IAAI,OAA8C;EACtD,MAAM,aACJ;EACF,IAAI,YAAY;EAChB,OAAO,YAAY,KAAK,eAAe;GACrC,aAAa;GAEb,MAAM,EAAE,MAAM,mBAAmB,MAAM,KAAK,MAAM,KAAK;IACrD;IACA;IACA,QAAQ,KAAK;IACb,UAAU,KAAK;GACjB,CAAC;GAGD,QAAQ,IAAI,cAAc;GAC1B,KAAK,mBAAmB,KAAK,IAAI,aAAa,UAAU,CAAC;GACzD,KAAK,mBAAmB,KAAK,IAAI,UAAU,cAAc,CAAC;GAE1D,MAAM,SAAS,MAAM,KAAK,aAAa,MAAM,cAAc;GAC3D,MAAM,QAAQ,KAAK,MAAM,QACtB,KAAK,UAAU;IAAE,GAAG;KAAM,KAAK,OAAO;GAAK,IAC5C,CAAC,CACH;GACA,IAAI,OAAO,SAAA,UACT,OAAO,OAAO,KAAK;GAErB,IAAI;GACJ,IAAI,OAAO,QAAQ,OAAO;IACxB,MAAM,OAAO,MAAM,OAAO;IAC1B,IAAI;IACJ,IAAI;KACF,cAAc,MAAM,KAAK,KAAK,OAAO,IAAI;IAC3C,SAAS,GAAG;KACV,cAAc,kBAAkB;IAClC;IACA,SAAS,WAAW,KAAK,KAAK,aAAa;GAC7C,OAAO,IAAI,OAAO,SAAS,SACzB,SAAS,UAAU,OAAO,KAAK;QAE/B,SAAS,oBAAoB,OAAO,KAAK;GAG3C,IAAI,cAAc,oBAAoB,eAAe,YAAY,OAAO;GACxE,IAAI,KAAK,cAAc;IACrB,MAAM,WAAW,KAAK,MAAM,KAAK,aAAa,KAAK,SAAS;IAC5D,IAAI,aAAa,OAAO,aAAa,QAAQ;KAC3C,QAAQ,IAAI,SAAS;KACrB,OAAO;IACT;IACA,eAAe;GACjB;GAEA,MAAM,YAAY,MAAM,KAAK,aAAa,gBAAgB,CAAC,WAAW,CAAC;GACvE,MAAM,KAAK,OAAO,aAAa,SAAS;GACxC,KAAK,mBAAmB,KAAK,IAAI,cAAc,MAAM,CAAC;EACxD;CAGF;AACF"}
+{"version":3,"file":"agent.js","names":["TokenTextSplitter"],"sources":["../../../src/experimental/autogpt/agent.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport {\n  AIMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  getEmbeddingContextSize,\n  getModelContextSize,\n} from \"@langchain/core/language_models/base\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\n\nimport { AutoGPTOutputParser } from \"./output_parser.js\";\nimport { AutoGPTPrompt } from \"./prompt.js\";\n// import { HumanInputRun } from \"./tools/human/tool\"; // TODO\nimport { ObjectTool, FINISH_NAME } from \"./schema.js\";\nimport { TokenTextSplitter } from \"../../text_splitter.js\";\n\n/**\n * Interface for the input parameters of the AutoGPT class.\n */\nexport interface AutoGPTInput {\n  aiName: string;\n  aiRole: string;\n  memory: VectorStoreRetrieverInterface;\n  humanInTheLoop?: boolean;\n  outputParser?: AutoGPTOutputParser;\n  maxIterations?: number;\n}\n\n/**\n * Class representing the AutoGPT concept with LangChain primitives. It is\n * designed to be used with a set of tools such as a search tool,\n * write-file tool, and a read-file tool.\n * @example\n * ```typescript\n * const autogpt = AutoGPT.fromLLMAndTools(\n *   new ChatOpenAI({ model: \"gpt-4o-mini\", temperature: 0 }),\n *   [\n *     new ReadFileTool({ store: new InMemoryFileStore() }),\n *     new WriteFileTool({ store: new InMemoryFileStore() }),\n *     new SerpAPI(\"YOUR_SERPAPI_API_KEY\", {\n *       location: \"San Francisco,California,United States\",\n *       hl: \"en\",\n *       gl: \"us\",\n *     }),\n *   ],\n *   {\n *     memory: new MemoryVectorStore(new OpenAIEmbeddings()).asRetriever(),\n *     aiName: \"Tom\",\n *     aiRole: \"Assistant\",\n *   },\n * );\n * const result = await autogpt.run([\"write a weather report for SF today\"]);\n * ```\n */\nexport class AutoGPT {\n  aiName: string;\n\n  memory: VectorStoreRetrieverInterface;\n\n  fullMessageHistory: BaseMessage[];\n\n  nextActionCount: number;\n\n  chain: LLMChain;\n\n  outputParser: AutoGPTOutputParser;\n\n  tools: ObjectTool[];\n\n  feedbackTool?: Tool;\n\n  maxIterations: number;\n\n  // Currently not generic enough to support any text splitter.\n  textSplitter: TokenTextSplitter;\n\n  constructor({\n    aiName,\n    memory,\n    chain,\n    outputParser,\n    tools,\n    feedbackTool,\n    maxIterations,\n  }: Omit<Required<AutoGPTInput>, \"aiRole\" | \"humanInTheLoop\"> & {\n    chain: LLMChain;\n    tools: ObjectTool[];\n    feedbackTool?: Tool;\n  }) {\n    this.aiName = aiName;\n    this.memory = memory;\n    this.fullMessageHistory = [];\n    this.nextActionCount = 0;\n    this.chain = chain;\n    this.outputParser = outputParser;\n    this.tools = tools;\n    this.feedbackTool = feedbackTool;\n    this.maxIterations = maxIterations;\n    const chunkSize = getEmbeddingContextSize(\n      \"modelName\" in memory.vectorStore.embeddings\n        ? (memory.vectorStore.embeddings.modelName as string)\n        : undefined\n    );\n    this.textSplitter = new TokenTextSplitter({\n      chunkSize,\n      chunkOverlap: Math.round(chunkSize / 10),\n    });\n  }\n\n  /**\n   * Creates a new AutoGPT instance from a given LLM and a set of tools.\n   * @param llm A BaseChatModel object.\n   * @param tools An array of ObjectTool objects.\n   * @param options.aiName The name of the AI.\n   * @param options.aiRole The role of the AI.\n   * @param options.memory A VectorStoreRetriever object that represents the memory of the AI.\n   * @param options.maxIterations The maximum number of iterations the AI can perform.\n   * @param options.outputParser An AutoGPTOutputParser object that parses the output of the AI.\n   * @returns A new instance of the AutoGPT class.\n   */\n  static fromLLMAndTools(\n    llm: BaseChatModel,\n    tools: ObjectTool[],\n    {\n      aiName,\n      aiRole,\n      memory,\n      maxIterations = 100,\n      // humanInTheLoop = false,\n      outputParser = new AutoGPTOutputParser(),\n    }: AutoGPTInput\n  ): AutoGPT {\n    const prompt = new AutoGPTPrompt({\n      aiName,\n      aiRole,\n      tools,\n      tokenCounter: llm.getNumTokens.bind(llm),\n      sendTokenLimit: getModelContextSize(\n        \"modelName\" in llm ? (llm.modelName as string) : \"gpt2\"\n      ),\n    });\n    // const feedbackTool = humanInTheLoop ? new HumanInputRun() : null;\n    const chain = new LLMChain({ llm, prompt });\n    return new AutoGPT({\n      aiName,\n      memory,\n      chain,\n      outputParser,\n      tools,\n      // feedbackTool,\n      maxIterations,\n    });\n  }\n\n  /**\n   * Runs the AI with a given set of goals.\n   * @param goals An array of strings representing the goals.\n   * @returns A string representing the result of the run or undefined if the maximum number of iterations is reached without a result.\n   */\n  async run(goals: string[]): Promise<string | undefined> {\n    const user_input =\n      \"Determine which next command to use, and respond using the format specified above:\";\n    let loopCount = 0;\n    while (loopCount < this.maxIterations) {\n      loopCount += 1;\n\n      const { text: assistantReply } = await this.chain.call({\n        goals,\n        user_input,\n        memory: this.memory,\n        messages: this.fullMessageHistory,\n      });\n\n      // Print the assistant reply\n      console.log(assistantReply);\n      this.fullMessageHistory.push(new HumanMessage(user_input));\n      this.fullMessageHistory.push(new AIMessage(assistantReply));\n\n      const action = await this.outputParser.parse(assistantReply);\n      const tools = this.tools.reduce(\n        (acc, tool) => ({ ...acc, [tool.name]: tool }),\n        {} as { [key: string]: ObjectTool }\n      );\n      if (action.name === FINISH_NAME) {\n        return action.args.response;\n      }\n      let result: string;\n      if (action.name in tools) {\n        const tool = tools[action.name];\n        let observation;\n        try {\n          observation = await tool.call(action.args);\n        } catch (e) {\n          observation = `Error in args: ${e}`;\n        }\n        result = `Command ${tool.name} returned: ${observation}`;\n      } else if (action.name === \"ERROR\") {\n        result = `Error: ${action.args}. `;\n      } else {\n        result = `Unknown command '${action.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format.`;\n      }\n\n      let memoryToAdd = `Assistant Reply: ${assistantReply}\\nResult: ${result} `;\n      if (this.feedbackTool) {\n        const feedback = `\\n${await this.feedbackTool.call(\"Input: \")}`;\n        if (feedback === \"q\" || feedback === \"stop\") {\n          console.log(\"EXITING\");\n          return \"EXITING\";\n        }\n        memoryToAdd += feedback;\n      }\n\n      const documents = await this.textSplitter.createDocuments([memoryToAdd]);\n      await this.memory.addDocuments(documents);\n      this.fullMessageHistory.push(new SystemMessage(result));\n    }\n\n    return undefined;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,IAAa,UAAb,MAAa,QAAQ;CACnB;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAGA;CAEA,YAAY,EACV,QACA,QACA,OACA,cACA,OACA,cACA,iBAKC;EACD,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,qBAAqB,CAAC;EAC3B,KAAK,kBAAkB;EACvB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,QAAQ;EACb,KAAK,eAAe;EACpB,KAAK,gBAAgB;EACrB,MAAM,YAAY,wBAChB,eAAe,OAAO,YAAY,aAC7B,OAAO,YAAY,WAAW,YAC/B,KAAA,CACN;EACA,KAAK,eAAe,IAAIA,sBAAAA,kBAAkB;GACxC;GACA,cAAc,KAAK,MAAM,YAAY,EAAE;EACzC,CAAC;CACH;;;;;;;;;;;;CAaA,OAAO,gBACL,KACA,OACA,EACE,QACA,QACA,QACA,gBAAgB,KAEhB,eAAe,IAAI,oBAAoB,KAEhC;EACT,MAAM,SAAS,IAAI,cAAc;GAC/B;GACA;GACA;GACA,cAAc,IAAI,aAAa,KAAK,GAAG;GACvC,gBAAgB,oBACd,eAAe,MAAO,IAAI,YAAuB,MACnD;EACF,CAAC;EAED,MAAM,QAAQ,IAAI,SAAS;GAAE;GAAK;EAAO,CAAC;EAC1C,OAAO,IAAI,QAAQ;GACjB;GACA;GACA;GACA;GACA;GAEA;EACF,CAAC;CACH;;;;;;CAOA,MAAM,IAAI,OAA8C;EACtD,MAAM,aACJ;EACF,IAAI,YAAY;EAChB,OAAO,YAAY,KAAK,eAAe;GACrC,aAAa;GAEb,MAAM,EAAE,MAAM,mBAAmB,MAAM,KAAK,MAAM,KAAK;IACrD;IACA;IACA,QAAQ,KAAK;IACb,UAAU,KAAK;GACjB,CAAC;GAGD,QAAQ,IAAI,cAAc;GAC1B,KAAK,mBAAmB,KAAK,IAAI,aAAa,UAAU,CAAC;GACzD,KAAK,mBAAmB,KAAK,IAAI,UAAU,cAAc,CAAC;GAE1D,MAAM,SAAS,MAAM,KAAK,aAAa,MAAM,cAAc;GAC3D,MAAM,QAAQ,KAAK,MAAM,QACtB,KAAK,UAAU;IAAE,GAAG;KAAM,KAAK,OAAO;GAAK,IAC5C,CAAC,CACH;GACA,IAAI,OAAO,SAAA,UACT,OAAO,OAAO,KAAK;GAErB,IAAI;GACJ,IAAI,OAAO,QAAQ,OAAO;IACxB,MAAM,OAAO,MAAM,OAAO;IAC1B,IAAI;IACJ,IAAI;KACF,cAAc,MAAM,KAAK,KAAK,OAAO,IAAI;IAC3C,SAAS,GAAG;KACV,cAAc,kBAAkB;IAClC;IACA,SAAS,WAAW,KAAK,KAAK,aAAa;GAC7C,OAAO,IAAI,OAAO,SAAS,SACzB,SAAS,UAAU,OAAO,KAAK;QAE/B,SAAS,oBAAoB,OAAO,KAAK;GAG3C,IAAI,cAAc,oBAAoB,eAAe,YAAY,OAAO;GACxE,IAAI,KAAK,cAAc;IACrB,MAAM,WAAW,KAAK,MAAM,KAAK,aAAa,KAAK,SAAS;IAC5D,IAAI,aAAa,OAAO,aAAa,QAAQ;KAC3C,QAAQ,IAAI,SAAS;KACrB,OAAO;IACT;IACA,eAAe;GACjB;GAEA,MAAM,YAAY,MAAM,KAAK,aAAa,gBAAgB,CAAC,WAAW,CAAC;GACvE,MAAM,KAAK,OAAO,aAAa,SAAS;GACxC,KAAK,mBAAmB,KAAK,IAAI,cAAc,MAAM,CAAC;EACxD;CAGF;AACF"}

package/dist/experimental/autogpt/prompt.cjs

@@ -66,7 +66,8 @@ var AutoGPTPrompt = class extends _langchain_core_prompts.BaseChatPromptTemplate
 			relevantMemory.pop();
 			relevantMemoryTokens = await relevantMemory.reduce(async (acc, doc) => await acc + await this.tokenCounter(doc), Promise.resolve(0));
 		}
-		const memoryMessage = new _langchain_core_messages.SystemMessage(`This reminds you of these events from your past:\n${relevantMemory.join("\n")}\n\n`);
+		const contentFormat = `This reminds you of these events from your past:\n${relevantMemory.join("\n")}\n\n`;
+		const memoryMessage = new _langchain_core_messages.SystemMessage(contentFormat);
 		if (typeof memoryMessage.content !== "string") throw new Error("Non-string message content is not supported.");
 		const usedTokensWithMemory = usedTokens + await this.tokenCounter(memoryMessage.content);
 		const historicalMessages = [];

package/dist/experimental/autogpt/prompt.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.cjs","names":["BaseChatPromptTemplate","getPrompt","SystemMessage","HumanMessage"],"sources":["../../../src/experimental/autogpt/prompt.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport {\n  BaseChatPromptTemplate,\n  SerializedBasePromptTemplate,\n} from \"@langchain/core/prompts\";\nimport {\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { PartialValues } from \"@langchain/core/utils/types\";\nimport { getPrompt } from \"./prompt_generator.js\";\nimport { ObjectTool } from \"./schema.js\";\n\n/**\n * Interface for the input parameters of the AutoGPTPrompt class.\n */\nexport interface AutoGPTPromptInput {\n  aiName: string;\n  aiRole: string;\n  tools: ObjectTool[];\n  tokenCounter: (text: string) => Promise<number>;\n  sendTokenLimit?: number;\n}\n\n/**\n * Class used to generate prompts for the AutoGPT model. It takes into\n * account the AI's name, role, tools, token counter, and send token\n * limit. The class also handles the formatting of messages and the\n * construction of the full prompt.\n */\nexport class AutoGPTPrompt\n  extends BaseChatPromptTemplate\n  implements AutoGPTPromptInput\n{\n  aiName: string;\n\n  aiRole: string;\n\n  tools: ObjectTool[];\n\n  tokenCounter: (text: string) => Promise<number>;\n\n  sendTokenLimit: number;\n\n  constructor(fields: AutoGPTPromptInput) {\n    super({ inputVariables: [\"goals\", \"memory\", \"messages\", \"user_input\"] });\n    this.aiName = fields.aiName;\n    this.aiRole = fields.aiRole;\n    this.tools = fields.tools;\n    this.tokenCounter = fields.tokenCounter;\n    this.sendTokenLimit = fields.sendTokenLimit || 4196;\n  }\n\n  _getPromptType() {\n    return \"autogpt\" as const;\n  }\n\n  /**\n   * Constructs the full prompt based on the provided goals.\n   * @param goals An array of goals.\n   * @returns The full prompt as a string.\n   */\n  constructFullPrompt(goals: string[]): string {\n    const promptStart = `Your decisions must always be made independently\n            without seeking user assistance. Play to your strengths\n            as an LLM and pursue simple strategies with no legal complications.\n            If you have completed all your tasks,\n            make sure to use the \"finish\" command.`;\n\n    let fullPrompt = `You are ${this.aiName}, ${this.aiRole}\\n${promptStart}\\n\\nGOALS:\\n\\n`;\n    goals.forEach((goal, index) => {\n      fullPrompt += `${index + 1}. ${goal}\\n`;\n    });\n\n    fullPrompt += `\\n\\n${getPrompt(this.tools)}`;\n    return fullPrompt;\n  }\n\n  /**\n   * Formats the messages based on the provided parameters.\n   * @param goals An array of goals.\n   * @param memory A VectorStoreRetriever instance.\n   * @param messages An array of previous messages.\n   * @param user_input The user's input.\n   * @returns An array of formatted messages.\n   */\n  async formatMessages({\n    goals,\n    memory,\n    messages: previousMessages,\n    user_input,\n  }: {\n    goals: string[];\n    memory: VectorStoreRetrieverInterface;\n    messages: BaseMessage[];\n    user_input: string;\n  }) {\n    const basePrompt = new SystemMessage(this.constructFullPrompt(goals));\n    const timePrompt = new SystemMessage(\n      `The current time and date is ${new Date().toLocaleString(\"en-US\")}`\n    );\n    if (\n      typeof basePrompt.content !== \"string\" ||\n      typeof timePrompt.content !== \"string\"\n    ) {\n      throw new Error(\"Non-string message content is not supported.\");\n    }\n    const usedTokens =\n      (await this.tokenCounter(basePrompt.content)) +\n      (await this.tokenCounter(timePrompt.content));\n    const relevantDocs = await memory.invoke(\n      JSON.stringify(previousMessages.slice(-10))\n    );\n    const relevantMemory = relevantDocs.map(\n      (d: { pageContent: string }) => d.pageContent\n    );\n    let relevantMemoryTokens = await relevantMemory.reduce(\n      async (acc: Promise<number>, doc: string) =>\n        (await acc) + (await this.tokenCounter(doc)),\n      Promise.resolve(0)\n    );\n\n    while (usedTokens + relevantMemoryTokens > 2500) {\n      relevantMemory.pop();\n      relevantMemoryTokens = await relevantMemory.reduce(\n        async (acc: Promise<number>, doc: string) =>\n          (await acc) + (await this.tokenCounter(doc)),\n        Promise.resolve(0)\n      );\n    }\n\n    const contentFormat = `This reminds you of these events from your past:\\n${relevantMemory.join(\n      \"\\n\"\n    )}\\n\\n`;\n    const memoryMessage = new SystemMessage(contentFormat);\n    if (typeof memoryMessage.content !== \"string\") {\n      throw new Error(\"Non-string message content is not supported.\");\n    }\n    const usedTokensWithMemory =\n      usedTokens + (await this.tokenCounter(memoryMessage.content));\n    const historicalMessages: BaseMessage[] = [];\n\n    for (const message of previousMessages.slice(-10).reverse()) {\n      if (typeof message.content !== \"string\") {\n        throw new Error(\"Non-string message content is not supported.\");\n      }\n      const messageTokens = await this.tokenCounter(message.content);\n      if (usedTokensWithMemory + messageTokens > this.sendTokenLimit - 1000) {\n        break;\n      }\n      historicalMessages.unshift(message);\n    }\n\n    const inputMessage = new HumanMessage(user_input);\n    const messages: BaseMessage[] = [\n      basePrompt,\n      timePrompt,\n      memoryMessage,\n      ...historicalMessages,\n      inputMessage,\n    ];\n    return messages;\n  }\n\n  /**\n   * This method is not implemented in the AutoGPTPrompt class and will\n   * throw an error if called.\n   * @param _values Partial values.\n   * @returns Throws an error.\n   */\n  async partial(_values: PartialValues): Promise<BaseChatPromptTemplate> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  serialize(): SerializedBasePromptTemplate {\n    throw new Error(\"Method not implemented.\");\n  }\n}\n"],"mappings":";;;;;;;;;;AA+BA,IAAa,gBAAb,cACUA,wBAAAA,uBAEV;CACE;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,QAA4B;EACtC,MAAM,EAAE,gBAAgB;GAAC;GAAS;GAAU;GAAY;EAAY,EAAE,CAAC;EACvE,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,QAAQ,OAAO;EACpB,KAAK,eAAe,OAAO;EAC3B,KAAK,iBAAiB,OAAO,kBAAkB;CACjD;CAEA,iBAAiB;EACf,OAAO;CACT;;;;;;CAOA,oBAAoB,OAAyB;EAO3C,IAAI,aAAa,WAAW,KAAK,OAAO,IAAI,KAAK,OAAO;;;;;EACxD,MAAM,SAAS,MAAM,UAAU;GAC7B,cAAc,GAAG,QAAQ,EAAE,IAAI,KAAK;EACtC,CAAC;EAED,cAAc,OAAOC,yBAAAA,UAAU,KAAK,KAAK;EACzC,OAAO;CACT;;;;;;;;;CAUA,MAAM,eAAe,EACnB,OACA,QACA,UAAU,kBACV,cAMC;EACD,MAAM,aAAa,IAAIC,yBAAAA,cAAc,KAAK,oBAAoB,KAAK,CAAC;EACpE,MAAM,aAAa,IAAIA,yBAAAA,cACrB,iDAAgC,IAAI,KAAK,EAAA,CAAE,eAAe,OAAO,GACnE;EACA,IACE,OAAO,WAAW,YAAY,YAC9B,OAAO,WAAW,YAAY,UAE9B,MAAM,IAAI,MAAM,8CAA8C;EAEhE,MAAM,aACH,MAAM,KAAK,aAAa,WAAW,OAAO,IAC1C,MAAM,KAAK,aAAa,WAAW,OAAO;EAI7C,MAAM,kBAAiB,MAHI,OAAO,OAChC,KAAK,UAAU,iBAAiB,MAAM,GAAG,CAAC,CAC5C,EAAA,CACoC,KACjC,MAA+B,EAAE,WACpC;EACA,IAAI,uBAAuB,MAAM,eAAe,OAC9C,OAAO,KAAsB,QAC1B,MAAM,MAAQ,MAAM,KAAK,aAAa,GAAG,GAC5C,QAAQ,QAAQ,CAAC,CACnB;EAEA,OAAO,aAAa,uBAAuB,MAAM;GAC/C,eAAe,IAAI;GACnB,uBAAuB,MAAM,eAAe,OAC1C,OAAO,KAAsB,QAC1B,MAAM,MAAQ,MAAM,KAAK,aAAa,GAAG,GAC5C,QAAQ,QAAQ,CAAC,CACnB;EACF;EAKA,MAAM,gBAAgB,IAAIA,yBAAAA,cAAc,qDAHmC,eAAe,KACxF,IACF,EAAE,KACmD;EACrD,IAAI,OAAO,cAAc,YAAY,UACnC,MAAM,IAAI,MAAM,8CAA8C;EAEhE,MAAM,uBACJ,aAAc,MAAM,KAAK,aAAa,cAAc,OAAO;EAC7D,MAAM,qBAAoC,CAAC;EAE3C,KAAK,MAAM,WAAW,iBAAiB,MAAM,GAAG,CAAC,CAAC,QAAQ,GAAG;GAC3D,IAAI,OAAO,QAAQ,YAAY,UAC7B,MAAM,IAAI,MAAM,8CAA8C;GAGhE,IAAI,uBAAuB,MADC,KAAK,aAAa,QAAQ,OAAO,IAClB,KAAK,iBAAiB,KAC/D;GAEF,mBAAmB,QAAQ,OAAO;EACpC;EAEA,MAAM,eAAe,IAAIC,yBAAAA,aAAa,UAAU;EAQhD,OAAO;GANL;GACA;GACA;GACA,GAAG;GACH;EAEY;CAChB;;;;;;;CAQA,MAAM,QAAQ,SAAyD;EACrE,MAAM,IAAI,MAAM,yBAAyB;CAC3C;CAEA,YAA0C;EACxC,MAAM,IAAI,MAAM,yBAAyB;CAC3C;AACF"}
+{"version":3,"file":"prompt.cjs","names":["BaseChatPromptTemplate","getPrompt","SystemMessage","HumanMessage"],"sources":["../../../src/experimental/autogpt/prompt.ts"],"sourcesContent":["import type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport {\n  BaseChatPromptTemplate,\n  SerializedBasePromptTemplate,\n} from \"@langchain/core/prompts\";\nimport {\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { PartialValues } from \"@langchain/core/utils/types\";\nimport { getPrompt } from \"./prompt_generator.js\";\nimport { ObjectTool } from \"./schema.js\";\n\n/**\n * Interface for the input parameters of the AutoGPTPrompt class.\n */\nexport interface AutoGPTPromptInput {\n  aiName: string;\n  aiRole: string;\n  tools: ObjectTool[];\n  tokenCounter: (text: string) => Promise<number>;\n  sendTokenLimit?: number;\n}\n\n/**\n * Class used to generate prompts for the AutoGPT model. It takes into\n * account the AI's name, role, tools, token counter, and send token\n * limit. The class also handles the formatting of messages and the\n * construction of the full prompt.\n */\nexport class AutoGPTPrompt\n  extends BaseChatPromptTemplate\n  implements AutoGPTPromptInput\n{\n  aiName: string;\n\n  aiRole: string;\n\n  tools: ObjectTool[];\n\n  tokenCounter: (text: string) => Promise<number>;\n\n  sendTokenLimit: number;\n\n  constructor(fields: AutoGPTPromptInput) {\n    super({ inputVariables: [\"goals\", \"memory\", \"messages\", \"user_input\"] });\n    this.aiName = fields.aiName;\n    this.aiRole = fields.aiRole;\n    this.tools = fields.tools;\n    this.tokenCounter = fields.tokenCounter;\n    this.sendTokenLimit = fields.sendTokenLimit || 4196;\n  }\n\n  _getPromptType() {\n    return \"autogpt\" as const;\n  }\n\n  /**\n   * Constructs the full prompt based on the provided goals.\n   * @param goals An array of goals.\n   * @returns The full prompt as a string.\n   */\n  constructFullPrompt(goals: string[]): string {\n    const promptStart = `Your decisions must always be made independently\n            without seeking user assistance. Play to your strengths\n            as an LLM and pursue simple strategies with no legal complications.\n            If you have completed all your tasks,\n            make sure to use the \"finish\" command.`;\n\n    let fullPrompt = `You are ${this.aiName}, ${this.aiRole}\\n${promptStart}\\n\\nGOALS:\\n\\n`;\n    goals.forEach((goal, index) => {\n      fullPrompt += `${index + 1}. ${goal}\\n`;\n    });\n\n    fullPrompt += `\\n\\n${getPrompt(this.tools)}`;\n    return fullPrompt;\n  }\n\n  /**\n   * Formats the messages based on the provided parameters.\n   * @param goals An array of goals.\n   * @param memory A VectorStoreRetriever instance.\n   * @param messages An array of previous messages.\n   * @param user_input The user's input.\n   * @returns An array of formatted messages.\n   */\n  async formatMessages({\n    goals,\n    memory,\n    messages: previousMessages,\n    user_input,\n  }: {\n    goals: string[];\n    memory: VectorStoreRetrieverInterface;\n    messages: BaseMessage[];\n    user_input: string;\n  }) {\n    const basePrompt = new SystemMessage(this.constructFullPrompt(goals));\n    const timePrompt = new SystemMessage(\n      `The current time and date is ${new Date().toLocaleString(\"en-US\")}`\n    );\n    if (\n      typeof basePrompt.content !== \"string\" ||\n      typeof timePrompt.content !== \"string\"\n    ) {\n      throw new Error(\"Non-string message content is not supported.\");\n    }\n    const usedTokens =\n      (await this.tokenCounter(basePrompt.content)) +\n      (await this.tokenCounter(timePrompt.content));\n    const relevantDocs = await memory.invoke(\n      JSON.stringify(previousMessages.slice(-10))\n    );\n    const relevantMemory = relevantDocs.map(\n      (d: { pageContent: string }) => d.pageContent\n    );\n    let relevantMemoryTokens = await relevantMemory.reduce(\n      async (acc: Promise<number>, doc: string) =>\n        (await acc) + (await this.tokenCounter(doc)),\n      Promise.resolve(0)\n    );\n\n    while (usedTokens + relevantMemoryTokens > 2500) {\n      relevantMemory.pop();\n      relevantMemoryTokens = await relevantMemory.reduce(\n        async (acc: Promise<number>, doc: string) =>\n          (await acc) + (await this.tokenCounter(doc)),\n        Promise.resolve(0)\n      );\n    }\n\n    const contentFormat = `This reminds you of these events from your past:\\n${relevantMemory.join(\n      \"\\n\"\n    )}\\n\\n`;\n    const memoryMessage = new SystemMessage(contentFormat);\n    if (typeof memoryMessage.content !== \"string\") {\n      throw new Error(\"Non-string message content is not supported.\");\n    }\n    const usedTokensWithMemory =\n      usedTokens + (await this.tokenCounter(memoryMessage.content));\n    const historicalMessages: BaseMessage[] = [];\n\n    for (const message of previousMessages.slice(-10).reverse()) {\n      if (typeof message.content !== \"string\") {\n        throw new Error(\"Non-string message content is not supported.\");\n      }\n      const messageTokens = await this.tokenCounter(message.content);\n      if (usedTokensWithMemory + messageTokens > this.sendTokenLimit - 1000) {\n        break;\n      }\n      historicalMessages.unshift(message);\n    }\n\n    const inputMessage = new HumanMessage(user_input);\n    const messages: BaseMessage[] = [\n      basePrompt,\n      timePrompt,\n      memoryMessage,\n      ...historicalMessages,\n      inputMessage,\n    ];\n    return messages;\n  }\n\n  /**\n   * This method is not implemented in the AutoGPTPrompt class and will\n   * throw an error if called.\n   * @param _values Partial values.\n   * @returns Throws an error.\n   */\n  async partial(_values: PartialValues): Promise<BaseChatPromptTemplate> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  serialize(): SerializedBasePromptTemplate {\n    throw new Error(\"Method not implemented.\");\n  }\n}\n"],"mappings":";;;;;;;;;;AA+BA,IAAa,gBAAb,cACUA,wBAAAA,uBAEV;CACE;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,QAA4B;EACtC,MAAM,EAAE,gBAAgB;GAAC;GAAS;GAAU;GAAY;EAAY,EAAE,CAAC;EACvE,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,QAAQ,OAAO;EACpB,KAAK,eAAe,OAAO;EAC3B,KAAK,iBAAiB,OAAO,kBAAkB;CACjD;CAEA,iBAAiB;EACf,OAAO;CACT;;;;;;CAOA,oBAAoB,OAAyB;EAO3C,IAAI,aAAa,WAAW,KAAK,OAAO,IAAI,KAAK,OAAO;;;;;EACxD,MAAM,SAAS,MAAM,UAAU;GAC7B,cAAc,GAAG,QAAQ,EAAE,IAAI,KAAK;EACtC,CAAC;EAED,cAAc,OAAOC,yBAAAA,UAAU,KAAK,KAAK;EACzC,OAAO;CACT;;;;;;;;;CAUA,MAAM,eAAe,EACnB,OACA,QACA,UAAU,kBACV,cAMC;EACD,MAAM,aAAa,IAAIC,yBAAAA,cAAc,KAAK,oBAAoB,KAAK,CAAC;EACpE,MAAM,aAAa,IAAIA,yBAAAA,cACrB,iDAAgC,IAAI,KAAK,EAAA,CAAE,eAAe,OAAO,GACnE;EACA,IACE,OAAO,WAAW,YAAY,YAC9B,OAAO,WAAW,YAAY,UAE9B,MAAM,IAAI,MAAM,8CAA8C;EAEhE,MAAM,aACH,MAAM,KAAK,aAAa,WAAW,OAAO,IAC1C,MAAM,KAAK,aAAa,WAAW,OAAO;EAI7C,MAAM,kBAAiB,MAHI,OAAO,OAChC,KAAK,UAAU,iBAAiB,MAAM,GAAG,CAAC,CAC5C,EAAA,CACoC,KACjC,MAA+B,EAAE,WACpC;EACA,IAAI,uBAAuB,MAAM,eAAe,OAC9C,OAAO,KAAsB,QAC1B,MAAM,MAAQ,MAAM,KAAK,aAAa,GAAG,GAC5C,QAAQ,QAAQ,CAAC,CACnB;EAEA,OAAO,aAAa,uBAAuB,MAAM;GAC/C,eAAe,IAAI;GACnB,uBAAuB,MAAM,eAAe,OAC1C,OAAO,KAAsB,QAC1B,MAAM,MAAQ,MAAM,KAAK,aAAa,GAAG,GAC5C,QAAQ,QAAQ,CAAC,CACnB;EACF;EAEA,MAAM,gBAAgB,qDAAqD,eAAe,KACxF,IACF,EAAE;EACF,MAAM,gBAAgB,IAAIA,yBAAAA,cAAc,aAAa;EACrD,IAAI,OAAO,cAAc,YAAY,UACnC,MAAM,IAAI,MAAM,8CAA8C;EAEhE,MAAM,uBACJ,aAAc,MAAM,KAAK,aAAa,cAAc,OAAO;EAC7D,MAAM,qBAAoC,CAAC;EAE3C,KAAK,MAAM,WAAW,iBAAiB,MAAM,GAAG,CAAC,CAAC,QAAQ,GAAG;GAC3D,IAAI,OAAO,QAAQ,YAAY,UAC7B,MAAM,IAAI,MAAM,8CAA8C;GAGhE,IAAI,uBAAuB,MADC,KAAK,aAAa,QAAQ,OAAO,IAClB,KAAK,iBAAiB,KAC/D;GAEF,mBAAmB,QAAQ,OAAO;EACpC;EAEA,MAAM,eAAe,IAAIC,yBAAAA,aAAa,UAAU;EAQhD,OAAO;GANL;GACA;GACA;GACA,GAAG;GACH;EAEY;CAChB;;;;;;;CAQA,MAAM,QAAQ,SAAyD;EACrE,MAAM,IAAI,MAAM,yBAAyB;CAC3C;CAEA,YAA0C;EACxC,MAAM,IAAI,MAAM,yBAAyB;CAC3C;AACF"}

package/dist/experimental/autogpt/prompt.js

@@ -66,7 +66,8 @@ var AutoGPTPrompt = class extends BaseChatPromptTemplate {
 			relevantMemory.pop();
 			relevantMemoryTokens = await relevantMemory.reduce(async (acc, doc) => await acc + await this.tokenCounter(doc), Promise.resolve(0));
 		}
-		const memoryMessage = new SystemMessage(`This reminds you of these events from your past:\n${relevantMemory.join("\n")}\n\n`);
+		const contentFormat = `This reminds you of these events from your past:\n${relevantMemory.join("\n")}\n\n`;
+		const memoryMessage = new SystemMessage(contentFormat);
 		if (typeof memoryMessage.content !== "string") throw new Error("Non-string message content is not supported.");
 		const usedTokensWithMemory = usedTokens + await this.tokenCounter(memoryMessage.content);
 		const historicalMessages = [];