@langchain/classic 1.0.23 → 1.0.24-dev-1773698445534

package/dist/embeddings/cache_backed.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"cache_backed.cjs","names":["Embeddings","EncoderBackedStore"],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2CA,sCAAW;CACpD,AAAU;CAEV,AAAU;CAEV,YAAY,QAAqC;AAC/C,QAAM,OAAO;AACb,OAAK,uBAAuB,OAAO;AACnC,OAAK,yBAAyB,OAAO;;;;;;;;;;;;;;;;;CAkBvC,MAAM,WAAW,UAAqC;AACpD,SAAO,KAAK,qBAAqB,WAAW,SAAS;;;;;;;;;;;;CAavD,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,UAAU;EACjE,MAAM,kBAAkB,EAAE;EAC1B,MAAM,mBAAmB,EAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,EACvC,KAAI,QAAQ,OAAO,QAAW;AAC5B,mBAAgB,KAAK,EAAE;AACvB,oBAAiB,KAAK,UAAU,GAAG;;AAGvC,MAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,iBAAiB;GAClE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,GAAG,CAC/C;AACD,SAAM,KAAK,uBAAuB,KAAK,cAAc;AACrD,QAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,EAC/C,SAAQ,gBAAgB,MAAM,eAAe;;AAGjD,SAAO;;;;;;;;;;CAWT,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,qBAAqB,IAAIC,kDAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,6CAAa,IAAI;GAC7D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,MAAM,CAAC;GACjE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,gBAAgB,CAAC;GAC9C,CAAC;AACF,SAAO,IAAI,KAAK;GACd;GACA,wBAAwB;GACzB,CAAC"}
+{"version":3,"file":"cache_backed.cjs","names":["Embeddings","EncoderBackedStore"],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2CA,2BAAAA,WAAW;CACpD;CAEA;CAEA,YAAY,QAAqC;AAC/C,QAAM,OAAO;AACb,OAAK,uBAAuB,OAAO;AACnC,OAAK,yBAAyB,OAAO;;;;;;;;;;;;;;;;;CAkBvC,MAAM,WAAW,UAAqC;AACpD,SAAO,KAAK,qBAAqB,WAAW,SAAS;;;;;;;;;;;;CAavD,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,UAAU;EACjE,MAAM,kBAAkB,EAAE;EAC1B,MAAM,mBAAmB,EAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,EACvC,KAAI,QAAQ,OAAO,KAAA,GAAW;AAC5B,mBAAgB,KAAK,EAAE;AACvB,oBAAiB,KAAK,UAAU,GAAG;;AAGvC,MAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,iBAAiB;GAClE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,GAAG,CAC/C;AACD,SAAM,KAAK,uBAAuB,KAAK,cAAc;AACrD,QAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,EAC/C,SAAQ,gBAAgB,MAAM,eAAe;;AAGjD,SAAO;;;;;;;;;;CAWT,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,qBAAqB,IAAIC,+BAAAA,mBAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,OAAA,GAAA,2BAAA,QAAa,IAAI;GAC7D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,MAAM,CAAC;GACjE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,gBAAgB,CAAC;GAC9C,CAAC;AACF,SAAO,IAAI,KAAK;GACd;GACA,wBAAwB;GACzB,CAAC"}

package/dist/embeddings/cache_backed.js

@@ -2,7 +2,6 @@ import { __exportAll } from "../_virtual/_rolldown/runtime.js";
 import { EncoderBackedStore } from "../storage/encoder_backed.js";
 import { sha256 } from "@langchain/core/utils/hash";
 import { Embeddings } from "@langchain/core/embeddings";
-
 //#region src/embeddings/cache_backed.ts
 var cache_backed_exports = /* @__PURE__ */ __exportAll({ CacheBackedEmbeddings: () => CacheBackedEmbeddings });
 /**
@@ -127,7 +126,7 @@ var CacheBackedEmbeddings = class extends Embeddings {
 		});
 	}
 };
-
 //#endregion
 export { CacheBackedEmbeddings, cache_backed_exports };
+
 //# sourceMappingURL=cache_backed.js.map

package/dist/embeddings/cache_backed.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache_backed.js","names":[],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2C,WAAW;CACpD,AAAU;CAEV,AAAU;CAEV,YAAY,QAAqC;AAC/C,QAAM,OAAO;AACb,OAAK,uBAAuB,OAAO;AACnC,OAAK,yBAAyB,OAAO;;;;;;;;;;;;;;;;;CAkBvC,MAAM,WAAW,UAAqC;AACpD,SAAO,KAAK,qBAAqB,WAAW,SAAS;;;;;;;;;;;;CAavD,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,UAAU;EACjE,MAAM,kBAAkB,EAAE;EAC1B,MAAM,mBAAmB,EAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,EACvC,KAAI,QAAQ,OAAO,QAAW;AAC5B,mBAAgB,KAAK,EAAE;AACvB,oBAAiB,KAAK,UAAU,GAAG;;AAGvC,MAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,iBAAiB;GAClE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,GAAG,CAC/C;AACD,SAAM,KAAK,uBAAuB,KAAK,cAAc;AACrD,QAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,EAC/C,SAAQ,gBAAgB,MAAM,eAAe;;AAGjD,SAAO;;;;;;;;;;CAWT,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,qBAAqB,IAAI,mBAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,MAAM,OAAO,IAAI;GAC7D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,MAAM,CAAC;GACjE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,gBAAgB,CAAC;GAC9C,CAAC;AACF,SAAO,IAAI,KAAK;GACd;GACA,wBAAwB;GACzB,CAAC"}
+{"version":3,"file":"cache_backed.js","names":[],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { sha256 } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors =\n        await this.underlyingEmbeddings.embedDocuments(missingDocuments);\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + sha256(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2C,WAAW;CACpD;CAEA;CAEA,YAAY,QAAqC;AAC/C,QAAM,OAAO;AACb,OAAK,uBAAuB,OAAO;AACnC,OAAK,yBAAyB,OAAO;;;;;;;;;;;;;;;;;CAkBvC,MAAM,WAAW,UAAqC;AACpD,SAAO,KAAK,qBAAqB,WAAW,SAAS;;;;;;;;;;;;CAavD,MAAM,eAAe,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,UAAU;EACjE,MAAM,kBAAkB,EAAE;EAC1B,MAAM,mBAAmB,EAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,EACvC,KAAI,QAAQ,OAAO,KAAA,GAAW;AAC5B,mBAAgB,KAAK,EAAE;AACvB,oBAAiB,KAAK,UAAU,GAAG;;AAGvC,MAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBACJ,MAAM,KAAK,qBAAqB,eAAe,iBAAiB;GAClE,MAAM,gBAAsC,iBAAiB,KAC1D,UAAU,MAAM,CAAC,UAAU,eAAe,GAAG,CAC/C;AACD,SAAM,KAAK,uBAAuB,KAAK,cAAc;AACrD,QAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,EAC/C,SAAQ,gBAAgB,MAAM,eAAe;;AAGjD,SAAO;;;;;;;;;;CAWT,OAAO,eACL,sBACA,wBACA,SAGA;EACA,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,UAAU,IAAI,aAAa;EACjC,MAAM,qBAAqB,IAAI,mBAI7B;GACA,OAAO;GACP,aAAa,SAAS,SAAS,aAAa,MAAM,OAAO,IAAI;GAC7D,kBAAkB,UAAU,QAAQ,OAAO,KAAK,UAAU,MAAM,CAAC;GACjE,oBAAoB,oBAClB,KAAK,MAAM,QAAQ,OAAO,gBAAgB,CAAC;GAC9C,CAAC;AACF,SAAO,IAAI,KAAK;GACd;GACA,wBAAwB;GACzB,CAAC"}

package/dist/embeddings/fake.cjs

@@ -1,23 +1,23 @@
-Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
-const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
-
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_runtime = require("../_virtual/_rolldown/runtime.cjs");
 //#region src/embeddings/fake.ts
 var fake_exports = /* @__PURE__ */ require_runtime.__exportAll({});
 require_runtime.__reExport(fake_exports, require("@langchain/core/utils/testing"));
-
 //#endregion
-Object.defineProperty(exports, 'fake_exports', {
-  enumerable: true,
-  get: function () {
-    return fake_exports;
-  }
+Object.defineProperty(exports, "fake_exports", {
+	enumerable: true,
+	get: function() {
+		return fake_exports;
+	}
 });
 var _langchain_core_utils_testing = require("@langchain/core/utils/testing");
-Object.keys(_langchain_core_utils_testing).forEach(function (k) {
-  if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
-    enumerable: true,
-    get: function () { return _langchain_core_utils_testing[k]; }
-  });
+Object.keys(_langchain_core_utils_testing).forEach(function(k) {
+	if (k !== "default" && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
+		enumerable: true,
+		get: function() {
+			return _langchain_core_utils_testing[k];
+		}
+	});
 });
 
 //# sourceMappingURL=fake.cjs.map

package/dist/embeddings/fake.js

@@ -1,12 +1,10 @@
 import { __exportAll, __reExport } from "../_virtual/_rolldown/runtime.js";
-
-export * from "@langchain/core/utils/testing"
-
+export * from "@langchain/core/utils/testing";
 //#region src/embeddings/fake.ts
 var fake_exports = /* @__PURE__ */ __exportAll({});
 import * as import__langchain_core_utils_testing from "@langchain/core/utils/testing";
 __reExport(fake_exports, import__langchain_core_utils_testing);
-
 //#endregion
 export { fake_exports };
+
 //# sourceMappingURL=fake.js.map

package/dist/evaluation/agents/index.cjs

@@ -1 +1 @@
-const require_trajectory = require('./trajectory.cjs');
+require("./trajectory.cjs");

package/dist/evaluation/agents/index.js

@@ -1,3 +1,2 @@
-import { TrajectoryEvalChain, TrajectoryOutputParser } from "./trajectory.js";
-
-export {  };
+import "./trajectory.js";
+export {};

package/dist/evaluation/agents/prompt.cjs

@@ -1,6 +1,5 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
+require("../../_virtual/_rolldown/runtime.cjs");
 let _langchain_core_prompts = require("@langchain/core/prompts");
-
 //#region src/evaluation/agents/prompt.ts
 /**
 * Prompt for trajectory evaluation chain.
@@ -96,7 +95,11 @@ const EVAL_CHAT_PROMPT = /* @__PURE__ */ _langchain_core_prompts.ChatPromptTempl
 	/* @__PURE__ */ _langchain_core_prompts.AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
 	/* @__PURE__ */ _langchain_core_prompts.HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE)
 ]);
-const TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.
+const TOOL_FREE_EVAL_CHAT_PROMPT = /* @__PURE__ */ _langchain_core_prompts.ChatPromptTemplate.fromMessages([
+	/* @__PURE__ */ _langchain_core_prompts.SystemMessagePromptTemplate.fromTemplate("You are a helpful assistant that evaluates language models."),
+	/* @__PURE__ */ _langchain_core_prompts.HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),
+	/* @__PURE__ */ _langchain_core_prompts.AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
+	/* @__PURE__ */ _langchain_core_prompts.HumanMessagePromptTemplate.fromTemplate(`An AI language model has been given access to a set of tools to help answer a user's question.
 
 The question the human asked the AI model was:
 [QUESTION]
@@ -121,15 +124,10 @@ i. Is the final answer helpful?
     ii. Does the AI language use a logical sequence of tools to answer the question?
     iii. Does the AI language model use the tools in a helpful way?
     iv. Does the AI language model use too many steps to answer the question?
-    v. Are the appropriate tools used to answer the question?`;
-const TOOL_FREE_EVAL_CHAT_PROMPT = /* @__PURE__ */ _langchain_core_prompts.ChatPromptTemplate.fromMessages([
-	/* @__PURE__ */ _langchain_core_prompts.SystemMessagePromptTemplate.fromTemplate("You are a helpful assistant that evaluates language models."),
-	/* @__PURE__ */ _langchain_core_prompts.HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),
-	/* @__PURE__ */ _langchain_core_prompts.AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
-	/* @__PURE__ */ _langchain_core_prompts.HumanMessagePromptTemplate.fromTemplate(TOOL_FREE_EVAL_TEMPLATE)
+    v. Are the appropriate tools used to answer the question?`)
 ]);
-
 //#endregion
 exports.EVAL_CHAT_PROMPT = EVAL_CHAT_PROMPT;
 exports.TOOL_FREE_EVAL_CHAT_PROMPT = TOOL_FREE_EVAL_CHAT_PROMPT;
+
 //# sourceMappingURL=prompt.cjs.map

package/dist/evaluation/agents/prompt.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.cjs","names":[],"sources":["../../../src/evaluation/agents/prompt.ts"],"sourcesContent":["/**\n * Prompt for trajectory evaluation chain.\n */\n\nimport {\n  AIMessagePromptTemplate,\n  ChatPromptTemplate,\n  HumanMessagePromptTemplate,\n  SystemMessagePromptTemplate,\n} from \"@langchain/core/prompts\";\n\nconst EVAL_TEMPLATE = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\n{toolDescriptions}\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_INPUT = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\nTool 1:\nName: Search\nDescription: useful for when you need to ask with search\n\nTool 2:\nName: Lookup\nDescription: useful for when you need to ask with lookup\n\nTool 3:\nName: Calculator\nDescription: useful for doing calculations\n\nTool 4:\nName: Search the Web (SerpAPI)\nDescription: useful for when you need to answer questions about current events\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\n\n    The AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\nStep 1:\nTool used: Search the Web (SerpAPI)\nTool input: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\nTool output: The Statue of Liberty was given to the United States by France, as a symbol of the two countries' friendship. It was erected atop an American-designed ...\n[END_AGENT_TRAJECTORY]\n\n[RESPONSE]\nThe AI language model's final answer to the question was: There are different ways to measure the length of the United States, but if we use the distance between the Statue of Liberty and the westernmost point of the contiguous United States (Cape Alava, Washington), which is approximately 2,857 miles (4,596 km), and assume that the Statue of Liberty is 305 feet (93 meters) tall, then the statue would stretch across the United States approximately 17.5 times if laid end to end.\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_OUTPUT = `First, let's evaluate the final answer. The final uses good reasoning but is wrong. 2,857 divided by 305 is not 17.5.\\\nThe model should have used the calculator to figure this out. Second does the model use a logical sequence of tools to answer the question?\\\nThe way model uses the search is not helpful. The model should have used the search tool to figure the width of the US or the height of the statue.\\\nThe model didn't use the calculator tool and gave an incorrect answer. The search API should be used for current events or specific questions.\\\nThe tools were not used in a helpful way. The model did not use too many steps to answer the question.\\\nThe model did not use the appropriate tools to answer the question.\\\n\nJudgment: Given the good reasoning in the final answer but otherwise poor performance, we give the model a score of 2.\n\nScore: 2`;\n\nexport const EVAL_CHAT_PROMPT = /* #__PURE__ */ ChatPromptTemplate.fromMessages(\n  [\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE),\n  ]\n);\n\nconst TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\n    ii. Does the AI language use a logical sequence of tools to answer the question?\n    iii. Does the AI language model use the tools in a helpful way?\n    iv. Does the AI language model use too many steps to answer the question?\n    v. Are the appropriate tools used to answer the question?`;\n\nexport const TOOL_FREE_EVAL_CHAT_PROMPT =\n  /* #__PURE__ */ ChatPromptTemplate.fromMessages([\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(\n      TOOL_FREE_EVAL_TEMPLATE\n    ),\n  ]);\n"],"mappings":";;;;;;;AAWA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCtB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CtB,MAAM,iBAAiB;;;;;;;;;;AAWvB,MAAa,mBAAmC,2DAAmB,aACjE;CACkB,oEAA4B,aAC1C,8DACD;CACe,mEAA2B,aAAa,cAAc;CACtD,gEAAwB,aAAa,eAAe;CACpD,mEAA2B,aAAa,cAAc;CACvE,CACF;AAED,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BhC,MAAa,6BACK,2DAAmB,aAAa;CAC9B,oEAA4B,aAC1C,8DACD;CACe,mEAA2B,aAAa,cAAc;CACtD,gEAAwB,aAAa,eAAe;CACpD,mEAA2B,aACzC,wBACD;CACF,CAAC"}
+{"version":3,"file":"prompt.cjs","names":[],"sources":["../../../src/evaluation/agents/prompt.ts"],"sourcesContent":["/**\n * Prompt for trajectory evaluation chain.\n */\n\nimport {\n  AIMessagePromptTemplate,\n  ChatPromptTemplate,\n  HumanMessagePromptTemplate,\n  SystemMessagePromptTemplate,\n} from \"@langchain/core/prompts\";\n\nconst EVAL_TEMPLATE = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\n{toolDescriptions}\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_INPUT = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\nTool 1:\nName: Search\nDescription: useful for when you need to ask with search\n\nTool 2:\nName: Lookup\nDescription: useful for when you need to ask with lookup\n\nTool 3:\nName: Calculator\nDescription: useful for doing calculations\n\nTool 4:\nName: Search the Web (SerpAPI)\nDescription: useful for when you need to answer questions about current events\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\n\n    The AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\nStep 1:\nTool used: Search the Web (SerpAPI)\nTool input: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\nTool output: The Statue of Liberty was given to the United States by France, as a symbol of the two countries' friendship. It was erected atop an American-designed ...\n[END_AGENT_TRAJECTORY]\n\n[RESPONSE]\nThe AI language model's final answer to the question was: There are different ways to measure the length of the United States, but if we use the distance between the Statue of Liberty and the westernmost point of the contiguous United States (Cape Alava, Washington), which is approximately 2,857 miles (4,596 km), and assume that the Statue of Liberty is 305 feet (93 meters) tall, then the statue would stretch across the United States approximately 17.5 times if laid end to end.\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_OUTPUT = `First, let's evaluate the final answer. The final uses good reasoning but is wrong. 2,857 divided by 305 is not 17.5.\\\nThe model should have used the calculator to figure this out. Second does the model use a logical sequence of tools to answer the question?\\\nThe way model uses the search is not helpful. The model should have used the search tool to figure the width of the US or the height of the statue.\\\nThe model didn't use the calculator tool and gave an incorrect answer. The search API should be used for current events or specific questions.\\\nThe tools were not used in a helpful way. The model did not use too many steps to answer the question.\\\nThe model did not use the appropriate tools to answer the question.\\\n\nJudgment: Given the good reasoning in the final answer but otherwise poor performance, we give the model a score of 2.\n\nScore: 2`;\n\nexport const EVAL_CHAT_PROMPT = /* #__PURE__ */ ChatPromptTemplate.fromMessages(\n  [\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE),\n  ]\n);\n\nconst TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\n    ii. Does the AI language use a logical sequence of tools to answer the question?\n    iii. Does the AI language model use the tools in a helpful way?\n    iv. Does the AI language model use too many steps to answer the question?\n    v. Are the appropriate tools used to answer the question?`;\n\nexport const TOOL_FREE_EVAL_CHAT_PROMPT =\n  /* #__PURE__ */ ChatPromptTemplate.fromMessages([\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(\n      TOOL_FREE_EVAL_TEMPLATE\n    ),\n  ]);\n"],"mappings":";;;;;;AAWA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCtB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CtB,MAAM,iBAAiB;;;;;;;;;;AAWvB,MAAa,mBAAmC,wCAAA,mBAAmB,aACjE;CACkB,wCAAA,4BAA4B,aAC1C,8DACD;CACe,wCAAA,2BAA2B,aAAa,cAAc;CACtD,wCAAA,wBAAwB,aAAa,eAAe;CACpD,wCAAA,2BAA2B,aAAa,cAAc;CACvE,CACF;AA6BD,MAAa,6BACK,wCAAA,mBAAmB,aAAa;CAC9B,wCAAA,4BAA4B,aAC1C,8DACD;CACe,wCAAA,2BAA2B,aAAa,cAAc;CACtD,wCAAA,wBAAwB,aAAa,eAAe;CACpD,wCAAA,2BAA2B,aAlCf;;;;;;;;;;;;;;;;;;;;;;;;;+DAoC3B;CACF,CAAC"}

package/dist/evaluation/agents/prompt.js

@@ -1,5 +1,4 @@
 import { AIMessagePromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate } from "@langchain/core/prompts";
-
 //#region src/evaluation/agents/prompt.ts
 /**
 * Prompt for trajectory evaluation chain.
@@ -95,7 +94,11 @@ const EVAL_CHAT_PROMPT = /* @__PURE__ */ ChatPromptTemplate.fromMessages([
 	/* @__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
 	/* @__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE)
 ]);
-const TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.
+const TOOL_FREE_EVAL_CHAT_PROMPT = /* @__PURE__ */ ChatPromptTemplate.fromMessages([
+	/* @__PURE__ */ SystemMessagePromptTemplate.fromTemplate("You are a helpful assistant that evaluates language models."),
+	/* @__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),
+	/* @__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
+	/* @__PURE__ */ HumanMessagePromptTemplate.fromTemplate(`An AI language model has been given access to a set of tools to help answer a user's question.
 
 The question the human asked the AI model was:
 [QUESTION]
@@ -120,14 +123,9 @@ i. Is the final answer helpful?
     ii. Does the AI language use a logical sequence of tools to answer the question?
     iii. Does the AI language model use the tools in a helpful way?
     iv. Does the AI language model use too many steps to answer the question?
-    v. Are the appropriate tools used to answer the question?`;
-const TOOL_FREE_EVAL_CHAT_PROMPT = /* @__PURE__ */ ChatPromptTemplate.fromMessages([
-	/* @__PURE__ */ SystemMessagePromptTemplate.fromTemplate("You are a helpful assistant that evaluates language models."),
-	/* @__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),
-	/* @__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),
-	/* @__PURE__ */ HumanMessagePromptTemplate.fromTemplate(TOOL_FREE_EVAL_TEMPLATE)
+    v. Are the appropriate tools used to answer the question?`)
 ]);
-
 //#endregion
 export { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT };
+
 //# sourceMappingURL=prompt.js.map

package/dist/evaluation/agents/prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/agents/prompt.ts"],"sourcesContent":["/**\n * Prompt for trajectory evaluation chain.\n */\n\nimport {\n  AIMessagePromptTemplate,\n  ChatPromptTemplate,\n  HumanMessagePromptTemplate,\n  SystemMessagePromptTemplate,\n} from \"@langchain/core/prompts\";\n\nconst EVAL_TEMPLATE = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\n{toolDescriptions}\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_INPUT = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\nTool 1:\nName: Search\nDescription: useful for when you need to ask with search\n\nTool 2:\nName: Lookup\nDescription: useful for when you need to ask with lookup\n\nTool 3:\nName: Calculator\nDescription: useful for doing calculations\n\nTool 4:\nName: Search the Web (SerpAPI)\nDescription: useful for when you need to answer questions about current events\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\n\n    The AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\nStep 1:\nTool used: Search the Web (SerpAPI)\nTool input: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\nTool output: The Statue of Liberty was given to the United States by France, as a symbol of the two countries' friendship. It was erected atop an American-designed ...\n[END_AGENT_TRAJECTORY]\n\n[RESPONSE]\nThe AI language model's final answer to the question was: There are different ways to measure the length of the United States, but if we use the distance between the Statue of Liberty and the westernmost point of the contiguous United States (Cape Alava, Washington), which is approximately 2,857 miles (4,596 km), and assume that the Statue of Liberty is 305 feet (93 meters) tall, then the statue would stretch across the United States approximately 17.5 times if laid end to end.\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_OUTPUT = `First, let's evaluate the final answer. The final uses good reasoning but is wrong. 2,857 divided by 305 is not 17.5.\\\nThe model should have used the calculator to figure this out. Second does the model use a logical sequence of tools to answer the question?\\\nThe way model uses the search is not helpful. The model should have used the search tool to figure the width of the US or the height of the statue.\\\nThe model didn't use the calculator tool and gave an incorrect answer. The search API should be used for current events or specific questions.\\\nThe tools were not used in a helpful way. The model did not use too many steps to answer the question.\\\nThe model did not use the appropriate tools to answer the question.\\\n\nJudgment: Given the good reasoning in the final answer but otherwise poor performance, we give the model a score of 2.\n\nScore: 2`;\n\nexport const EVAL_CHAT_PROMPT = /* #__PURE__ */ ChatPromptTemplate.fromMessages(\n  [\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE),\n  ]\n);\n\nconst TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\n    ii. Does the AI language use a logical sequence of tools to answer the question?\n    iii. Does the AI language model use the tools in a helpful way?\n    iv. Does the AI language model use too many steps to answer the question?\n    v. Are the appropriate tools used to answer the question?`;\n\nexport const TOOL_FREE_EVAL_CHAT_PROMPT =\n  /* #__PURE__ */ ChatPromptTemplate.fromMessages([\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(\n      TOOL_FREE_EVAL_TEMPLATE\n    ),\n  ]);\n"],"mappings":";;;;;;AAWA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCtB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CtB,MAAM,iBAAiB;;;;;;;;;;AAWvB,MAAa,mBAAmC,mCAAmB,aACjE;CACkB,4CAA4B,aAC1C,8DACD;CACe,2CAA2B,aAAa,cAAc;CACtD,wCAAwB,aAAa,eAAe;CACpD,2CAA2B,aAAa,cAAc;CACvE,CACF;AAED,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BhC,MAAa,6BACK,mCAAmB,aAAa;CAC9B,4CAA4B,aAC1C,8DACD;CACe,2CAA2B,aAAa,cAAc;CACtD,wCAAwB,aAAa,eAAe;CACpD,2CAA2B,aACzC,wBACD;CACF,CAAC"}
+{"version":3,"file":"prompt.js","names":[],"sources":["../../../src/evaluation/agents/prompt.ts"],"sourcesContent":["/**\n * Prompt for trajectory evaluation chain.\n */\n\nimport {\n  AIMessagePromptTemplate,\n  ChatPromptTemplate,\n  HumanMessagePromptTemplate,\n  SystemMessagePromptTemplate,\n} from \"@langchain/core/prompts\";\n\nconst EVAL_TEMPLATE = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\n{toolDescriptions}\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_INPUT = `An AI language model has been given access to the following set of tools to help answer a user's question.\n\nThe tools given to the AI model are:\n[TOOL_DESCRIPTIONS]\nTool 1:\nName: Search\nDescription: useful for when you need to ask with search\n\nTool 2:\nName: Lookup\nDescription: useful for when you need to ask with lookup\n\nTool 3:\nName: Calculator\nDescription: useful for doing calculations\n\nTool 4:\nName: Search the Web (SerpAPI)\nDescription: useful for when you need to answer questions about current events\n[END_TOOL_DESCRIPTIONS]\n\nThe question the human asked the AI model was: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\n\n    The AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\nStep 1:\nTool used: Search the Web (SerpAPI)\nTool input: If laid the Statue of Liberty end to end, how many times would it stretch across the United States?\nTool output: The Statue of Liberty was given to the United States by France, as a symbol of the two countries' friendship. It was erected atop an American-designed ...\n[END_AGENT_TRAJECTORY]\n\n[RESPONSE]\nThe AI language model's final answer to the question was: There are different ways to measure the length of the United States, but if we use the distance between the Statue of Liberty and the westernmost point of the contiguous United States (Cape Alava, Washington), which is approximately 2,857 miles (4,596 km), and assume that the Statue of Liberty is 305 feet (93 meters) tall, then the statue would stretch across the United States approximately 17.5 times if laid end to end.\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\nii. Does the AI language use a logical sequence of tools to answer the question?\niii. Does the AI language model use the tools in a helpful way?\niv. Does the AI language model use too many steps to answer the question?\nv. Are the appropriate tools used to answer the question?`;\n\nconst EXAMPLE_OUTPUT = `First, let's evaluate the final answer. The final uses good reasoning but is wrong. 2,857 divided by 305 is not 17.5.\\\nThe model should have used the calculator to figure this out. Second does the model use a logical sequence of tools to answer the question?\\\nThe way model uses the search is not helpful. The model should have used the search tool to figure the width of the US or the height of the statue.\\\nThe model didn't use the calculator tool and gave an incorrect answer. The search API should be used for current events or specific questions.\\\nThe tools were not used in a helpful way. The model did not use too many steps to answer the question.\\\nThe model did not use the appropriate tools to answer the question.\\\n\nJudgment: Given the good reasoning in the final answer but otherwise poor performance, we give the model a score of 2.\n\nScore: 2`;\n\nexport const EVAL_CHAT_PROMPT = /* #__PURE__ */ ChatPromptTemplate.fromMessages(\n  [\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EVAL_TEMPLATE),\n  ]\n);\n\nconst TOOL_FREE_EVAL_TEMPLATE = `An AI language model has been given access to a set of tools to help answer a user's question.\n\nThe question the human asked the AI model was:\n[QUESTION]\n{question}\n[END_QUESTION]{reference}\n\nThe AI language model decided to use the following set of tools to answer the question:\n[AGENT_TRAJECTORY]\n{agentTrajectory}\n[END_AGENT_TRAJECTORY]\n\nThe AI language model's final answer to the question was:\n[RESPONSE]\n{answer}\n[END_RESPONSE]\n\nLet's do a detailed evaluation of the AI language model's answer step by step.\n\nWe consider the following criteria before giving a score from 1 to 5:\n\ni. Is the final answer helpful?\n    ii. Does the AI language use a logical sequence of tools to answer the question?\n    iii. Does the AI language model use the tools in a helpful way?\n    iv. Does the AI language model use too many steps to answer the question?\n    v. Are the appropriate tools used to answer the question?`;\n\nexport const TOOL_FREE_EVAL_CHAT_PROMPT =\n  /* #__PURE__ */ ChatPromptTemplate.fromMessages([\n    /* #__PURE__ */ SystemMessagePromptTemplate.fromTemplate(\n      \"You are a helpful assistant that evaluates language models.\"\n    ),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(EXAMPLE_INPUT),\n    /* #__PURE__ */ AIMessagePromptTemplate.fromTemplate(EXAMPLE_OUTPUT),\n    /* #__PURE__ */ HumanMessagePromptTemplate.fromTemplate(\n      TOOL_FREE_EVAL_TEMPLATE\n    ),\n  ]);\n"],"mappings":";;;;;AAWA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCtB,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6CtB,MAAM,iBAAiB;;;;;;;;;;AAWvB,MAAa,mBAAmC,mCAAmB,aACjE;CACkB,4CAA4B,aAC1C,8DACD;CACe,2CAA2B,aAAa,cAAc;CACtD,wCAAwB,aAAa,eAAe;CACpD,2CAA2B,aAAa,cAAc;CACvE,CACF;AA6BD,MAAa,6BACK,mCAAmB,aAAa;CAC9B,4CAA4B,aAC1C,8DACD;CACe,2CAA2B,aAAa,cAAc;CACtD,wCAAwB,aAAa,eAAe;CACpD,2CAA2B,aAlCf;;;;;;;;;;;;;;;;;;;;;;;;;+DAoC3B;CACF,CAAC"}

package/dist/evaluation/agents/trajectory.cjs

@@ -1,9 +1,8 @@
-const require_runtime = require('../../_virtual/_rolldown/runtime.cjs');
-const require_base = require('../base.cjs');
-const require_prompt = require('./prompt.cjs');
+require("../../_virtual/_rolldown/runtime.cjs");
+const require_base = require("../base.cjs");
+const require_prompt = require("./prompt.cjs");
 let _langchain_core_outputs = require("@langchain/core/outputs");
 let _langchain_core_output_parsers = require("@langchain/core/output_parsers");
-
 //#region src/evaluation/agents/trajectory.ts
 /**
 * A parser for the output of the TrajectoryEvalChain.
@@ -125,8 +124,8 @@ ${reference}
 		return this._prepareOutput(result);
 	}
 };
-
 //#endregion
 exports.TrajectoryEvalChain = TrajectoryEvalChain;
 exports.TrajectoryOutputParser = TrajectoryOutputParser;
+
 //# sourceMappingURL=trajectory.cjs.map

package/dist/evaluation/agents/trajectory.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.cjs","names":["BaseLLMOutputParser","AgentTrajectoryEvaluator","EVAL_CHAT_PROMPT","TOOL_FREE_EVAL_CHAT_PROMPT","RUN_KEY"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;;AAwBA,IAAa,yBAAb,cAA4CA,mDAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyCC,sCAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAUC;MAEV,WAAUC;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,mCAAW,UAAU,OAAOA,iCAC9B,QAAOA,mCAAW,OAAOA;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}
+{"version":3,"file":"trajectory.cjs","names":["BaseLLMOutputParser","AgentTrajectoryEvaluator","EVAL_CHAT_PROMPT","TOOL_FREE_EVAL_CHAT_PROMPT","RUN_KEY"],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4CA,+BAAAA,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyCC,aAAAA,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAUC,eAAAA;MAEV,WAAUC,eAAAA;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}

package/dist/evaluation/agents/trajectory.js

@@ -2,7 +2,6 @@ import { AgentTrajectoryEvaluator } from "../base.js";
 import { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from "./prompt.js";
 import { RUN_KEY } from "@langchain/core/outputs";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-
 //#region src/evaluation/agents/trajectory.ts
 /**
 * A parser for the output of the TrajectoryEvalChain.
@@ -124,7 +123,7 @@ ${reference}
 		return this._prepareOutput(result);
 	}
 };
-
 //#endregion
 export { TrajectoryEvalChain, TrajectoryOutputParser };
+
 //# sourceMappingURL=trajectory.js.map

package/dist/evaluation/agents/trajectory.js.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.js","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAU;MAEV,WAAU;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}
+{"version":3,"file":"trajectory.js","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"sourcesContent":["import type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\n\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  AgentTrajectoryEvaluator,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMTrajectoryEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\nimport { EVAL_CHAT_PROMPT, TOOL_FREE_EVAL_CHAT_PROMPT } from \"./prompt.js\";\n\n/**\n * A parser for the output of the TrajectoryEvalChain.\n */\nexport class TrajectoryOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"TrajectoryOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"agents\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    if (!text.includes(\"Score:\")) {\n      throw new Error(`Could not find score in model eval output: ${text}`);\n    }\n\n    let [reasoning, scoreStr] = text.split(\"Score:\", 2);\n    reasoning = reasoning.trim();\n    scoreStr = scoreStr.trim();\n\n    // Use regex to extract the score.\n    // This will get the number in the string, even if it is a float or more than 10.\n    // E.g. \"Score: 1\" will return 1, \"Score: 3.5\" will return 3.5, and\n    // \"Score: 10\" will return 10.\n    // The score should be an integer digit in the range 1-5.\n\n    const scoreMatch = scoreStr.match(/(\\d+(\\.\\d+)?)/);\n    if (scoreMatch === null || scoreMatch[1].includes(\".\")) {\n      throw new Error(\n        `Score is not an integer digit in the range 1-5: ${text}`\n      );\n    }\n\n    const score = +scoreMatch[1];\n    if (score < 1 || score > 5) {\n      throw new Error(`Score is not a digit in the range 1-5: ${text}`);\n    }\n\n    const normalizedScore = (score - 1) / 4;\n\n    return Promise.resolve({\n      reasoning,\n      score: normalizedScore,\n    });\n  }\n}\n\n/**\n * A chain for evaluating ReAct style agents.\n *\n * This chain is used to evaluate ReAct style agents by reasoning about\n * the sequence of actions taken and their outcomes.\n */\nexport class TrajectoryEvalChain extends AgentTrajectoryEvaluator {\n  static lc_name(): string {\n    return \"TrajectoryEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  outputParser = new TrajectoryOutputParser();\n\n  static resolveTrajectoryPrompt(\n    prompt?: BasePromptTemplate | undefined,\n    agentTools?: StructuredToolInterface[]\n  ) {\n    let _prompt;\n    if (prompt) {\n      _prompt = prompt;\n    } else if (agentTools) {\n      _prompt = EVAL_CHAT_PROMPT;\n    } else {\n      _prompt = TOOL_FREE_EVAL_CHAT_PROMPT;\n    }\n\n    return _prompt;\n  }\n\n  /**\n   * Get the description of the agent tools.\n   *\n   * @returns The description of the agent tools.\n   */\n  static toolsDescription(agentTools: StructuredToolInterface[]): string {\n    return agentTools\n      .map(\n        (tool, i) =>\n          `Tool ${i + 1}: ${tool.name}\\n Description: ${tool.description}`\n      )\n      .join(\"\\n\\n\");\n  }\n\n  /**\n   * Create a new TrajectoryEvalChain.\n   * @param llm\n   * @param agentTools - The tools used by the agent.\n   * @param chainOptions - The options for the chain.\n   */\n  static async fromLLM(\n    llm: BaseChatModel,\n    agentTools?: StructuredToolInterface[],\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolveTrajectoryPrompt(chainOptions?.prompt, agentTools);\n    if (agentTools) {\n      const toolDescriptions = this.toolsDescription(agentTools);\n      prompt = await prompt.partial({ toolDescriptions });\n    }\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  /**\n   * Get the agent trajectory as a formatted string.\n   *\n   * @param steps - The agent trajectory.\n   * @returns The formatted agent trajectory.\n   */\n  getAgentTrajectory(steps: AgentStep[]): string {\n    return steps\n      .map((step, i) => {\n        const { action, observation } = step;\n\n        return (\n          `Step ${i + 1}:\\n` +\n          `Tool used: ${action.tool}\\n` +\n          `Tool input: ${action.toolInput}\\n` +\n          `Tool output: ${observation}`\n        );\n      })\n      .join(\"\\n\\n\");\n  }\n\n  formatReference(reference?: string): string {\n    if (!reference) {\n      return \"\";\n    }\n    return `\nThe following is the expected answer. Use this to measure correctness:\n[GROUND_TRUTH]\n${reference}\n[END_GROUND_TRUTH]\n        `;\n  }\n\n  async _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const { input, prediction, reference, agentTrajectory } = args;\n\n    const inputs = {\n      question: input,\n      agentTrajectory: this.getAgentTrajectory(agentTrajectory),\n      answer: prediction,\n      reference: this.formatReference(reference),\n    };\n\n    const result = await this.call({ ...inputs, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n"],"mappings":";;;;;;;;AAwBA,IAAa,yBAAb,cAA4C,oBAAoC;CAC9E,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAS;CAEpD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;AAE7B,MAAI,CAAC,KAAK,SAAS,SAAS,CAC1B,OAAM,IAAI,MAAM,8CAA8C,OAAO;EAGvE,IAAI,CAAC,WAAW,YAAY,KAAK,MAAM,UAAU,EAAE;AACnD,cAAY,UAAU,MAAM;AAC5B,aAAW,SAAS,MAAM;EAQ1B,MAAM,aAAa,SAAS,MAAM,gBAAgB;AAClD,MAAI,eAAe,QAAQ,WAAW,GAAG,SAAS,IAAI,CACpD,OAAM,IAAI,MACR,mDAAmD,OACpD;EAGH,MAAM,QAAQ,CAAC,WAAW;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EACvB,OAAM,IAAI,MAAM,0CAA0C,OAAO;EAGnE,MAAM,mBAAmB,QAAQ,KAAK;AAEtC,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACR,CAAC;;;;;;;;;AAUN,IAAa,sBAAb,cAAyC,yBAAyB;CAChE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,eAAe,IAAI,wBAAwB;CAE3C,OAAO,wBACL,QACA,YACA;EACA,IAAI;AACJ,MAAI,OACF,WAAU;WACD,WACT,WAAU;MAEV,WAAU;AAGZ,SAAO;;;;;;;CAQT,OAAO,iBAAiB,YAA+C;AACrE,SAAO,WACJ,KACE,MAAM,MACL,QAAQ,IAAI,EAAE,IAAI,KAAK,KAAK,kBAAkB,KAAK,cACtD,CACA,KAAK,OAAO;;;;;;;;CASjB,aAAa,QACX,KACA,YACA,cACA;EACA,IAAI,SAAS,KAAK,wBAAwB,cAAc,QAAQ,WAAW;AAC3E,MAAI,YAAY;GACd,MAAM,mBAAmB,KAAK,iBAAiB,WAAW;AAC1D,YAAS,MAAM,OAAO,QAAQ,EAAE,kBAAkB,CAAC;;EAGrD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;;;;;;;CAST,mBAAmB,OAA4B;AAC7C,SAAO,MACJ,KAAK,MAAM,MAAM;GAChB,MAAM,EAAE,QAAQ,gBAAgB;AAEhC,UACE,QAAQ,IAAI,EAAE,gBACA,OAAO,KAAK,gBACX,OAAO,UAAU,iBAChB;IAElB,CACD,KAAK,OAAO;;CAGjB,gBAAgB,WAA4B;AAC1C,MAAI,CAAC,UACH,QAAO;AAET,SAAO;;;EAGT,UAAU;;;;CAKV,MAAM,yBACJ,MACA,aACA,QACsB;EACtB,MAAM,EAAE,OAAO,YAAY,WAAW,oBAAoB;EAE1D,MAAM,SAAS;GACb,UAAU;GACV,iBAAiB,KAAK,mBAAmB,gBAAgB;GACzD,QAAQ;GACR,WAAW,KAAK,gBAAgB,UAAU;GAC3C;EAED,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAQ,GAAG;GAAa,EAAE,OAAO;AAErE,SAAO,KAAK,eAAe,OAAO"}

package/dist/evaluation/base.cjs

@@ -1,7 +1,6 @@
-const require_base = require('../chains/base.cjs');
-const require_llm_chain = require('../chains/llm_chain.cjs');
-require('../chains/index.cjs');
-
+const require_base = require("../chains/base.cjs");
+const require_llm_chain = require("../chains/llm_chain.cjs");
+require("../chains/index.cjs");
 //#region src/evaluation/base.ts
 /**
 * Compare two sets for equality
@@ -158,7 +157,6 @@ var AgentTrajectoryEvaluator = class extends LLMEvalChain {
 		return this._evaluateAgentTrajectory(args, callOptions, config);
 	}
 };
-
 //#endregion
 exports.AgentTrajectoryEvaluator = AgentTrajectoryEvaluator;
 exports.LLMPairwiseStringEvaluator = LLMPairwiseStringEvaluator;
@@ -166,4 +164,5 @@ exports.LLMStringEvaluator = LLMStringEvaluator;
 exports.PairwiseStringEvaluator = PairwiseStringEvaluator;
 exports.StringEvaluator = StringEvaluator;
 exports.eqSet = eqSet;
+
 //# sourceMappingURL=base.cjs.map

package/dist/evaluation/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,2BAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,uBAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}
+{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,kBAAAA,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,aAAAA,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}

package/dist/evaluation/base.d.ts

@@ -1,6 +1,5 @@
 import { BaseChain } from "../chains/base.js";
 import { LLMChain, LLMChainInput } from "../chains/llm_chain.js";
-import "../chains/index.js";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { ChainValues } from "@langchain/core/utils/types";

package/dist/evaluation/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/base.js

@@ -1,7 +1,6 @@
 import { BaseChain } from "../chains/base.js";
 import { LLMChain } from "../chains/llm_chain.js";
 import "../chains/index.js";
-
 //#region src/evaluation/base.ts
 /**
 * Compare two sets for equality
@@ -158,7 +157,7 @@ var AgentTrajectoryEvaluator = class extends LLMEvalChain {
 		return this._evaluateAgentTrajectory(args, callOptions, config);
 	}
 };
-
 //#endregion
 export { AgentTrajectoryEvaluator, LLMPairwiseStringEvaluator, LLMStringEvaluator, PairwiseStringEvaluator, StringEvaluator, eqSet };
+
 //# sourceMappingURL=base.js.map