@langchain/classic 1.0.15 → 1.0.17

package/dist/retrievers/time_weighted.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"time_weighted.cjs","names":["BaseRetriever","fields: TimeWeightedVectorStoreRetrieverFields","memoryStream: DocumentInterface[]","query: string","runManager?: CallbackManagerForRetrieverRun","docs: DocumentInterface[]","memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    >","docAndScores: [DocumentInterface, number][]","results: Record<number, { doc: DocumentInterface; score: number }>","docsAndScores: Record<number, { doc: DocumentInterface; score: number }>","now: number","results: DocumentInterface[]","doc: DocumentInterface","vectorRelevance: number | null","nowMsec: number","time: number","refTime: number"],"sources":["../../src/retrievers/time_weighted.ts"],"sourcesContent":["import { BaseRetriever, BaseRetrieverInput } from \"@langchain/core/retrievers\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * Interface for the fields required to initialize a\n * TimeWeightedVectorStoreRetriever instance.\n */\nexport interface TimeWeightedVectorStoreRetrieverFields\n  extends BaseRetrieverInput {\n  vectorStore: VectorStoreInterface;\n  searchKwargs?: number;\n  memoryStream?: DocumentInterface[];\n  decayRate?: number;\n  k?: number;\n  otherScoreKeys?: string[];\n  defaultSalience?: number;\n}\n\nexport const LAST_ACCESSED_AT_KEY = \"last_accessed_at\";\nexport const BUFFER_IDX = \"buffer_idx\";\n\n/**\n * TimeWeightedVectorStoreRetriever retrieves documents based on their time-weighted relevance.\n * ref: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/retrievers/time_weighted_retriever.py\n * @example\n * ```typescript\n * const retriever = new TimeWeightedVectorStoreRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   memoryStream: [],\n *   searchKwargs: 2,\n * });\n * await retriever.addDocuments([\n *   { pageContent: \"My name is John.\", metadata: {} },\n *   { pageContent: \"My favourite food is pizza.\", metadata: {} },\n *\n * ]);\n * const results = await retriever.invoke(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class TimeWeightedVectorStoreRetriever extends BaseRetriever {\n  static lc_name() {\n    return \"TimeWeightedVectorStoreRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"time_weighted\"];\n  }\n\n  /**\n   * The vectorstore to store documents and determine salience.\n   */\n  private vectorStore: VectorStoreInterface;\n\n  /**\n   * The number of top K most relevant documents to consider when searching.\n   */\n  private searchKwargs: number;\n\n  /**\n   * The memory_stream of documents to search through.\n   */\n  private memoryStream: DocumentInterface[];\n\n  /**\n   * The exponential decay factor used as (1.0-decay_rate)**(hrs_passed).\n   */\n  private decayRate: number;\n\n  /**\n   * The maximum number of documents to retrieve in a given call.\n   */\n  private k: number;\n\n  /**\n   * Other keys in the metadata to factor into the score, e.g. 'importance'.\n   */\n  private otherScoreKeys: string[];\n\n  /**\n   * The salience to assign memories not retrieved from the vector store.\n   */\n  private defaultSalience: number | null;\n\n  /**\n   * Constructor to initialize the required fields\n   * @param fields - The fields required for initializing the TimeWeightedVectorStoreRetriever\n   */\n  constructor(fields: TimeWeightedVectorStoreRetrieverFields) {\n    super(fields);\n    this.vectorStore = fields.vectorStore;\n    this.searchKwargs = fields.searchKwargs ?? 100;\n    this.memoryStream = fields.memoryStream ?? [];\n    this.decayRate = fields.decayRate ?? 0.01;\n    this.k = fields.k ?? 4;\n    this.otherScoreKeys = fields.otherScoreKeys ?? [];\n    this.defaultSalience = fields.defaultSalience ?? null;\n  }\n\n  /**\n   * Get the memory stream of documents.\n   * @returns The memory stream of documents.\n   */\n  getMemoryStream(): DocumentInterface[] {\n    return this.memoryStream;\n  }\n\n  /**\n   * Set the memory stream of documents.\n   * @param memoryStream The new memory stream of documents.\n   */\n  setMemoryStream(memoryStream: DocumentInterface[]) {\n    this.memoryStream = memoryStream;\n  }\n\n  /**\n   * Get relevant documents based on time-weighted relevance\n   * @param query - The query to search for\n   * @returns The relevant documents\n   */\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<DocumentInterface[]> {\n    const now = Math.floor(Date.now() / 1000);\n    const memoryDocsAndScores = this.getMemoryDocsAndScores();\n\n    const salientDocsAndScores = await this.getSalientDocuments(\n      query,\n      runManager\n    );\n    const docsAndScores = { ...memoryDocsAndScores, ...salientDocsAndScores };\n\n    return this.computeResults(docsAndScores, now);\n  }\n\n  /**\n   * NOTE: When adding documents to a vector store, use addDocuments\n   * via retriever instead of directly to the vector store.\n   * This is because it is necessary to process the document\n   * in prepareDocuments.\n   *\n   * @param docs - The documents to add to vector store in the retriever\n   */\n  async addDocuments(docs: DocumentInterface[]): Promise<void> {\n    const now = Math.floor(Date.now() / 1000);\n    const savedDocs = this.prepareDocuments(docs, now);\n\n    this.memoryStream.push(...savedDocs);\n    await this.vectorStore.addDocuments(savedDocs);\n  }\n\n  /**\n   * Get memory documents and their scores\n   * @returns An object containing memory documents and their scores\n   */\n  private getMemoryDocsAndScores(): Record<\n    number,\n    { doc: DocumentInterface; score: number }\n  > {\n    const memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    > = {};\n    for (const doc of this.memoryStream.slice(-this.k)) {\n      const bufferIdx = doc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      memoryDocsAndScores[bufferIdx] = {\n        doc,\n        score: this.defaultSalience ?? 0,\n      };\n    }\n    return memoryDocsAndScores;\n  }\n\n  /**\n   * Get salient documents and their scores based on the query\n   * @param query - The query to search for\n   * @returns An object containing salient documents and their scores\n   */\n  private async getSalientDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Record<number, { doc: DocumentInterface; score: number }>> {\n    const docAndScores: [DocumentInterface, number][] =\n      await this.vectorStore.similaritySearchWithScore(\n        query,\n        this.searchKwargs,\n        undefined,\n        runManager?.getChild()\n      );\n    const results: Record<number, { doc: DocumentInterface; score: number }> =\n      {};\n    for (const [fetchedDoc, score] of docAndScores) {\n      const bufferIdx = fetchedDoc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      const doc = this.memoryStream[bufferIdx];\n      results[bufferIdx] = { doc, score };\n    }\n    return results;\n  }\n\n  /**\n   * Compute the final result set of documents based on the combined scores\n   * @param docsAndScores - An object containing documents and their scores\n   * @param now - The current timestamp\n   * @returns The final set of documents\n   */\n  private computeResults(\n    docsAndScores: Record<number, { doc: DocumentInterface; score: number }>,\n    now: number\n  ): DocumentInterface[] {\n    const recordedDocs = Object.values(docsAndScores)\n      .map(({ doc, score }) => ({\n        doc,\n        score: this.getCombinedScore(doc, score, now),\n      }))\n      .sort((a, b) => b.score - a.score);\n\n    const results: DocumentInterface[] = [];\n    for (const { doc } of recordedDocs) {\n      const bufferedDoc = this.memoryStream[doc.metadata[BUFFER_IDX]];\n      bufferedDoc.metadata[LAST_ACCESSED_AT_KEY] = now;\n      results.push(bufferedDoc);\n      if (results.length > this.k) {\n        break;\n      }\n    }\n    return results;\n  }\n\n  /**\n   * Prepare documents with necessary metadata before saving\n   * @param docs - The documents to prepare\n   * @param now - The current timestamp\n   * @returns The prepared documents\n   */\n  private prepareDocuments(\n    docs: DocumentInterface[],\n    now: number\n  ): DocumentInterface[] {\n    return docs.map((doc, i) => ({\n      ...doc,\n      metadata: {\n        ...doc.metadata,\n        [LAST_ACCESSED_AT_KEY]: doc.metadata[LAST_ACCESSED_AT_KEY] ?? now,\n        created_at: doc.metadata.created_at ?? now,\n        [BUFFER_IDX]: this.memoryStream.length + i,\n      },\n    }));\n  }\n\n  /**\n   * Calculate the combined score based on vector relevance and other factors\n   * @param doc - The document to calculate the score for\n   * @param vectorRelevance - The relevance score from the vector store\n   * @param nowMsec - The current timestamp in milliseconds\n   * @returns The combined score for the document\n   */\n  private getCombinedScore(\n    doc: DocumentInterface,\n    vectorRelevance: number | null,\n    nowMsec: number\n  ): number {\n    const hoursPassed = this.getHoursPassed(\n      nowMsec,\n      doc.metadata[LAST_ACCESSED_AT_KEY]\n    );\n    let score = (1.0 - this.decayRate) ** hoursPassed;\n    for (const key of this.otherScoreKeys) {\n      score += doc.metadata[key];\n    }\n    if (vectorRelevance !== null) {\n      score += vectorRelevance;\n    }\n    return score;\n  }\n\n  /**\n   * Calculate the hours passed between two time points\n   * @param time - The current time in seconds\n   * @param refTime - The reference time in seconds\n   * @returns The number of hours passed between the two time points\n   */\n  private getHoursPassed(time: number, refTime: number): number {\n    return (time - refTime) / 3600;\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,MAAa,uBAAuB;AACpC,MAAa,aAAa;;;;;;;;;;;;;;;;;;;;;AAsB1B,IAAa,mCAAb,cAAsDA,0CAAc;CAClE,OAAO,UAAU;AACf,SAAO;CACR;CAED,IAAI,eAAe;AACjB,SAAO;GAAC;GAAa;GAAc;EAAgB;CACpD;;;;CAKD,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;;CAMR,YAAYC,QAAgD;EAC1D,MAAM,OAAO;EACb,KAAK,cAAc,OAAO;EAC1B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,eAAe,OAAO,gBAAgB,CAAE;EAC7C,KAAK,YAAY,OAAO,aAAa;EACrC,KAAK,IAAI,OAAO,KAAK;EACrB,KAAK,iBAAiB,OAAO,kBAAkB,CAAE;EACjD,KAAK,kBAAkB,OAAO,mBAAmB;CAClD;;;;;CAMD,kBAAuC;AACrC,SAAO,KAAK;CACb;;;;;CAMD,gBAAgBC,cAAmC;EACjD,KAAK,eAAe;CACrB;;;;;;CAOD,MAAM,sBACJC,OACAC,YAC8B;EAC9B,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,sBAAsB,KAAK,wBAAwB;EAEzD,MAAM,uBAAuB,MAAM,KAAK,oBACtC,OACA,WACD;EACD,MAAM,gBAAgB;GAAE,GAAG;GAAqB,GAAG;EAAsB;AAEzE,SAAO,KAAK,eAAe,eAAe,IAAI;CAC/C;;;;;;;;;CAUD,MAAM,aAAaC,MAA0C;EAC3D,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,YAAY,KAAK,iBAAiB,MAAM,IAAI;EAElD,KAAK,aAAa,KAAK,GAAG,UAAU;EACpC,MAAM,KAAK,YAAY,aAAa,UAAU;CAC/C;;;;;CAMD,AAAQ,yBAGN;EACA,MAAMC,sBAGF,CAAE;AACN,OAAK,MAAM,OAAO,KAAK,aAAa,MAAM,CAAC,KAAK,EAAE,EAAE;GAClD,MAAM,YAAY,IAAI,SAAS;AAC/B,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,oBAAoB,aAAa;IAC/B;IACA,OAAO,KAAK,mBAAmB;GAChC;EACF;AACD,SAAO;CACR;;;;;;CAOD,MAAc,oBACZH,OACAC,YACoE;EACpE,MAAMG,eACJ,MAAM,KAAK,YAAY,0BACrB,OACA,KAAK,cACL,QACA,YAAY,UAAU,CACvB;EACH,MAAMC,UACJ,CAAE;AACJ,OAAK,MAAM,CAAC,YAAY,MAAM,IAAI,cAAc;GAC9C,MAAM,YAAY,WAAW,SAAS;AACtC,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,MAAM,MAAM,KAAK,aAAa;GAC9B,QAAQ,aAAa;IAAE;IAAK;GAAO;EACpC;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,eACNC,eACAC,KACqB;EACrB,MAAM,eAAe,OAAO,OAAO,cAAc,CAC9C,IAAI,CAAC,EAAE,KAAK,OAAO,MAAM;GACxB;GACA,OAAO,KAAK,iBAAiB,KAAK,OAAO,IAAI;EAC9C,GAAE,CACF,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;EAEpC,MAAMC,UAA+B,CAAE;AACvC,OAAK,MAAM,EAAE,KAAK,IAAI,cAAc;GAClC,MAAM,cAAc,KAAK,aAAa,IAAI,SAAS;GACnD,YAAY,SAAS,wBAAwB;GAC7C,QAAQ,KAAK,YAAY;AACzB,OAAI,QAAQ,SAAS,KAAK,EACxB;EAEH;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,iBACNN,MACAK,KACqB;AACrB,SAAO,KAAK,IAAI,CAAC,KAAK,OAAO;GAC3B,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,uBAAuB,IAAI,SAAS,yBAAyB;IAC9D,YAAY,IAAI,SAAS,cAAc;KACtC,aAAa,KAAK,aAAa,SAAS;GAC1C;EACF,GAAE;CACJ;;;;;;;;CASD,AAAQ,iBACNE,KACAC,iBACAC,SACQ;EACR,MAAM,cAAc,KAAK,eACvB,SACA,IAAI,SAAS,sBACd;EACD,IAAI,SAAS,IAAM,KAAK,cAAc;AACtC,OAAK,MAAM,OAAO,KAAK,gBACrB,SAAS,IAAI,SAAS;AAExB,MAAI,oBAAoB,MACtB,SAAS;AAEX,SAAO;CACR;;;;;;;CAQD,AAAQ,eAAeC,MAAcC,SAAyB;AAC5D,UAAQ,OAAO,WAAW;CAC3B;AACF"}
+{"version":3,"file":"time_weighted.cjs","names":["BaseRetriever","fields: TimeWeightedVectorStoreRetrieverFields","memoryStream: DocumentInterface[]","query: string","runManager?: CallbackManagerForRetrieverRun","docs: DocumentInterface[]","memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    >","docAndScores: [DocumentInterface, number][]","results: Record<number, { doc: DocumentInterface; score: number }>","docsAndScores: Record<number, { doc: DocumentInterface; score: number }>","now: number","results: DocumentInterface[]","doc: DocumentInterface","vectorRelevance: number | null","nowMsec: number","time: number","refTime: number"],"sources":["../../src/retrievers/time_weighted.ts"],"sourcesContent":["import { BaseRetriever, BaseRetrieverInput } from \"@langchain/core/retrievers\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * Interface for the fields required to initialize a\n * TimeWeightedVectorStoreRetriever instance.\n */\nexport interface TimeWeightedVectorStoreRetrieverFields extends BaseRetrieverInput {\n  vectorStore: VectorStoreInterface;\n  searchKwargs?: number;\n  memoryStream?: DocumentInterface[];\n  decayRate?: number;\n  k?: number;\n  otherScoreKeys?: string[];\n  defaultSalience?: number;\n}\n\nexport const LAST_ACCESSED_AT_KEY = \"last_accessed_at\";\nexport const BUFFER_IDX = \"buffer_idx\";\n\n/**\n * TimeWeightedVectorStoreRetriever retrieves documents based on their time-weighted relevance.\n * ref: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/retrievers/time_weighted_retriever.py\n * @example\n * ```typescript\n * const retriever = new TimeWeightedVectorStoreRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   memoryStream: [],\n *   searchKwargs: 2,\n * });\n * await retriever.addDocuments([\n *   { pageContent: \"My name is John.\", metadata: {} },\n *   { pageContent: \"My favourite food is pizza.\", metadata: {} },\n *\n * ]);\n * const results = await retriever.invoke(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class TimeWeightedVectorStoreRetriever extends BaseRetriever {\n  static lc_name() {\n    return \"TimeWeightedVectorStoreRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"time_weighted\"];\n  }\n\n  /**\n   * The vectorstore to store documents and determine salience.\n   */\n  private vectorStore: VectorStoreInterface;\n\n  /**\n   * The number of top K most relevant documents to consider when searching.\n   */\n  private searchKwargs: number;\n\n  /**\n   * The memory_stream of documents to search through.\n   */\n  private memoryStream: DocumentInterface[];\n\n  /**\n   * The exponential decay factor used as (1.0-decay_rate)**(hrs_passed).\n   */\n  private decayRate: number;\n\n  /**\n   * The maximum number of documents to retrieve in a given call.\n   */\n  private k: number;\n\n  /**\n   * Other keys in the metadata to factor into the score, e.g. 'importance'.\n   */\n  private otherScoreKeys: string[];\n\n  /**\n   * The salience to assign memories not retrieved from the vector store.\n   */\n  private defaultSalience: number | null;\n\n  /**\n   * Constructor to initialize the required fields\n   * @param fields - The fields required for initializing the TimeWeightedVectorStoreRetriever\n   */\n  constructor(fields: TimeWeightedVectorStoreRetrieverFields) {\n    super(fields);\n    this.vectorStore = fields.vectorStore;\n    this.searchKwargs = fields.searchKwargs ?? 100;\n    this.memoryStream = fields.memoryStream ?? [];\n    this.decayRate = fields.decayRate ?? 0.01;\n    this.k = fields.k ?? 4;\n    this.otherScoreKeys = fields.otherScoreKeys ?? [];\n    this.defaultSalience = fields.defaultSalience ?? null;\n  }\n\n  /**\n   * Get the memory stream of documents.\n   * @returns The memory stream of documents.\n   */\n  getMemoryStream(): DocumentInterface[] {\n    return this.memoryStream;\n  }\n\n  /**\n   * Set the memory stream of documents.\n   * @param memoryStream The new memory stream of documents.\n   */\n  setMemoryStream(memoryStream: DocumentInterface[]) {\n    this.memoryStream = memoryStream;\n  }\n\n  /**\n   * Get relevant documents based on time-weighted relevance\n   * @param query - The query to search for\n   * @returns The relevant documents\n   */\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<DocumentInterface[]> {\n    const now = Math.floor(Date.now() / 1000);\n    const memoryDocsAndScores = this.getMemoryDocsAndScores();\n\n    const salientDocsAndScores = await this.getSalientDocuments(\n      query,\n      runManager\n    );\n    const docsAndScores = { ...memoryDocsAndScores, ...salientDocsAndScores };\n\n    return this.computeResults(docsAndScores, now);\n  }\n\n  /**\n   * NOTE: When adding documents to a vector store, use addDocuments\n   * via retriever instead of directly to the vector store.\n   * This is because it is necessary to process the document\n   * in prepareDocuments.\n   *\n   * @param docs - The documents to add to vector store in the retriever\n   */\n  async addDocuments(docs: DocumentInterface[]): Promise<void> {\n    const now = Math.floor(Date.now() / 1000);\n    const savedDocs = this.prepareDocuments(docs, now);\n\n    this.memoryStream.push(...savedDocs);\n    await this.vectorStore.addDocuments(savedDocs);\n  }\n\n  /**\n   * Get memory documents and their scores\n   * @returns An object containing memory documents and their scores\n   */\n  private getMemoryDocsAndScores(): Record<\n    number,\n    { doc: DocumentInterface; score: number }\n  > {\n    const memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    > = {};\n    for (const doc of this.memoryStream.slice(-this.k)) {\n      const bufferIdx = doc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      memoryDocsAndScores[bufferIdx] = {\n        doc,\n        score: this.defaultSalience ?? 0,\n      };\n    }\n    return memoryDocsAndScores;\n  }\n\n  /**\n   * Get salient documents and their scores based on the query\n   * @param query - The query to search for\n   * @returns An object containing salient documents and their scores\n   */\n  private async getSalientDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Record<number, { doc: DocumentInterface; score: number }>> {\n    const docAndScores: [DocumentInterface, number][] =\n      await this.vectorStore.similaritySearchWithScore(\n        query,\n        this.searchKwargs,\n        undefined,\n        runManager?.getChild()\n      );\n    const results: Record<number, { doc: DocumentInterface; score: number }> =\n      {};\n    for (const [fetchedDoc, score] of docAndScores) {\n      const bufferIdx = fetchedDoc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      const doc = this.memoryStream[bufferIdx];\n      results[bufferIdx] = { doc, score };\n    }\n    return results;\n  }\n\n  /**\n   * Compute the final result set of documents based on the combined scores\n   * @param docsAndScores - An object containing documents and their scores\n   * @param now - The current timestamp\n   * @returns The final set of documents\n   */\n  private computeResults(\n    docsAndScores: Record<number, { doc: DocumentInterface; score: number }>,\n    now: number\n  ): DocumentInterface[] {\n    const recordedDocs = Object.values(docsAndScores)\n      .map(({ doc, score }) => ({\n        doc,\n        score: this.getCombinedScore(doc, score, now),\n      }))\n      .sort((a, b) => b.score - a.score);\n\n    const results: DocumentInterface[] = [];\n    for (const { doc } of recordedDocs) {\n      const bufferedDoc = this.memoryStream[doc.metadata[BUFFER_IDX]];\n      bufferedDoc.metadata[LAST_ACCESSED_AT_KEY] = now;\n      results.push(bufferedDoc);\n      if (results.length > this.k) {\n        break;\n      }\n    }\n    return results;\n  }\n\n  /**\n   * Prepare documents with necessary metadata before saving\n   * @param docs - The documents to prepare\n   * @param now - The current timestamp\n   * @returns The prepared documents\n   */\n  private prepareDocuments(\n    docs: DocumentInterface[],\n    now: number\n  ): DocumentInterface[] {\n    return docs.map((doc, i) => ({\n      ...doc,\n      metadata: {\n        ...doc.metadata,\n        [LAST_ACCESSED_AT_KEY]: doc.metadata[LAST_ACCESSED_AT_KEY] ?? now,\n        created_at: doc.metadata.created_at ?? now,\n        [BUFFER_IDX]: this.memoryStream.length + i,\n      },\n    }));\n  }\n\n  /**\n   * Calculate the combined score based on vector relevance and other factors\n   * @param doc - The document to calculate the score for\n   * @param vectorRelevance - The relevance score from the vector store\n   * @param nowMsec - The current timestamp in milliseconds\n   * @returns The combined score for the document\n   */\n  private getCombinedScore(\n    doc: DocumentInterface,\n    vectorRelevance: number | null,\n    nowMsec: number\n  ): number {\n    const hoursPassed = this.getHoursPassed(\n      nowMsec,\n      doc.metadata[LAST_ACCESSED_AT_KEY]\n    );\n    let score = (1.0 - this.decayRate) ** hoursPassed;\n    for (const key of this.otherScoreKeys) {\n      score += doc.metadata[key];\n    }\n    if (vectorRelevance !== null) {\n      score += vectorRelevance;\n    }\n    return score;\n  }\n\n  /**\n   * Calculate the hours passed between two time points\n   * @param time - The current time in seconds\n   * @param refTime - The reference time in seconds\n   * @returns The number of hours passed between the two time points\n   */\n  private getHoursPassed(time: number, refTime: number): number {\n    return (time - refTime) / 3600;\n  }\n}\n"],"mappings":";;;;;;;;;;AAmBA,MAAa,uBAAuB;AACpC,MAAa,aAAa;;;;;;;;;;;;;;;;;;;;;AAsB1B,IAAa,mCAAb,cAAsDA,0CAAc;CAClE,OAAO,UAAU;AACf,SAAO;CACR;CAED,IAAI,eAAe;AACjB,SAAO;GAAC;GAAa;GAAc;EAAgB;CACpD;;;;CAKD,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;;CAMR,YAAYC,QAAgD;EAC1D,MAAM,OAAO;EACb,KAAK,cAAc,OAAO;EAC1B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,eAAe,OAAO,gBAAgB,CAAE;EAC7C,KAAK,YAAY,OAAO,aAAa;EACrC,KAAK,IAAI,OAAO,KAAK;EACrB,KAAK,iBAAiB,OAAO,kBAAkB,CAAE;EACjD,KAAK,kBAAkB,OAAO,mBAAmB;CAClD;;;;;CAMD,kBAAuC;AACrC,SAAO,KAAK;CACb;;;;;CAMD,gBAAgBC,cAAmC;EACjD,KAAK,eAAe;CACrB;;;;;;CAOD,MAAM,sBACJC,OACAC,YAC8B;EAC9B,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,sBAAsB,KAAK,wBAAwB;EAEzD,MAAM,uBAAuB,MAAM,KAAK,oBACtC,OACA,WACD;EACD,MAAM,gBAAgB;GAAE,GAAG;GAAqB,GAAG;EAAsB;AAEzE,SAAO,KAAK,eAAe,eAAe,IAAI;CAC/C;;;;;;;;;CAUD,MAAM,aAAaC,MAA0C;EAC3D,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,YAAY,KAAK,iBAAiB,MAAM,IAAI;EAElD,KAAK,aAAa,KAAK,GAAG,UAAU;EACpC,MAAM,KAAK,YAAY,aAAa,UAAU;CAC/C;;;;;CAMD,AAAQ,yBAGN;EACA,MAAMC,sBAGF,CAAE;AACN,OAAK,MAAM,OAAO,KAAK,aAAa,MAAM,CAAC,KAAK,EAAE,EAAE;GAClD,MAAM,YAAY,IAAI,SAAS;AAC/B,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,oBAAoB,aAAa;IAC/B;IACA,OAAO,KAAK,mBAAmB;GAChC;EACF;AACD,SAAO;CACR;;;;;;CAOD,MAAc,oBACZH,OACAC,YACoE;EACpE,MAAMG,eACJ,MAAM,KAAK,YAAY,0BACrB,OACA,KAAK,cACL,QACA,YAAY,UAAU,CACvB;EACH,MAAMC,UACJ,CAAE;AACJ,OAAK,MAAM,CAAC,YAAY,MAAM,IAAI,cAAc;GAC9C,MAAM,YAAY,WAAW,SAAS;AACtC,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,MAAM,MAAM,KAAK,aAAa;GAC9B,QAAQ,aAAa;IAAE;IAAK;GAAO;EACpC;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,eACNC,eACAC,KACqB;EACrB,MAAM,eAAe,OAAO,OAAO,cAAc,CAC9C,IAAI,CAAC,EAAE,KAAK,OAAO,MAAM;GACxB;GACA,OAAO,KAAK,iBAAiB,KAAK,OAAO,IAAI;EAC9C,GAAE,CACF,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;EAEpC,MAAMC,UAA+B,CAAE;AACvC,OAAK,MAAM,EAAE,KAAK,IAAI,cAAc;GAClC,MAAM,cAAc,KAAK,aAAa,IAAI,SAAS;GACnD,YAAY,SAAS,wBAAwB;GAC7C,QAAQ,KAAK,YAAY;AACzB,OAAI,QAAQ,SAAS,KAAK,EACxB;EAEH;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,iBACNN,MACAK,KACqB;AACrB,SAAO,KAAK,IAAI,CAAC,KAAK,OAAO;GAC3B,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,uBAAuB,IAAI,SAAS,yBAAyB;IAC9D,YAAY,IAAI,SAAS,cAAc;KACtC,aAAa,KAAK,aAAa,SAAS;GAC1C;EACF,GAAE;CACJ;;;;;;;;CASD,AAAQ,iBACNE,KACAC,iBACAC,SACQ;EACR,MAAM,cAAc,KAAK,eACvB,SACA,IAAI,SAAS,sBACd;EACD,IAAI,SAAS,IAAM,KAAK,cAAc;AACtC,OAAK,MAAM,OAAO,KAAK,gBACrB,SAAS,IAAI,SAAS;AAExB,MAAI,oBAAoB,MACtB,SAAS;AAEX,SAAO;CACR;;;;;;;CAQD,AAAQ,eAAeC,MAAcC,SAAyB;AAC5D,UAAQ,OAAO,WAAW;CAC3B;AACF"}

package/dist/retrievers/time_weighted.js.map

@@ -1 +1 @@
-{"version":3,"file":"time_weighted.js","names":["fields: TimeWeightedVectorStoreRetrieverFields","memoryStream: DocumentInterface[]","query: string","runManager?: CallbackManagerForRetrieverRun","docs: DocumentInterface[]","memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    >","docAndScores: [DocumentInterface, number][]","results: Record<number, { doc: DocumentInterface; score: number }>","docsAndScores: Record<number, { doc: DocumentInterface; score: number }>","now: number","results: DocumentInterface[]","doc: DocumentInterface","vectorRelevance: number | null","nowMsec: number","time: number","refTime: number"],"sources":["../../src/retrievers/time_weighted.ts"],"sourcesContent":["import { BaseRetriever, BaseRetrieverInput } from \"@langchain/core/retrievers\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * Interface for the fields required to initialize a\n * TimeWeightedVectorStoreRetriever instance.\n */\nexport interface TimeWeightedVectorStoreRetrieverFields\n  extends BaseRetrieverInput {\n  vectorStore: VectorStoreInterface;\n  searchKwargs?: number;\n  memoryStream?: DocumentInterface[];\n  decayRate?: number;\n  k?: number;\n  otherScoreKeys?: string[];\n  defaultSalience?: number;\n}\n\nexport const LAST_ACCESSED_AT_KEY = \"last_accessed_at\";\nexport const BUFFER_IDX = \"buffer_idx\";\n\n/**\n * TimeWeightedVectorStoreRetriever retrieves documents based on their time-weighted relevance.\n * ref: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/retrievers/time_weighted_retriever.py\n * @example\n * ```typescript\n * const retriever = new TimeWeightedVectorStoreRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   memoryStream: [],\n *   searchKwargs: 2,\n * });\n * await retriever.addDocuments([\n *   { pageContent: \"My name is John.\", metadata: {} },\n *   { pageContent: \"My favourite food is pizza.\", metadata: {} },\n *\n * ]);\n * const results = await retriever.invoke(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class TimeWeightedVectorStoreRetriever extends BaseRetriever {\n  static lc_name() {\n    return \"TimeWeightedVectorStoreRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"time_weighted\"];\n  }\n\n  /**\n   * The vectorstore to store documents and determine salience.\n   */\n  private vectorStore: VectorStoreInterface;\n\n  /**\n   * The number of top K most relevant documents to consider when searching.\n   */\n  private searchKwargs: number;\n\n  /**\n   * The memory_stream of documents to search through.\n   */\n  private memoryStream: DocumentInterface[];\n\n  /**\n   * The exponential decay factor used as (1.0-decay_rate)**(hrs_passed).\n   */\n  private decayRate: number;\n\n  /**\n   * The maximum number of documents to retrieve in a given call.\n   */\n  private k: number;\n\n  /**\n   * Other keys in the metadata to factor into the score, e.g. 'importance'.\n   */\n  private otherScoreKeys: string[];\n\n  /**\n   * The salience to assign memories not retrieved from the vector store.\n   */\n  private defaultSalience: number | null;\n\n  /**\n   * Constructor to initialize the required fields\n   * @param fields - The fields required for initializing the TimeWeightedVectorStoreRetriever\n   */\n  constructor(fields: TimeWeightedVectorStoreRetrieverFields) {\n    super(fields);\n    this.vectorStore = fields.vectorStore;\n    this.searchKwargs = fields.searchKwargs ?? 100;\n    this.memoryStream = fields.memoryStream ?? [];\n    this.decayRate = fields.decayRate ?? 0.01;\n    this.k = fields.k ?? 4;\n    this.otherScoreKeys = fields.otherScoreKeys ?? [];\n    this.defaultSalience = fields.defaultSalience ?? null;\n  }\n\n  /**\n   * Get the memory stream of documents.\n   * @returns The memory stream of documents.\n   */\n  getMemoryStream(): DocumentInterface[] {\n    return this.memoryStream;\n  }\n\n  /**\n   * Set the memory stream of documents.\n   * @param memoryStream The new memory stream of documents.\n   */\n  setMemoryStream(memoryStream: DocumentInterface[]) {\n    this.memoryStream = memoryStream;\n  }\n\n  /**\n   * Get relevant documents based on time-weighted relevance\n   * @param query - The query to search for\n   * @returns The relevant documents\n   */\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<DocumentInterface[]> {\n    const now = Math.floor(Date.now() / 1000);\n    const memoryDocsAndScores = this.getMemoryDocsAndScores();\n\n    const salientDocsAndScores = await this.getSalientDocuments(\n      query,\n      runManager\n    );\n    const docsAndScores = { ...memoryDocsAndScores, ...salientDocsAndScores };\n\n    return this.computeResults(docsAndScores, now);\n  }\n\n  /**\n   * NOTE: When adding documents to a vector store, use addDocuments\n   * via retriever instead of directly to the vector store.\n   * This is because it is necessary to process the document\n   * in prepareDocuments.\n   *\n   * @param docs - The documents to add to vector store in the retriever\n   */\n  async addDocuments(docs: DocumentInterface[]): Promise<void> {\n    const now = Math.floor(Date.now() / 1000);\n    const savedDocs = this.prepareDocuments(docs, now);\n\n    this.memoryStream.push(...savedDocs);\n    await this.vectorStore.addDocuments(savedDocs);\n  }\n\n  /**\n   * Get memory documents and their scores\n   * @returns An object containing memory documents and their scores\n   */\n  private getMemoryDocsAndScores(): Record<\n    number,\n    { doc: DocumentInterface; score: number }\n  > {\n    const memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    > = {};\n    for (const doc of this.memoryStream.slice(-this.k)) {\n      const bufferIdx = doc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      memoryDocsAndScores[bufferIdx] = {\n        doc,\n        score: this.defaultSalience ?? 0,\n      };\n    }\n    return memoryDocsAndScores;\n  }\n\n  /**\n   * Get salient documents and their scores based on the query\n   * @param query - The query to search for\n   * @returns An object containing salient documents and their scores\n   */\n  private async getSalientDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Record<number, { doc: DocumentInterface; score: number }>> {\n    const docAndScores: [DocumentInterface, number][] =\n      await this.vectorStore.similaritySearchWithScore(\n        query,\n        this.searchKwargs,\n        undefined,\n        runManager?.getChild()\n      );\n    const results: Record<number, { doc: DocumentInterface; score: number }> =\n      {};\n    for (const [fetchedDoc, score] of docAndScores) {\n      const bufferIdx = fetchedDoc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      const doc = this.memoryStream[bufferIdx];\n      results[bufferIdx] = { doc, score };\n    }\n    return results;\n  }\n\n  /**\n   * Compute the final result set of documents based on the combined scores\n   * @param docsAndScores - An object containing documents and their scores\n   * @param now - The current timestamp\n   * @returns The final set of documents\n   */\n  private computeResults(\n    docsAndScores: Record<number, { doc: DocumentInterface; score: number }>,\n    now: number\n  ): DocumentInterface[] {\n    const recordedDocs = Object.values(docsAndScores)\n      .map(({ doc, score }) => ({\n        doc,\n        score: this.getCombinedScore(doc, score, now),\n      }))\n      .sort((a, b) => b.score - a.score);\n\n    const results: DocumentInterface[] = [];\n    for (const { doc } of recordedDocs) {\n      const bufferedDoc = this.memoryStream[doc.metadata[BUFFER_IDX]];\n      bufferedDoc.metadata[LAST_ACCESSED_AT_KEY] = now;\n      results.push(bufferedDoc);\n      if (results.length > this.k) {\n        break;\n      }\n    }\n    return results;\n  }\n\n  /**\n   * Prepare documents with necessary metadata before saving\n   * @param docs - The documents to prepare\n   * @param now - The current timestamp\n   * @returns The prepared documents\n   */\n  private prepareDocuments(\n    docs: DocumentInterface[],\n    now: number\n  ): DocumentInterface[] {\n    return docs.map((doc, i) => ({\n      ...doc,\n      metadata: {\n        ...doc.metadata,\n        [LAST_ACCESSED_AT_KEY]: doc.metadata[LAST_ACCESSED_AT_KEY] ?? now,\n        created_at: doc.metadata.created_at ?? now,\n        [BUFFER_IDX]: this.memoryStream.length + i,\n      },\n    }));\n  }\n\n  /**\n   * Calculate the combined score based on vector relevance and other factors\n   * @param doc - The document to calculate the score for\n   * @param vectorRelevance - The relevance score from the vector store\n   * @param nowMsec - The current timestamp in milliseconds\n   * @returns The combined score for the document\n   */\n  private getCombinedScore(\n    doc: DocumentInterface,\n    vectorRelevance: number | null,\n    nowMsec: number\n  ): number {\n    const hoursPassed = this.getHoursPassed(\n      nowMsec,\n      doc.metadata[LAST_ACCESSED_AT_KEY]\n    );\n    let score = (1.0 - this.decayRate) ** hoursPassed;\n    for (const key of this.otherScoreKeys) {\n      score += doc.metadata[key];\n    }\n    if (vectorRelevance !== null) {\n      score += vectorRelevance;\n    }\n    return score;\n  }\n\n  /**\n   * Calculate the hours passed between two time points\n   * @param time - The current time in seconds\n   * @param refTime - The reference time in seconds\n   * @returns The number of hours passed between the two time points\n   */\n  private getHoursPassed(time: number, refTime: number): number {\n    return (time - refTime) / 3600;\n  }\n}\n"],"mappings":";;;;;;;;;;AAoBA,MAAa,uBAAuB;AACpC,MAAa,aAAa;;;;;;;;;;;;;;;;;;;;;AAsB1B,IAAa,mCAAb,cAAsD,cAAc;CAClE,OAAO,UAAU;AACf,SAAO;CACR;CAED,IAAI,eAAe;AACjB,SAAO;GAAC;GAAa;GAAc;EAAgB;CACpD;;;;CAKD,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;;CAMR,YAAYA,QAAgD;EAC1D,MAAM,OAAO;EACb,KAAK,cAAc,OAAO;EAC1B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,eAAe,OAAO,gBAAgB,CAAE;EAC7C,KAAK,YAAY,OAAO,aAAa;EACrC,KAAK,IAAI,OAAO,KAAK;EACrB,KAAK,iBAAiB,OAAO,kBAAkB,CAAE;EACjD,KAAK,kBAAkB,OAAO,mBAAmB;CAClD;;;;;CAMD,kBAAuC;AACrC,SAAO,KAAK;CACb;;;;;CAMD,gBAAgBC,cAAmC;EACjD,KAAK,eAAe;CACrB;;;;;;CAOD,MAAM,sBACJC,OACAC,YAC8B;EAC9B,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,sBAAsB,KAAK,wBAAwB;EAEzD,MAAM,uBAAuB,MAAM,KAAK,oBACtC,OACA,WACD;EACD,MAAM,gBAAgB;GAAE,GAAG;GAAqB,GAAG;EAAsB;AAEzE,SAAO,KAAK,eAAe,eAAe,IAAI;CAC/C;;;;;;;;;CAUD,MAAM,aAAaC,MAA0C;EAC3D,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,YAAY,KAAK,iBAAiB,MAAM,IAAI;EAElD,KAAK,aAAa,KAAK,GAAG,UAAU;EACpC,MAAM,KAAK,YAAY,aAAa,UAAU;CAC/C;;;;;CAMD,AAAQ,yBAGN;EACA,MAAMC,sBAGF,CAAE;AACN,OAAK,MAAM,OAAO,KAAK,aAAa,MAAM,CAAC,KAAK,EAAE,EAAE;GAClD,MAAM,YAAY,IAAI,SAAS;AAC/B,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,oBAAoB,aAAa;IAC/B;IACA,OAAO,KAAK,mBAAmB;GAChC;EACF;AACD,SAAO;CACR;;;;;;CAOD,MAAc,oBACZH,OACAC,YACoE;EACpE,MAAMG,eACJ,MAAM,KAAK,YAAY,0BACrB,OACA,KAAK,cACL,QACA,YAAY,UAAU,CACvB;EACH,MAAMC,UACJ,CAAE;AACJ,OAAK,MAAM,CAAC,YAAY,MAAM,IAAI,cAAc;GAC9C,MAAM,YAAY,WAAW,SAAS;AACtC,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,MAAM,MAAM,KAAK,aAAa;GAC9B,QAAQ,aAAa;IAAE;IAAK;GAAO;EACpC;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,eACNC,eACAC,KACqB;EACrB,MAAM,eAAe,OAAO,OAAO,cAAc,CAC9C,IAAI,CAAC,EAAE,KAAK,OAAO,MAAM;GACxB;GACA,OAAO,KAAK,iBAAiB,KAAK,OAAO,IAAI;EAC9C,GAAE,CACF,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;EAEpC,MAAMC,UAA+B,CAAE;AACvC,OAAK,MAAM,EAAE,KAAK,IAAI,cAAc;GAClC,MAAM,cAAc,KAAK,aAAa,IAAI,SAAS;GACnD,YAAY,SAAS,wBAAwB;GAC7C,QAAQ,KAAK,YAAY;AACzB,OAAI,QAAQ,SAAS,KAAK,EACxB;EAEH;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,iBACNN,MACAK,KACqB;AACrB,SAAO,KAAK,IAAI,CAAC,KAAK,OAAO;GAC3B,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,uBAAuB,IAAI,SAAS,yBAAyB;IAC9D,YAAY,IAAI,SAAS,cAAc;KACtC,aAAa,KAAK,aAAa,SAAS;GAC1C;EACF,GAAE;CACJ;;;;;;;;CASD,AAAQ,iBACNE,KACAC,iBACAC,SACQ;EACR,MAAM,cAAc,KAAK,eACvB,SACA,IAAI,SAAS,sBACd;EACD,IAAI,SAAS,IAAM,KAAK,cAAc;AACtC,OAAK,MAAM,OAAO,KAAK,gBACrB,SAAS,IAAI,SAAS;AAExB,MAAI,oBAAoB,MACtB,SAAS;AAEX,SAAO;CACR;;;;;;;CAQD,AAAQ,eAAeC,MAAcC,SAAyB;AAC5D,UAAQ,OAAO,WAAW;CAC3B;AACF"}
+{"version":3,"file":"time_weighted.js","names":["fields: TimeWeightedVectorStoreRetrieverFields","memoryStream: DocumentInterface[]","query: string","runManager?: CallbackManagerForRetrieverRun","docs: DocumentInterface[]","memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    >","docAndScores: [DocumentInterface, number][]","results: Record<number, { doc: DocumentInterface; score: number }>","docsAndScores: Record<number, { doc: DocumentInterface; score: number }>","now: number","results: DocumentInterface[]","doc: DocumentInterface","vectorRelevance: number | null","nowMsec: number","time: number","refTime: number"],"sources":["../../src/retrievers/time_weighted.ts"],"sourcesContent":["import { BaseRetriever, BaseRetrieverInput } from \"@langchain/core/retrievers\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * Interface for the fields required to initialize a\n * TimeWeightedVectorStoreRetriever instance.\n */\nexport interface TimeWeightedVectorStoreRetrieverFields extends BaseRetrieverInput {\n  vectorStore: VectorStoreInterface;\n  searchKwargs?: number;\n  memoryStream?: DocumentInterface[];\n  decayRate?: number;\n  k?: number;\n  otherScoreKeys?: string[];\n  defaultSalience?: number;\n}\n\nexport const LAST_ACCESSED_AT_KEY = \"last_accessed_at\";\nexport const BUFFER_IDX = \"buffer_idx\";\n\n/**\n * TimeWeightedVectorStoreRetriever retrieves documents based on their time-weighted relevance.\n * ref: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/retrievers/time_weighted_retriever.py\n * @example\n * ```typescript\n * const retriever = new TimeWeightedVectorStoreRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   memoryStream: [],\n *   searchKwargs: 2,\n * });\n * await retriever.addDocuments([\n *   { pageContent: \"My name is John.\", metadata: {} },\n *   { pageContent: \"My favourite food is pizza.\", metadata: {} },\n *\n * ]);\n * const results = await retriever.invoke(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class TimeWeightedVectorStoreRetriever extends BaseRetriever {\n  static lc_name() {\n    return \"TimeWeightedVectorStoreRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"time_weighted\"];\n  }\n\n  /**\n   * The vectorstore to store documents and determine salience.\n   */\n  private vectorStore: VectorStoreInterface;\n\n  /**\n   * The number of top K most relevant documents to consider when searching.\n   */\n  private searchKwargs: number;\n\n  /**\n   * The memory_stream of documents to search through.\n   */\n  private memoryStream: DocumentInterface[];\n\n  /**\n   * The exponential decay factor used as (1.0-decay_rate)**(hrs_passed).\n   */\n  private decayRate: number;\n\n  /**\n   * The maximum number of documents to retrieve in a given call.\n   */\n  private k: number;\n\n  /**\n   * Other keys in the metadata to factor into the score, e.g. 'importance'.\n   */\n  private otherScoreKeys: string[];\n\n  /**\n   * The salience to assign memories not retrieved from the vector store.\n   */\n  private defaultSalience: number | null;\n\n  /**\n   * Constructor to initialize the required fields\n   * @param fields - The fields required for initializing the TimeWeightedVectorStoreRetriever\n   */\n  constructor(fields: TimeWeightedVectorStoreRetrieverFields) {\n    super(fields);\n    this.vectorStore = fields.vectorStore;\n    this.searchKwargs = fields.searchKwargs ?? 100;\n    this.memoryStream = fields.memoryStream ?? [];\n    this.decayRate = fields.decayRate ?? 0.01;\n    this.k = fields.k ?? 4;\n    this.otherScoreKeys = fields.otherScoreKeys ?? [];\n    this.defaultSalience = fields.defaultSalience ?? null;\n  }\n\n  /**\n   * Get the memory stream of documents.\n   * @returns The memory stream of documents.\n   */\n  getMemoryStream(): DocumentInterface[] {\n    return this.memoryStream;\n  }\n\n  /**\n   * Set the memory stream of documents.\n   * @param memoryStream The new memory stream of documents.\n   */\n  setMemoryStream(memoryStream: DocumentInterface[]) {\n    this.memoryStream = memoryStream;\n  }\n\n  /**\n   * Get relevant documents based on time-weighted relevance\n   * @param query - The query to search for\n   * @returns The relevant documents\n   */\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<DocumentInterface[]> {\n    const now = Math.floor(Date.now() / 1000);\n    const memoryDocsAndScores = this.getMemoryDocsAndScores();\n\n    const salientDocsAndScores = await this.getSalientDocuments(\n      query,\n      runManager\n    );\n    const docsAndScores = { ...memoryDocsAndScores, ...salientDocsAndScores };\n\n    return this.computeResults(docsAndScores, now);\n  }\n\n  /**\n   * NOTE: When adding documents to a vector store, use addDocuments\n   * via retriever instead of directly to the vector store.\n   * This is because it is necessary to process the document\n   * in prepareDocuments.\n   *\n   * @param docs - The documents to add to vector store in the retriever\n   */\n  async addDocuments(docs: DocumentInterface[]): Promise<void> {\n    const now = Math.floor(Date.now() / 1000);\n    const savedDocs = this.prepareDocuments(docs, now);\n\n    this.memoryStream.push(...savedDocs);\n    await this.vectorStore.addDocuments(savedDocs);\n  }\n\n  /**\n   * Get memory documents and their scores\n   * @returns An object containing memory documents and their scores\n   */\n  private getMemoryDocsAndScores(): Record<\n    number,\n    { doc: DocumentInterface; score: number }\n  > {\n    const memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    > = {};\n    for (const doc of this.memoryStream.slice(-this.k)) {\n      const bufferIdx = doc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      memoryDocsAndScores[bufferIdx] = {\n        doc,\n        score: this.defaultSalience ?? 0,\n      };\n    }\n    return memoryDocsAndScores;\n  }\n\n  /**\n   * Get salient documents and their scores based on the query\n   * @param query - The query to search for\n   * @returns An object containing salient documents and their scores\n   */\n  private async getSalientDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Record<number, { doc: DocumentInterface; score: number }>> {\n    const docAndScores: [DocumentInterface, number][] =\n      await this.vectorStore.similaritySearchWithScore(\n        query,\n        this.searchKwargs,\n        undefined,\n        runManager?.getChild()\n      );\n    const results: Record<number, { doc: DocumentInterface; score: number }> =\n      {};\n    for (const [fetchedDoc, score] of docAndScores) {\n      const bufferIdx = fetchedDoc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      const doc = this.memoryStream[bufferIdx];\n      results[bufferIdx] = { doc, score };\n    }\n    return results;\n  }\n\n  /**\n   * Compute the final result set of documents based on the combined scores\n   * @param docsAndScores - An object containing documents and their scores\n   * @param now - The current timestamp\n   * @returns The final set of documents\n   */\n  private computeResults(\n    docsAndScores: Record<number, { doc: DocumentInterface; score: number }>,\n    now: number\n  ): DocumentInterface[] {\n    const recordedDocs = Object.values(docsAndScores)\n      .map(({ doc, score }) => ({\n        doc,\n        score: this.getCombinedScore(doc, score, now),\n      }))\n      .sort((a, b) => b.score - a.score);\n\n    const results: DocumentInterface[] = [];\n    for (const { doc } of recordedDocs) {\n      const bufferedDoc = this.memoryStream[doc.metadata[BUFFER_IDX]];\n      bufferedDoc.metadata[LAST_ACCESSED_AT_KEY] = now;\n      results.push(bufferedDoc);\n      if (results.length > this.k) {\n        break;\n      }\n    }\n    return results;\n  }\n\n  /**\n   * Prepare documents with necessary metadata before saving\n   * @param docs - The documents to prepare\n   * @param now - The current timestamp\n   * @returns The prepared documents\n   */\n  private prepareDocuments(\n    docs: DocumentInterface[],\n    now: number\n  ): DocumentInterface[] {\n    return docs.map((doc, i) => ({\n      ...doc,\n      metadata: {\n        ...doc.metadata,\n        [LAST_ACCESSED_AT_KEY]: doc.metadata[LAST_ACCESSED_AT_KEY] ?? now,\n        created_at: doc.metadata.created_at ?? now,\n        [BUFFER_IDX]: this.memoryStream.length + i,\n      },\n    }));\n  }\n\n  /**\n   * Calculate the combined score based on vector relevance and other factors\n   * @param doc - The document to calculate the score for\n   * @param vectorRelevance - The relevance score from the vector store\n   * @param nowMsec - The current timestamp in milliseconds\n   * @returns The combined score for the document\n   */\n  private getCombinedScore(\n    doc: DocumentInterface,\n    vectorRelevance: number | null,\n    nowMsec: number\n  ): number {\n    const hoursPassed = this.getHoursPassed(\n      nowMsec,\n      doc.metadata[LAST_ACCESSED_AT_KEY]\n    );\n    let score = (1.0 - this.decayRate) ** hoursPassed;\n    for (const key of this.otherScoreKeys) {\n      score += doc.metadata[key];\n    }\n    if (vectorRelevance !== null) {\n      score += vectorRelevance;\n    }\n    return score;\n  }\n\n  /**\n   * Calculate the hours passed between two time points\n   * @param time - The current time in seconds\n   * @param refTime - The reference time in seconds\n   * @returns The number of hours passed between the two time points\n   */\n  private getHoursPassed(time: number, refTime: number): number {\n    return (time - refTime) / 3600;\n  }\n}\n"],"mappings":";;;;;;;;;;AAmBA,MAAa,uBAAuB;AACpC,MAAa,aAAa;;;;;;;;;;;;;;;;;;;;;AAsB1B,IAAa,mCAAb,cAAsD,cAAc;CAClE,OAAO,UAAU;AACf,SAAO;CACR;CAED,IAAI,eAAe;AACjB,SAAO;GAAC;GAAa;GAAc;EAAgB;CACpD;;;;CAKD,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;CAKR,AAAQ;;;;;CAMR,YAAYA,QAAgD;EAC1D,MAAM,OAAO;EACb,KAAK,cAAc,OAAO;EAC1B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,eAAe,OAAO,gBAAgB,CAAE;EAC7C,KAAK,YAAY,OAAO,aAAa;EACrC,KAAK,IAAI,OAAO,KAAK;EACrB,KAAK,iBAAiB,OAAO,kBAAkB,CAAE;EACjD,KAAK,kBAAkB,OAAO,mBAAmB;CAClD;;;;;CAMD,kBAAuC;AACrC,SAAO,KAAK;CACb;;;;;CAMD,gBAAgBC,cAAmC;EACjD,KAAK,eAAe;CACrB;;;;;;CAOD,MAAM,sBACJC,OACAC,YAC8B;EAC9B,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,sBAAsB,KAAK,wBAAwB;EAEzD,MAAM,uBAAuB,MAAM,KAAK,oBACtC,OACA,WACD;EACD,MAAM,gBAAgB;GAAE,GAAG;GAAqB,GAAG;EAAsB;AAEzE,SAAO,KAAK,eAAe,eAAe,IAAI;CAC/C;;;;;;;;;CAUD,MAAM,aAAaC,MAA0C;EAC3D,MAAM,MAAM,KAAK,MAAM,KAAK,KAAK,GAAG,IAAK;EACzC,MAAM,YAAY,KAAK,iBAAiB,MAAM,IAAI;EAElD,KAAK,aAAa,KAAK,GAAG,UAAU;EACpC,MAAM,KAAK,YAAY,aAAa,UAAU;CAC/C;;;;;CAMD,AAAQ,yBAGN;EACA,MAAMC,sBAGF,CAAE;AACN,OAAK,MAAM,OAAO,KAAK,aAAa,MAAM,CAAC,KAAK,EAAE,EAAE;GAClD,MAAM,YAAY,IAAI,SAAS;AAC/B,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,oBAAoB,aAAa;IAC/B;IACA,OAAO,KAAK,mBAAmB;GAChC;EACF;AACD,SAAO;CACR;;;;;;CAOD,MAAc,oBACZH,OACAC,YACoE;EACpE,MAAMG,eACJ,MAAM,KAAK,YAAY,0BACrB,OACA,KAAK,cACL,QACA,YAAY,UAAU,CACvB;EACH,MAAMC,UACJ,CAAE;AACJ,OAAK,MAAM,CAAC,YAAY,MAAM,IAAI,cAAc;GAC9C,MAAM,YAAY,WAAW,SAAS;AACtC,OAAI,cAAc,OAChB,OAAM,IAAI,MACR,CAAC,2RAA2R,CAAC;GAGjS,MAAM,MAAM,KAAK,aAAa;GAC9B,QAAQ,aAAa;IAAE;IAAK;GAAO;EACpC;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,eACNC,eACAC,KACqB;EACrB,MAAM,eAAe,OAAO,OAAO,cAAc,CAC9C,IAAI,CAAC,EAAE,KAAK,OAAO,MAAM;GACxB;GACA,OAAO,KAAK,iBAAiB,KAAK,OAAO,IAAI;EAC9C,GAAE,CACF,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;EAEpC,MAAMC,UAA+B,CAAE;AACvC,OAAK,MAAM,EAAE,KAAK,IAAI,cAAc;GAClC,MAAM,cAAc,KAAK,aAAa,IAAI,SAAS;GACnD,YAAY,SAAS,wBAAwB;GAC7C,QAAQ,KAAK,YAAY;AACzB,OAAI,QAAQ,SAAS,KAAK,EACxB;EAEH;AACD,SAAO;CACR;;;;;;;CAQD,AAAQ,iBACNN,MACAK,KACqB;AACrB,SAAO,KAAK,IAAI,CAAC,KAAK,OAAO;GAC3B,GAAG;GACH,UAAU;IACR,GAAG,IAAI;KACN,uBAAuB,IAAI,SAAS,yBAAyB;IAC9D,YAAY,IAAI,SAAS,cAAc;KACtC,aAAa,KAAK,aAAa,SAAS;GAC1C;EACF,GAAE;CACJ;;;;;;;;CASD,AAAQ,iBACNE,KACAC,iBACAC,SACQ;EACR,MAAM,cAAc,KAAK,eACvB,SACA,IAAI,SAAS,sBACd;EACD,IAAI,SAAS,IAAM,KAAK,cAAc;AACtC,OAAK,MAAM,OAAO,KAAK,gBACrB,SAAS,IAAI,SAAS;AAExB,MAAI,oBAAoB,MACtB,SAAS;AAEX,SAAO;CACR;;;;;;;CAQD,AAAQ,eAAeC,MAAcC,SAAyB;AAC5D,UAAQ,OAAO,WAAW;CAC3B;AACF"}

package/dist/smith/runner_utils.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.cjs","names":["_chain: Serialized","_inputs: ChainValues","runId: string","BaseTracer","run: Run","evaluator: RunEvaluatorLike","RunnableLambda","example?: Example","LangChainTracer","evaluator: any","RunTree","config: RunTreeConfig","callbackManager: CallbackManager","Runnable","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig | keyof EvaluatorType","loadEvaluator","evaluators: (RunEvaluator | DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","isCustomEvaluator","isOffTheShelfEvaluator","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","ProgressBar","AsyncCaller","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig | undefined","Client","randomName","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // eslint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams\n  extends Omit<RunEvalConfig, \"customEvaluators\"> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiCC,yCAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAIC,0CAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYF,KAAUG,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+BC,kBAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqDE,oCAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYC,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,kEAAkC,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAIN,kBAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEO,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAMC,6BAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYjB,KAAUG,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBc,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOC,iCAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,sCAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAsCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAIb,oCAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAIP,0CAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMqB,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAInB,0DAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAIoB,6BAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAIC,gDAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,oEACG,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAIC;CACjC,MAAM,kBAAkB,eAAeC,oCAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAIhC,0CAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAIsB,6BAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMW,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,iBAAiB;EAC5D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}
+{"version":3,"file":"runner_utils.cjs","names":["_chain: Serialized","_inputs: ChainValues","runId: string","BaseTracer","run: Run","evaluator: RunEvaluatorLike","RunnableLambda","example?: Example","LangChainTracer","evaluator: any","RunTree","config: RunTreeConfig","callbackManager: CallbackManager","Runnable","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig | keyof EvaluatorType","loadEvaluator","evaluators: (RunEvaluator | DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","isCustomEvaluator","isOffTheShelfEvaluator","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","ProgressBar","AsyncCaller","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig | undefined","Client","randomName","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // eslint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiCC,yCAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAIC,0CAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYF,KAAUG,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+BC,kBAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqDE,oCAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYC,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,kEAAkC,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAIN,kBAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEO,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAMC,6BAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYjB,KAAUG,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBc,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOC,iCAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,sCAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAwCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAIb,oCAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAIP,0CAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMqB,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAInB,0DAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAIoB,6BAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAIC,gDAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,oEACG,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAIC;CACjC,MAAM,kBAAkB,eAAeC,oCAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAIhC,0CAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAIsB,6BAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMW,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,iBAAiB;EAC5D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}

package/dist/smith/runner_utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner_utils.js","names":["_chain: Serialized","_inputs: ChainValues","runId: string","run: Run","evaluator: RunEvaluatorLike","example?: Example","evaluator: any","config: RunTreeConfig","callbackManager: CallbackManager","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig | keyof EvaluatorType","evaluators: (RunEvaluator | DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig | undefined","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // eslint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams\n  extends Omit<RunEvalConfig, \"customEvaluators\"> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiC,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAI,eAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYD,KAAUE,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+B,QAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqD,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYE,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,MAAM,4BAA4B,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAI,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEC,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAM,cAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYZ,KAAUE,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBW,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAO,kBAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAO,uBAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAsCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAI,SAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAI,eAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMC,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAI,gBAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAI,YAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAC7B,gCAAgC,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAI;CACjC,MAAM,kBAAkB,eAAe,YAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAI,eAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMC,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,iBAAiB;EAC5D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}
+{"version":3,"file":"runner_utils.js","names":["_chain: Serialized","_inputs: ChainValues","runId: string","run: Run","evaluator: RunEvaluatorLike","example?: Example","evaluator: any","config: RunTreeConfig","callbackManager: CallbackManager","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig | keyof EvaluatorType","evaluators: (RunEvaluator | DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig | undefined","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n  Runnable,\n  RunnableConfig,\n  RunnableLambda,\n  getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n  CallbackManager,\n  CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  Client,\n  Example,\n  Feedback,\n  Run,\n  RunTree,\n  RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n  isOffTheShelfEvaluator,\n  type DynamicRunEvaluatorParams,\n  type EvalConfig,\n  type EvaluatorInputFormatter,\n  type RunEvalConfig,\n  type RunEvaluatorLike,\n  isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n  | Runnable\n  | (() => Runnable)\n  | AnyTraceableFunction\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => any)\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  | ((obj: any) => Promise<any>)\n  | (() => (obj: unknown) => unknown)\n  | (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n  runIdPromiseResolver: (runId: string) => void;\n\n  runIdPromise: Promise<string>;\n\n  constructor() {\n    this.runIdPromise = new Promise<string>((extract) => {\n      this.runIdPromiseResolver = extract;\n    });\n  }\n\n  handleChainStart = (\n    _chain: Serialized,\n    _inputs: ChainValues,\n    runId: string\n  ) => {\n    this.runIdPromiseResolver(runId);\n  };\n\n  async extract(): Promise<string> {\n    return this.runIdPromise;\n  }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n  runPromiseResolver: (run: Run) => void;\n\n  runPromise: Promise<Run>;\n\n  /** The name of the callback handler. */\n  name = \"single_run_extractor\";\n\n  constructor() {\n    super();\n    this.runPromise = new Promise<Run>((extract) => {\n      this.runPromiseResolver = extract;\n    });\n  }\n\n  async persistRun(run: Run) {\n    this.runPromiseResolver(run);\n  }\n\n  async extract(): Promise<Run> {\n    return this.runPromise;\n  }\n}\n\n/**\n * Wraps an evaluator function + implements the RunEvaluator interface.\n */\nclass DynamicRunEvaluator implements RunEvaluator {\n  evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n  constructor(evaluator: RunEvaluatorLike) {\n    this.evaluator = new RunnableLambda({ func: evaluator });\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    const result = await this.evaluator.invoke(\n      {\n        run,\n        example,\n        input: run.inputs,\n        prediction: run.outputs,\n        reference: example?.outputs,\n      },\n      {\n        callbacks: [extractor, tracer],\n      }\n    );\n    const runId = await extractor.extract();\n    return {\n      sourceRunId: runId,\n      ...result,\n    };\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n  return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/**\n * Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n *\n * The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n */\nclass CallbackManagerRunTree extends RunTree {\n  callbackManager: CallbackManager;\n\n  activeCallbackManager: CallbackManagerForChainRun | undefined = undefined;\n\n  constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n    super(config);\n\n    this.callbackManager = callbackManager;\n  }\n\n  createChild(config: RunTreeConfig): CallbackManagerRunTree {\n    const child = new CallbackManagerRunTree(\n      {\n        ...config,\n        parent_run: this,\n        project_name: this.project_name,\n        client: this.client,\n      },\n      this.activeCallbackManager?.getChild() ?? this.callbackManager\n    );\n    this.child_runs.push(child);\n    return child;\n  }\n\n  async postRun(): Promise<void> {\n    // how it is translated in comparison to basic RunTree?\n    this.activeCallbackManager = await this.callbackManager.handleChainStart(\n      typeof this.serialized !== \"object\" &&\n        this.serialized != null &&\n        \"lc\" in this.serialized\n        ? this.serialized\n        : {\n            id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n            lc: 1,\n            type: \"not_implemented\",\n          },\n      this.inputs,\n      this.id,\n      this.run_type,\n      undefined,\n      undefined,\n      this.name\n    );\n  }\n\n  async patchRun(): Promise<void> {\n    if (this.error) {\n      await this.activeCallbackManager?.handleChainError(\n        this.error,\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    } else {\n      await this.activeCallbackManager?.handleChainEnd(\n        this.outputs ?? {},\n        this.id,\n        this.parent_run?.id,\n        undefined,\n        undefined\n      );\n    }\n  }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n  RunInput,\n  RunOutput\n> {\n  lc_serializable = false;\n\n  lc_namespace = [\"langchain_core\", \"runnables\"];\n\n  protected func: AnyTraceableFunction;\n\n  constructor(fields: { func: AnyTraceableFunction }) {\n    super(fields);\n\n    if (!isLangsmithTraceableFunction(fields.func)) {\n      throw new Error(\n        \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n      );\n    }\n\n    this.func = fields.func;\n  }\n\n  async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n    const [config] = this._getOptionsList(options ?? {}, 1);\n    const callbackManager = await getCallbackManagerForConfig(config);\n\n    const partialConfig =\n      \"langsmith:traceable\" in this.func\n        ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n        : { name: \"<lambda>\" };\n\n    if (!callbackManager) throw new Error(\"CallbackManager not found\");\n    const runTree = new CallbackManagerRunTree(\n      {\n        ...partialConfig,\n        parent_run: callbackManager?._parentRunId\n          ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n          : undefined,\n      },\n      callbackManager\n    );\n\n    if (\n      typeof input === \"object\" &&\n      input != null &&\n      Object.keys(input).length === 1\n    ) {\n      if (\"args\" in input && Array.isArray(input)) {\n        return (await this.func(runTree, ...input)) as RunOutput;\n      }\n\n      if (\n        \"input\" in input &&\n        !(\n          typeof input === \"object\" &&\n          input != null &&\n          !Array.isArray(input) &&\n          // eslint-disable-next-line no-instanceof/no-instanceof\n          !(input instanceof Date)\n        )\n      ) {\n        try {\n          return (await this.func(runTree, input.input)) as RunOutput;\n        } catch {\n          return (await this.func(runTree, input)) as RunOutput;\n        }\n      }\n    }\n\n    return (await this.func(runTree, input)) as RunOutput;\n  }\n}\n\n/**\n * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n */\nclass PreparedRunEvaluator implements RunEvaluator {\n  evaluator: LLMStringEvaluator;\n\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n\n  isStringEvaluator: boolean;\n\n  evaluationName: string;\n\n  constructor(\n    evaluator: LLMStringEvaluator,\n    evaluationName: string,\n    formatEvaluatorInputs: EvaluatorInputFormatter\n  ) {\n    this.evaluator = evaluator;\n    this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n    this.evaluationName = evaluationName;\n    this.formatEvaluatorInputs = formatEvaluatorInputs;\n  }\n\n  static async fromEvalConfig(\n    config: EvalConfig | keyof EvaluatorType\n  ): Promise<PreparedRunEvaluator> {\n    const evaluatorType =\n      typeof config === \"string\" ? config : config.evaluatorType;\n    const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n    const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n    const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n    if (!isLLMStringEvaluator(evaluator)) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} not yet supported. ` +\n          \"Please use a string evaluator, or implement your \" +\n          \"evaluation logic as a custom evaluator.\"\n      );\n    }\n    if (!feedbackKey) {\n      throw new Error(\n        `Evaluator of type ${evaluatorType} must have an evaluationName` +\n          ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n      );\n    }\n    return new PreparedRunEvaluator(\n      evaluator as LLMStringEvaluator,\n      feedbackKey,\n      evalConfig?.formatEvaluatorInputs\n    );\n  }\n\n  /**\n   * Evaluates a run with an optional example and returns the evaluation result.\n   * @param run The run to evaluate.\n   * @param example The optional example to use for evaluation.\n   * @returns A promise that extracts to the evaluation result.\n   */\n  async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n    const { prediction, input, reference } = this.formatEvaluatorInputs({\n      rawInput: run.inputs,\n      rawPrediction: run.outputs,\n      rawReferenceOutput: example?.outputs,\n      run,\n    });\n    const extractor = new SingleRunIdExtractor();\n    const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n    if (this.isStringEvaluator) {\n      const evalResult = await this.evaluator.evaluateStrings(\n        {\n          prediction: prediction as string,\n          reference: reference as string,\n          input: input as string,\n        },\n        {\n          callbacks: [extractor, tracer],\n        }\n      );\n      const runId = await extractor.extract();\n      return {\n        key: this.evaluationName,\n        comment: evalResult?.reasoning,\n        sourceRunId: runId,\n        ...evalResult,\n      };\n    }\n    throw new Error(\n      \"Evaluator not yet supported. \" +\n        \"Please use a string evaluator, or implement your \" +\n        \"evaluation logic as a custom evaluator.\"\n    );\n  }\n}\n\nclass LoadedEvalConfig {\n  constructor(public evaluators: (RunEvaluator | DynamicRunEvaluator)[]) {}\n\n  static async fromRunEvalConfig(\n    config: RunEvalConfig<keyof EvaluatorType>\n  ): Promise<LoadedEvalConfig> {\n    // Custom evaluators are applied \"as-is\"\n    const customEvaluators = (\n      config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n    )?.map((evaluator) => {\n      if (typeof evaluator === \"function\") {\n        return new DynamicRunEvaluator(evaluator);\n      } else {\n        return evaluator;\n      }\n    });\n\n    const offTheShelfEvaluators = await Promise.all(\n      config?.evaluators\n        ?.filter(isOffTheShelfEvaluator)\n        ?.map(\n          async (evaluator) =>\n            await PreparedRunEvaluator.fromEvalConfig(evaluator)\n        ) ?? []\n    );\n    return new LoadedEvalConfig(\n      (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n    );\n  }\n}\n\nexport interface RunOnDatasetParams extends Omit<\n  RunEvalConfig,\n  \"customEvaluators\"\n> {\n  /**\n   * Name of the project for logging and tracking.\n   */\n  projectName?: string;\n\n  /**\n   * Additional metadata for the project.\n   */\n  projectMetadata?: Record<string, unknown>;\n\n  /**\n   * Client instance for LangSmith service interaction.\n   */\n  client?: Client;\n\n  /**\n   * Maximum concurrency level for dataset processing.\n   */\n  maxConcurrency?: number;\n\n  /**\n   * @deprecated Pass keys directly to the RunOnDatasetParams instead\n   */\n  evaluationConfig?: RunEvalConfig;\n}\n\n/**\n * Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n */\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n  if (Runnable.isRunnable(modelOrFactory)) {\n    return () => modelOrFactory;\n  }\n  if (typeof modelOrFactory === \"function\") {\n    if (isLangsmithTraceableFunction(modelOrFactory)) {\n      const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n\n    try {\n      // If it works with no arguments, assume it's a factory\n      let res = (modelOrFactory as () => Runnable)();\n      if (\n        res &&\n        typeof (res as unknown as Promise<Runnable>).then === \"function\"\n      ) {\n        res = await res;\n      }\n      return modelOrFactory as () => Runnable;\n    } catch {\n      // Otherwise, it's a custom UDF, and we'll wrap\n      // the function in a lambda\n      const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n      return () => wrappedModel;\n    }\n  }\n  throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n  datasetName,\n  client,\n  projectName,\n}: {\n  datasetName: string;\n  client: Client;\n  projectName: string;\n  maxConcurrency: number;\n}) => {\n  const exampleIterator = client.listExamples({ datasetName });\n  const configs: RunnableConfig[] = [];\n  const runExtractors = [];\n  const examples = [];\n  for await (const example of exampleIterator) {\n    const runExtractor = new SingleRunExtractor();\n    configs.push({\n      callbacks: [\n        new LangChainTracer({ exampleId: example.id, projectName }),\n        runExtractor,\n      ],\n    });\n    examples.push(example);\n    runExtractors.push(runExtractor);\n  }\n  return {\n    configs,\n    examples,\n    runExtractors,\n  };\n};\n\nconst applyEvaluators = async ({\n  evaluation,\n  runs,\n  examples,\n  client,\n  maxConcurrency,\n}: {\n  evaluation: LoadedEvalConfig;\n  runs: Run[];\n  examples: Example[];\n  client: Client;\n  maxConcurrency: number;\n}): Promise<{\n  [key: string]: {\n    execution_time?: number;\n    run_id: string;\n    feedback: Feedback[];\n  };\n}> => {\n  // TODO: Parallelize and/or put in callbacks to speed up evals.\n  const { evaluators } = evaluation;\n  const progress = new ProgressBar({\n    total: examples.length,\n    format: \"Running Evaluators: {bar} {percentage}% | {value}/{total}\\n\",\n  });\n  const caller = new AsyncCaller({\n    maxConcurrency,\n  });\n  const requests = runs.map(\n    async (\n      run,\n      i\n    ): Promise<{\n      run_id: string;\n      execution_time?: number;\n      feedback: Feedback[];\n    }> =>\n      caller.call(async () => {\n        const evaluatorResults = await Promise.allSettled(\n          evaluators.map((evaluator) =>\n            client.evaluateRun(run, evaluator, {\n              referenceExample: examples[i],\n              loadChildRuns: false,\n            })\n          )\n        );\n        progress.increment();\n        return {\n          execution_time:\n            run?.end_time && run.start_time\n              ? new Date(run.end_time).getTime() -\n                new Date(run.start_time).getTime()\n              : undefined,\n          feedback: evaluatorResults.map((evalResult) =>\n            evalResult.status === \"fulfilled\"\n              ? evalResult.value\n              : evalResult.reason\n          ),\n          run_id: run.id,\n        };\n      })\n  );\n  const results = await Promise.all(requests);\n\n  return results.reduce(\n    (acc, result, i) => ({\n      ...acc,\n      [examples[i].id]: result,\n    }),\n    {}\n  );\n};\n\nexport type EvalResults = {\n  projectName: string;\n  results: {\n    [key: string]: {\n      execution_time?: number;\n      run_id: string;\n      feedback: Feedback[];\n    };\n  };\n};\n\nconst getExamplesInputs = (\n  examples: Example[],\n  chainOrFactory: ChainOrFactory,\n  dataType?: DataType\n) => {\n  if (dataType === \"chat\") {\n    // For some batty reason, we store the chat dataset differently.\n    // { type: \"system\", data: { content: inputs.input } },\n    // But we need to create AIMesage, SystemMessage, etc.\n    return examples.map(({ inputs }) =>\n      mapStoredMessagesToChatMessages(inputs.input)\n    );\n  }\n  // If it's a language model and ALL example inputs have a single value,\n  // then we can be friendly and flatten the inputs to a list of strings.\n  const isLanguageModel =\n    typeof chainOrFactory === \"object\" &&\n    typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n  if (\n    isLanguageModel &&\n    examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n  ) {\n    return examples.map(({ inputs }) => Object.values(inputs)[0]);\n  }\n  return examples.map(({ inputs }) => inputs);\n};\n\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n *   - `projectName` (string): Name of the project for logging and tracking.\n *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n *   - `client` (Client): Client instance for LangSmith service interaction.\n *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n *   const chain = /* ...create your model or chain...*\\//\n *   const datasetName = 'example-dataset';\n *   const client = new Client(/* ...config... *\\//);\n *\n *   const results = await runOnDataset(chain, datasetName, {\n *     evaluators: [/* ...evaluators... *\\//],\n *     client,\n *   });\n *\n *   console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n  chainOrFactory: ChainOrFactory,\n  datasetName: string,\n  options?: RunOnDatasetParams\n) {\n  const {\n    projectName,\n    projectMetadata,\n    client,\n    maxConcurrency,\n  }: RunOnDatasetParams = options ?? {};\n\n  const evaluationConfig: RunEvalConfig | undefined =\n    options?.evaluationConfig ??\n    (options?.evaluators != null\n      ? {\n          evaluators: options.evaluators,\n          formatEvaluatorInputs: options.formatEvaluatorInputs,\n        }\n      : undefined);\n\n  const wrappedModel = await createWrappedModel(chainOrFactory);\n  const testClient = client ?? new Client();\n  const testProjectName = projectName ?? randomName();\n  const dataset = await testClient.readDataset({ datasetName });\n  const datasetId = dataset.id;\n  const testConcurrency = maxConcurrency ?? 5;\n  const { configs, examples, runExtractors } = await loadExamples({\n    datasetName,\n    client: testClient,\n    projectName: testProjectName,\n    maxConcurrency: testConcurrency,\n  });\n\n  await testClient.createProject({\n    projectName: testProjectName,\n    referenceDatasetId: datasetId,\n    projectExtra: { metadata: { ...projectMetadata } },\n  });\n  const wrappedRunnable: Runnable = new RunnableLambda({\n    func: wrappedModel,\n  }).withConfig({ runName: \"evaluationRun\" });\n  const runInputs = getExamplesInputs(\n    examples,\n    chainOrFactory,\n    dataset.data_type\n  );\n  const progress = new ProgressBar({\n    total: runInputs.length,\n    format: \"Predicting: {bar} {percentage}% | {value}/{total}\",\n  });\n  // TODO: Collect the runs as well.\n  await wrappedRunnable\n    .withListeners({\n      onEnd: () => progress.increment(),\n    })\n    // TODO: Insert evaluation inline for immediate feedback.\n    .batch(runInputs, configs, {\n      maxConcurrency,\n      returnExceptions: true,\n    });\n\n  progress.complete();\n  const runs: Run[] = [];\n  for (let i = 0; i < examples.length; i += 1) {\n    runs.push(await runExtractors[i].extract());\n  }\n  let evalResults: Record<\n    string,\n    { run_id: string; execution_time?: number; feedback: Feedback[] }\n  > = {};\n  if (evaluationConfig) {\n    const loadedEvalConfig =\n      await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);\n    evalResults = await applyEvaluators({\n      evaluation: loadedEvalConfig,\n      runs,\n      examples,\n      client: testClient,\n      maxConcurrency: testConcurrency,\n    });\n  }\n  const results: EvalResults = {\n    projectName: testProjectName,\n    results: evalResults ?? {},\n  };\n  return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n  return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiC,WAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAI,eAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYD,KAAUE,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+B,QAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqD,SAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYE,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,MAAM,4BAA4B,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAI,QAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEC,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAM,cAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYZ,KAAUE,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAI,gBAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBW,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAO,kBAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAO,uBAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAwCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAI,SAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAI,eAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMC,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAI,gBAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAI,YAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAC7B,gCAAgC,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAI;CACjC,MAAM,kBAAkB,eAAe,YAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAI,eAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAI,YAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMC,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBACJ,MAAM,iBAAiB,kBAAkB,iBAAiB;EAC5D,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/classic",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "Old abstractions from LangChain.js",
   "author": "LangChain",
   "license": "MIT",
@@ -106,7 +106,7 @@
     "@getmetal/metal-sdk": "^4.0.0",
     "@getzep/zep-cloud": "^1.0.12",
     "@getzep/zep-js": "^0.9.0",
-    "@gomomento/sdk": "1.51.1",
+    "@gomomento/sdk": "1.117.2",
     "@google/generative-ai": "^0.7.0",
     "@lancedb/lancedb": "^0.19.1",
     "@langchain/langgraph": "1.0.0-alpha.5",
@@ -125,13 +125,13 @@
     "@types/pg": "^8.15.5",
     "@types/uuid": "^9",
     "@types/ws": "^8",
-    "@upstash/redis": "^1.35.3",
+    "@upstash/redis": "^1.36.2",
     "@upstash/vector": "^1.2.2",
     "@vercel/kv": "^3.0.0",
     "@vitest/coverage-v8": "^3.2.4",
     "@xata.io/client": "^0.28.4",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
-    "axios": "^1.13.3",
+    "axios": "^1.13.5",
     "cheerio": "1.0.0-rc.12",
     "chromadb": "^1.5.3",
     "cohere-ai": "^7.14.0",
@@ -142,7 +142,7 @@
     "duck-duck-scrape": "^2.2.5",
     "eslint": "^9.36.0",
     "exa-js": "^1.0.12",
-    "firebase-admin": "^12.0.0",
+    "firebase-admin": "^13.6.1",
     "graphql": "^16.6.0",
     "hdb": "^0.19.8",
     "ioredis": "^5.3.2",
@@ -163,29 +163,30 @@
     "reflect-metadata": "^0.2.2",
     "rimraf": "^5.0.1",
     "sqlite3": "^5.1.4",
-    "typeorm": "^0.3.26",
+    "typeorm": "^0.3.28",
     "typescript": "~5.8.3",
-    "typesense": "^1.5.3",
+    "typesense": "^3.0.1",
     "vitest": "^3.2.4",
     "voy-search": "0.6.2",
     "weaviate-client": "^3.8.0",
     "zod-to-json-schema": "^3.24.6",
-    "@langchain/anthropic": "1.3.15",
+    "@langchain/anthropic": "1.3.17",
     "@langchain/aws": "1.2.2",
     "@langchain/azure-cosmosdb": "1.1.0",
     "@langchain/azure-dynamic-sessions": "1.0.1",
     "@langchain/baidu-qianfan": "1.0.1",
     "@langchain/cloudflare": "1.0.2",
     "@langchain/cohere": "1.0.2",
-    "@langchain/core": "1.1.19",
-    "@langchain/deepseek": "1.0.9",
+    "@langchain/core": "^1.1.21",
+    "@langchain/deepseek": "1.0.11",
     "@langchain/eslint": "0.1.1",
-    "@langchain/google-cloud-sql-pg": "1.0.20",
-    "@langchain/google-common": "2.1.15",
-    "@langchain/google-genai": "2.1.15",
-    "@langchain/google-vertexai": "2.1.15",
-    "@langchain/google-vertexai-web": "2.1.15",
-    "@langchain/groq": "1.0.4",
+    "@langchain/exa": "1.0.1",
+    "@langchain/google-cloud-sql-pg": "1.0.21",
+    "@langchain/google-common": "2.1.17",
+    "@langchain/google-genai": "2.1.17",
+    "@langchain/google-vertexai": "2.1.17",
+    "@langchain/google-vertexai-web": "2.1.17",
+    "@langchain/groq": "1.1.0",
     "@langchain/mistralai": "1.0.4",
     "@langchain/mongodb": "1.1.0",
     "@langchain/nomic": "1.0.1",
@@ -196,11 +197,10 @@
     "@langchain/tavily": "1.2.0",
     "@langchain/textsplitters": "1.0.1",
     "@langchain/weaviate": "1.0.1",
-    "@langchain/xai": "1.3.1",
+    "@langchain/xai": "1.3.3",
     "@langchain/yandex": "1.0.1",
     "@langchain/tsconfig": "0.0.1",
-    "langchain": "1.2.18",
-    "@langchain/exa": "1.0.1"
+    "langchain": "1.2.20"
   },
   "peerDependencies": {
     "@langchain/core": "^1.0.0",
@@ -227,7 +227,7 @@
     "uuid": "^10.0.0",
     "yaml": "^2.2.1",
     "zod": "^3.25.76 || ^4",
-    "@langchain/openai": "1.2.5",
+    "@langchain/openai": "1.2.7",
     "@langchain/textsplitters": "1.0.1"
   },
   "optionalDependencies": {