@langchain/core 1.1.24 → 1.1.25

package/dist/example_selectors/semantic_similarity.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.cjs","names":["values: Record<string, T>","BaseExampleSelector","data: SemanticSimilarityExampleSelectorInput<V>","example: Example","Document","inputVariables: Record<string, T>","examples: Record<string, string>[]","embeddings: Embeddings","vectorStoreCls: C","options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3]"],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;;AAUA,SAAS,aAAgBA,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,IAAI,CAAC,QAAQ,OAAO,KAAK;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDD,IAAa,oCAAb,MAAa,0CAEHC,iCAAoB;CAC5B;CAEA;CAEA;CAEA,YAAYC,MAAiD;EAC3D,MAAM,KAAK;EACX,KAAK,cAAc,KAAK;EACxB,KAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,QACvB,KAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;EACd,EAAC;WACO,KAAK,sBACd,KAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,CAAC,iEAAiE,CAAC;CAGxE;;;;;;;CAQD,MAAM,WAAWC,SAAiC;EAChD,MAAM,YAAY,KAAK,aAAa,OAAO,KAAK,QAAQ;EACxD,MAAM,gBAAgB,aACpB,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,CACF,CAAC,KAAK,IAAI;EAEX,MAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAIC,0BAAS;GACX,aAAa;GACb,UAAU;EACX,EACF,EAAC;CACH;;;;;;;;CASD,MAAM,eACJC,gBACoB;EACpB,MAAM,YAAY,KAAK,aAAa,OAAO,KAAK,eAAe;EAC/D,MAAM,QAAQ,aACZ,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;EAAM,IACrD,CAAE,EACH,CACF,CAAC,KAAK,IAAI;EAEX,MAAM,cAAc,MAAM,KAAK,qBAAqB,OAAO,MAAM;EAEjE,MAAM,WAAW,YAAY,IAAI,CAAC,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,IAAI,CAAC,YAClB,KAAK,YAAyB,OAC7B,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,CACF;AAEH,SAAO;CACR;;;;;;;;;;;;;;;CAgBD,aAAa,aACXC,UACAC,YACAC,gBACAC,UAGoC,CAAE,GACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,IAAI,CAAC,YACnC,aACE,YACI,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;EAED,MAAM,cAAc,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;AAED,SAAO,IAAI,kCAAkC;GAC3C;GACA,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;EACpB;CACF;AACF"}
+{"version":3,"file":"semantic_similarity.cjs","names":["BaseExampleSelector","Document"],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEHA,iCAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,OACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAIC,0BAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAFc,MAAM,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aARkB,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}

package/dist/example_selectors/semantic_similarity.d.cts

@@ -4,7 +4,6 @@ import { BaseExampleSelector } from "./base.cjs";
 import { VectorStore, VectorStoreInterface, VectorStoreRetrieverInterface } from "../vectorstores.cjs";
 
 //#region src/example_selectors/semantic_similarity.d.ts
-
 /**
  * Interface for the input data of the SemanticSimilarityExampleSelector
  * class.

package/dist/example_selectors/semantic_similarity.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.d.cts","names":["Embeddings","VectorStoreInterface","VectorStoreRetrieverInterface","VectorStore","Example","BaseExampleSelector","SemanticSimilarityExampleSelectorInput","V","SemanticSimilarityExampleSelector","T","C","Promise","Record","Parameters"],"sources":["../../src/example_selectors/semantic_similarity.d.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type { VectorStoreInterface, VectorStoreRetrieverInterface, VectorStore } from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<V extends VectorStoreInterface = VectorStoreInterface> = {\n    vectorStore: V;\n    k?: number;\n    filter?: V[\"FilterType\"];\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    vectorStoreRetriever?: never;\n} | {\n    vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    vectorStore?: never;\n    k?: never;\n    filter?: never;\n};\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport declare class SemanticSimilarityExampleSelector<V extends VectorStoreInterface = VectorStoreInterface> extends BaseExampleSelector {\n    vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    constructor(data: SemanticSimilarityExampleSelectorInput<V>);\n    /**\n     * Method that adds a new example to the vectorStore. The example is\n     * converted to a string and added to the vectorStore as a document.\n     * @param example The example to be added to the vectorStore.\n     * @returns Promise that resolves when the example has been added to the vectorStore.\n     */\n    addExample(example: Example): Promise<void>;\n    /**\n     * Method that selects which examples to use based on semantic similarity.\n     * It performs a similarity search in the vectorStore using the input\n     * variables and returns the examples with the highest similarity.\n     * @param inputVariables The input variables used for the similarity search.\n     * @returns Promise that resolves with an array of the selected examples.\n     */\n    selectExamples<T>(inputVariables: Record<string, T>): Promise<Example[]>;\n    /**\n     * Static method that creates a new instance of\n     * SemanticSimilarityExampleSelector. It takes a list of examples, an\n     * instance of Embeddings, a VectorStore class, and an options object as\n     * parameters. It converts the examples to strings, creates a VectorStore\n     * from the strings and the embeddings, and returns a new\n     * SemanticSimilarityExampleSelector with the created VectorStore and the\n     * options provided.\n     * @param examples The list of examples to be used.\n     * @param embeddings The instance of Embeddings to be used.\n     * @param vectorStoreCls The VectorStore class to be used.\n     * @param options The options object for the SemanticSimilarityExampleSelector.\n     * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n     */\n    static fromExamples<C extends typeof VectorStore>(examples: Record<string, string>[], embeddings: Embeddings, vectorStoreCls: C, options?: {\n        k?: number;\n        inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3]): Promise<SemanticSimilarityExampleSelector>;\n}\n//# sourceMappingURL=semantic_similarity.d.ts.map"],"mappings":";;;;;;;;;AAQA;;AAAoFC,KAAxEK,sCAAwEL,CAAAA,UAAvBA,oBAAuBA,GAAAA,oBAAAA,CAAAA,GAAAA;EACnEM,WAAAA,EAAAA,CAAAA;EAEJA,CAAAA,CAAAA,EAAAA,MAAAA;EAK2CA,MAAAA,CAAAA,EAL3CA,CAK2CA,CAAAA,YAAAA,CAAAA;EAA9BL,WAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAA6B,SAAA,CAAA,EAAA,MAAA,EAAA;EAoClCM,oBAAAA,CAAAA,EAAAA,KAAAA;CAA4CP,GAAAA;EAAuBA,oBAAAA,EApC9DC,6BAoC8DD,CApChCM,CAoCgCN,CAAAA;EAChCM,WAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAA9BL,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAGmCK,WAAAA,CAAAA,EAAAA,KAAAA;EAAvCD,CAAAA,CAAAA,EAAAA,KAAAA;EAOEF,MAAAA,CAAAA,EAAAA,KAAAA;CAAUO;;;;;;;;;;;;;;AAXuG;;;;;;;;;;;;;;;;cAApHH,4CAA4CP,uBAAuBA,8BAA8BI,mBAAAA;wBAC5FH,8BAA8BK;;;oBAGlCD,uCAAuCC;;;;;;;sBAOrCH,UAAUO;;;;;;;;oCAQIC,eAAeH,KAAKE,QAAQP;;;;;;;;;;;;;;;uCAezBD,uBAAuBS,sCAAsCZ,4BAA4BU;;;MAG1HG,WAAWH,qBAAqBC,QAAQH"}
+{"version":3,"file":"semantic_similarity.d.cts","names":[],"sources":["../../src/example_selectors/semantic_similarity.ts"],"mappings":";;;;;;;;AAoBA;;KAAY,sCAAA,WACA,oBAAA,GAAuB,oBAAA;EAG7B,WAAA,EAAa,CAAA;EACb,CAAA;EACA,MAAA,GAAS,CAAA;EACT,WAAA;EACA,SAAA;EACA,oBAAA;AAAA;EAGA,oBAAA,EAAsB,6BAAA,CAA8B,CAAA;EACpD,WAAA;EACA,SAAA;EACA,WAAA;EACA,CAAA;EACA,MAAA;AAAA;;;;;;;;;;;;;;;;AAgCN;;;;;;;;;;;;;;cAAa,iCAAA,WACD,oBAAA,GAAuB,oBAAA,UACzB,mBAAA;EACR,oBAAA,EAAsB,6BAAA,CAA8B,CAAA;EAEpD,WAAA;EAEA,SAAA;EAEA,WAAA,CAAY,IAAA,EAAM,sCAAA,CAAuC,CAAA;EA+FxC;;;;;;EAvEX,UAAA,CAAW,OAAA,EAAS,OAAA,GAAU,OAAA;EAhCpC;;;;;;;EAwDM,cAAA,GAAA,CACJ,cAAA,EAAgB,MAAA,SAAe,CAAA,IAC9B,OAAA,CAAQ,OAAA;EApDX;;;;;;;;;;;;;;EAAA,OA0Fa,YAAA,kBAA8B,WAAA,CAAA,CACzC,QAAA,EAAU,MAAA,oBACV,UAAA,EAAY,UAAA,EACZ,cAAA,EAAgB,CAAA,EAChB,OAAA;IACE,CAAA;IACA,SAAA;EAAA,IACE,UAAA,CAAW,CAAA,oBACd,OAAA,CAAQ,iCAAA;AAAA"}

package/dist/example_selectors/semantic_similarity.d.ts

@@ -4,7 +4,6 @@ import { BaseExampleSelector } from "./base.js";
 import { VectorStore, VectorStoreInterface, VectorStoreRetrieverInterface } from "../vectorstores.js";
 
 //#region src/example_selectors/semantic_similarity.d.ts
-
 /**
  * Interface for the input data of the SemanticSimilarityExampleSelector
  * class.

package/dist/example_selectors/semantic_similarity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.d.ts","names":["Embeddings","VectorStoreInterface","VectorStoreRetrieverInterface","VectorStore","Example","BaseExampleSelector","SemanticSimilarityExampleSelectorInput","V","SemanticSimilarityExampleSelector","T","C","Promise","Record","Parameters"],"sources":["../../src/example_selectors/semantic_similarity.d.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type { VectorStoreInterface, VectorStoreRetrieverInterface, VectorStore } from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<V extends VectorStoreInterface = VectorStoreInterface> = {\n    vectorStore: V;\n    k?: number;\n    filter?: V[\"FilterType\"];\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    vectorStoreRetriever?: never;\n} | {\n    vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    vectorStore?: never;\n    k?: never;\n    filter?: never;\n};\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport declare class SemanticSimilarityExampleSelector<V extends VectorStoreInterface = VectorStoreInterface> extends BaseExampleSelector {\n    vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n    exampleKeys?: string[];\n    inputKeys?: string[];\n    constructor(data: SemanticSimilarityExampleSelectorInput<V>);\n    /**\n     * Method that adds a new example to the vectorStore. The example is\n     * converted to a string and added to the vectorStore as a document.\n     * @param example The example to be added to the vectorStore.\n     * @returns Promise that resolves when the example has been added to the vectorStore.\n     */\n    addExample(example: Example): Promise<void>;\n    /**\n     * Method that selects which examples to use based on semantic similarity.\n     * It performs a similarity search in the vectorStore using the input\n     * variables and returns the examples with the highest similarity.\n     * @param inputVariables The input variables used for the similarity search.\n     * @returns Promise that resolves with an array of the selected examples.\n     */\n    selectExamples<T>(inputVariables: Record<string, T>): Promise<Example[]>;\n    /**\n     * Static method that creates a new instance of\n     * SemanticSimilarityExampleSelector. It takes a list of examples, an\n     * instance of Embeddings, a VectorStore class, and an options object as\n     * parameters. It converts the examples to strings, creates a VectorStore\n     * from the strings and the embeddings, and returns a new\n     * SemanticSimilarityExampleSelector with the created VectorStore and the\n     * options provided.\n     * @param examples The list of examples to be used.\n     * @param embeddings The instance of Embeddings to be used.\n     * @param vectorStoreCls The VectorStore class to be used.\n     * @param options The options object for the SemanticSimilarityExampleSelector.\n     * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n     */\n    static fromExamples<C extends typeof VectorStore>(examples: Record<string, string>[], embeddings: Embeddings, vectorStoreCls: C, options?: {\n        k?: number;\n        inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3]): Promise<SemanticSimilarityExampleSelector>;\n}\n//# sourceMappingURL=semantic_similarity.d.ts.map"],"mappings":";;;;;;;;;AAQA;;AAAoFC,KAAxEK,sCAAwEL,CAAAA,UAAvBA,oBAAuBA,GAAAA,oBAAAA,CAAAA,GAAAA;EACnEM,WAAAA,EAAAA,CAAAA;EAEJA,CAAAA,CAAAA,EAAAA,MAAAA;EAK2CA,MAAAA,CAAAA,EAL3CA,CAK2CA,CAAAA,YAAAA,CAAAA;EAA9BL,WAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAA6B,SAAA,CAAA,EAAA,MAAA,EAAA;EAoClCM,oBAAAA,CAAAA,EAAAA,KAAAA;CAA4CP,GAAAA;EAAuBA,oBAAAA,EApC9DC,6BAoC8DD,CApChCM,CAoCgCN,CAAAA;EAChCM,WAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAA9BL,SAAAA,CAAAA,EAAAA,MAAAA,EAAAA;EAGmCK,WAAAA,CAAAA,EAAAA,KAAAA;EAAvCD,CAAAA,CAAAA,EAAAA,KAAAA;EAOEF,MAAAA,CAAAA,EAAAA,KAAAA;CAAUO;;;;;;;;;;;;;;AAXuG;;;;;;;;;;;;;;;;cAApHH,4CAA4CP,uBAAuBA,8BAA8BI,mBAAAA;wBAC5FH,8BAA8BK;;;oBAGlCD,uCAAuCC;;;;;;;sBAOrCH,UAAUO;;;;;;;;oCAQIC,eAAeH,KAAKE,QAAQP;;;;;;;;;;;;;;;uCAezBD,uBAAuBS,sCAAsCZ,4BAA4BU;;;MAG1HG,WAAWH,qBAAqBC,QAAQH"}
+{"version":3,"file":"semantic_similarity.d.ts","names":[],"sources":["../../src/example_selectors/semantic_similarity.ts"],"mappings":";;;;;;;;AAoBA;;KAAY,sCAAA,WACA,oBAAA,GAAuB,oBAAA;EAG7B,WAAA,EAAa,CAAA;EACb,CAAA;EACA,MAAA,GAAS,CAAA;EACT,WAAA;EACA,SAAA;EACA,oBAAA;AAAA;EAGA,oBAAA,EAAsB,6BAAA,CAA8B,CAAA;EACpD,WAAA;EACA,SAAA;EACA,WAAA;EACA,CAAA;EACA,MAAA;AAAA;;;;;;;;;;;;;;;;AAgCN;;;;;;;;;;;;;;cAAa,iCAAA,WACD,oBAAA,GAAuB,oBAAA,UACzB,mBAAA;EACR,oBAAA,EAAsB,6BAAA,CAA8B,CAAA;EAEpD,WAAA;EAEA,SAAA;EAEA,WAAA,CAAY,IAAA,EAAM,sCAAA,CAAuC,CAAA;EA+FxC;;;;;;EAvEX,UAAA,CAAW,OAAA,EAAS,OAAA,GAAU,OAAA;EAhCpC;;;;;;;EAwDM,cAAA,GAAA,CACJ,cAAA,EAAgB,MAAA,SAAe,CAAA,IAC9B,OAAA,CAAQ,OAAA;EApDX;;;;;;;;;;;;;;EAAA,OA0Fa,YAAA,kBAA8B,WAAA,CAAA,CACzC,QAAA,EAAU,MAAA,oBACV,UAAA,EAAY,UAAA,EACZ,cAAA,EAAgB,CAAA,EAChB,OAAA;IACE,CAAA;IACA,SAAA;EAAA,IACE,UAAA,CAAW,CAAA,oBACd,OAAA,CAAQ,iCAAA;AAAA"}

package/dist/example_selectors/semantic_similarity.js

@@ -56,8 +56,7 @@ var SemanticSimilarityExampleSelector = class SemanticSimilarityExampleSelector
 	* @returns Promise that resolves when the example has been added to the vectorStore.
 	*/
 	async addExample(example) {
-		const inputKeys = this.inputKeys ?? Object.keys(example);
-		const stringExample = sortedValues(inputKeys.reduce((acc, key) => ({
+		const stringExample = sortedValues((this.inputKeys ?? Object.keys(example)).reduce((acc, key) => ({
 			...acc,
 			[key]: example[key]
 		}), {})).join(" ");
@@ -74,13 +73,11 @@ var SemanticSimilarityExampleSelector = class SemanticSimilarityExampleSelector
 	* @returns Promise that resolves with an array of the selected examples.
 	*/
 	async selectExamples(inputVariables) {
-		const inputKeys = this.inputKeys ?? Object.keys(inputVariables);
-		const query = sortedValues(inputKeys.reduce((acc, key) => ({
+		const query = sortedValues((this.inputKeys ?? Object.keys(inputVariables)).reduce((acc, key) => ({
 			...acc,
 			[key]: inputVariables[key]
 		}), {})).join(" ");
-		const exampleDocs = await this.vectorStoreRetriever.invoke(query);
-		const examples = exampleDocs.map((doc) => doc.metadata);
+		const examples = (await this.vectorStoreRetriever.invoke(query)).map((doc) => doc.metadata);
 		if (this.exampleKeys) return examples.map((example) => this.exampleKeys.reduce((acc, key) => ({
 			...acc,
 			[key]: example[key]
@@ -107,9 +104,8 @@ var SemanticSimilarityExampleSelector = class SemanticSimilarityExampleSelector
 			...acc,
 			[key]: example[key]
 		}), {}) : example).join(" "));
-		const vectorStore = await vectorStoreCls.fromTexts(stringExamples, examples, embeddings, options);
 		return new SemanticSimilarityExampleSelector({
-			vectorStore,
+			vectorStore: await vectorStoreCls.fromTexts(stringExamples, examples, embeddings, options),
 			k: options.k ?? 4,
 			exampleKeys: options.exampleKeys,
 			inputKeys: options.inputKeys

package/dist/example_selectors/semantic_similarity.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic_similarity.js","names":["values: Record<string, T>","data: SemanticSimilarityExampleSelectorInput<V>","example: Example","inputVariables: Record<string, T>","examples: Record<string, string>[]","embeddings: Embeddings","vectorStoreCls: C","options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3]"],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;;AAUA,SAAS,aAAgBA,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,IAAI,CAAC,QAAQ,OAAO,KAAK;AAC7B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDD,IAAa,oCAAb,MAAa,0CAEH,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAYC,MAAiD;EAC3D,MAAM,KAAK;EACX,KAAK,cAAc,KAAK;EACxB,KAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,QACvB,KAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;EACd,EAAC;WACO,KAAK,sBACd,KAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,CAAC,iEAAiE,CAAC;CAGxE;;;;;;;CAQD,MAAM,WAAWC,SAAiC;EAChD,MAAM,YAAY,KAAK,aAAa,OAAO,KAAK,QAAQ;EACxD,MAAM,gBAAgB,aACpB,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,CACF,CAAC,KAAK,IAAI;EAEX,MAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAI,SAAS;GACX,aAAa;GACb,UAAU;EACX,EACF,EAAC;CACH;;;;;;;;CASD,MAAM,eACJC,gBACoB;EACpB,MAAM,YAAY,KAAK,aAAa,OAAO,KAAK,eAAe;EAC/D,MAAM,QAAQ,aACZ,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;EAAM,IACrD,CAAE,EACH,CACF,CAAC,KAAK,IAAI;EAEX,MAAM,cAAc,MAAM,KAAK,qBAAqB,OAAO,MAAM;EAEjE,MAAM,WAAW,YAAY,IAAI,CAAC,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,IAAI,CAAC,YAClB,KAAK,YAAyB,OAC7B,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,CACF;AAEH,SAAO;CACR;;;;;;;;;;;;;;;CAgBD,aAAa,aACXC,UACAC,YACAC,gBACAC,UAGoC,CAAE,GACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,IAAI,CAAC,YACnC,aACE,YACI,UAAU,OACR,CAAC,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;EAAM,IAC9C,CAAE,EACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;EAED,MAAM,cAAc,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;AAED,SAAO,IAAI,kCAAkC;GAC3C;GACA,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;EACpB;CACF;AACF"}
+{"version":3,"file":"semantic_similarity.js","names":[],"sources":["../../src/example_selectors/semantic_similarity.ts"],"sourcesContent":["import type { Embeddings } from \"../embeddings.js\";\nimport type {\n  VectorStoreInterface,\n  VectorStoreRetrieverInterface,\n  VectorStore,\n} from \"../vectorstores.js\";\nimport type { Example } from \"../prompts/base.js\";\nimport { Document } from \"../documents/document.js\";\nimport { BaseExampleSelector } from \"./base.js\";\n\nfunction sortedValues<T>(values: Record<string, T>): T[] {\n  return Object.keys(values)\n    .sort()\n    .map((key) => values[key]);\n}\n\n/**\n * Interface for the input data of the SemanticSimilarityExampleSelector\n * class.\n */\nexport type SemanticSimilarityExampleSelectorInput<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> =\n  | {\n      vectorStore: V;\n      k?: number;\n      filter?: V[\"FilterType\"];\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStoreRetriever?: never;\n    }\n  | {\n      vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n      exampleKeys?: string[];\n      inputKeys?: string[];\n      vectorStore?: never;\n      k?: never;\n      filter?: never;\n    };\n\n/**\n * Class that selects examples based on semantic similarity. It extends\n * the BaseExampleSelector class.\n * @example\n * ```typescript\n * const exampleSelector = await SemanticSimilarityExampleSelector.fromExamples(\n *   [\n *     { input: \"happy\", output: \"sad\" },\n *     { input: \"tall\", output: \"short\" },\n *     { input: \"energetic\", output: \"lethargic\" },\n *     { input: \"sunny\", output: \"gloomy\" },\n *     { input: \"windy\", output: \"calm\" },\n *   ],\n *   new OpenAIEmbeddings(),\n *   HNSWLib,\n *   { k: 1 },\n * );\n * const dynamicPrompt = new FewShotPromptTemplate({\n *   exampleSelector,\n *   examplePrompt: PromptTemplate.fromTemplate(\n *     \"Input: {input}\\nOutput: {output}\",\n *   ),\n *   prefix: \"Give the antonym of every input\",\n *   suffix: \"Input: {adjective}\\nOutput:\",\n *   inputVariables: [\"adjective\"],\n * });\n * console.log(await dynamicPrompt.format({ adjective: \"rainy\" }));\n * ```\n */\nexport class SemanticSimilarityExampleSelector<\n  V extends VectorStoreInterface = VectorStoreInterface,\n> extends BaseExampleSelector {\n  vectorStoreRetriever: VectorStoreRetrieverInterface<V>;\n\n  exampleKeys?: string[];\n\n  inputKeys?: string[];\n\n  constructor(data: SemanticSimilarityExampleSelectorInput<V>) {\n    super(data);\n    this.exampleKeys = data.exampleKeys;\n    this.inputKeys = data.inputKeys;\n    if (data.vectorStore !== undefined) {\n      this.vectorStoreRetriever = data.vectorStore.asRetriever({\n        k: data.k ?? 4,\n        filter: data.filter,\n      });\n    } else if (data.vectorStoreRetriever) {\n      this.vectorStoreRetriever = data.vectorStoreRetriever;\n    } else {\n      throw new Error(\n        `You must specify one of \"vectorStore\" and \"vectorStoreRetriever\".`\n      );\n    }\n  }\n\n  /**\n   * Method that adds a new example to the vectorStore. The example is\n   * converted to a string and added to the vectorStore as a document.\n   * @param example The example to be added to the vectorStore.\n   * @returns Promise that resolves when the example has been added to the vectorStore.\n   */\n  async addExample(example: Example): Promise<void> {\n    const inputKeys = this.inputKeys ?? Object.keys(example);\n    const stringExample = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: example[key] }),\n        {} as Example\n      )\n    ).join(\" \");\n\n    await this.vectorStoreRetriever.addDocuments([\n      new Document({\n        pageContent: stringExample,\n        metadata: example,\n      }),\n    ]);\n  }\n\n  /**\n   * Method that selects which examples to use based on semantic similarity.\n   * It performs a similarity search in the vectorStore using the input\n   * variables and returns the examples with the highest similarity.\n   * @param inputVariables The input variables used for the similarity search.\n   * @returns Promise that resolves with an array of the selected examples.\n   */\n  async selectExamples<T>(\n    inputVariables: Record<string, T>\n  ): Promise<Example[]> {\n    const inputKeys = this.inputKeys ?? Object.keys(inputVariables);\n    const query = sortedValues(\n      inputKeys.reduce(\n        (acc, key) => ({ ...acc, [key]: inputVariables[key] }),\n        {} as Record<string, T>\n      )\n    ).join(\" \");\n\n    const exampleDocs = await this.vectorStoreRetriever.invoke(query);\n\n    const examples = exampleDocs.map((doc) => doc.metadata);\n    if (this.exampleKeys) {\n      // If example keys are provided, filter examples to those keys.\n      return examples.map((example) =>\n        (this.exampleKeys as string[]).reduce(\n          (acc, key) => ({ ...acc, [key]: example[key] }),\n          {}\n        )\n      );\n    }\n    return examples;\n  }\n\n  /**\n   * Static method that creates a new instance of\n   * SemanticSimilarityExampleSelector. It takes a list of examples, an\n   * instance of Embeddings, a VectorStore class, and an options object as\n   * parameters. It converts the examples to strings, creates a VectorStore\n   * from the strings and the embeddings, and returns a new\n   * SemanticSimilarityExampleSelector with the created VectorStore and the\n   * options provided.\n   * @param examples The list of examples to be used.\n   * @param embeddings The instance of Embeddings to be used.\n   * @param vectorStoreCls The VectorStore class to be used.\n   * @param options The options object for the SemanticSimilarityExampleSelector.\n   * @returns Promise that resolves with a new instance of SemanticSimilarityExampleSelector.\n   */\n  static async fromExamples<C extends typeof VectorStore>(\n    examples: Record<string, string>[],\n    embeddings: Embeddings,\n    vectorStoreCls: C,\n    options: {\n      k?: number;\n      inputKeys?: string[];\n    } & Parameters<C[\"fromTexts\"]>[3] = {}\n  ): Promise<SemanticSimilarityExampleSelector> {\n    const inputKeys = options.inputKeys ?? null;\n    const stringExamples = examples.map((example) =>\n      sortedValues(\n        inputKeys\n          ? inputKeys.reduce(\n              (acc, key) => ({ ...acc, [key]: example[key] }),\n              {} as Record<string, string>\n            )\n          : example\n      ).join(\" \")\n    );\n\n    const vectorStore = await vectorStoreCls.fromTexts(\n      stringExamples,\n      examples, // metadatas\n      embeddings,\n      options\n    );\n\n    return new SemanticSimilarityExampleSelector({\n      vectorStore,\n      k: options.k ?? 4,\n      exampleKeys: options.exampleKeys,\n      inputKeys: options.inputKeys,\n    });\n  }\n}\n"],"mappings":";;;;AAUA,SAAS,aAAgB,QAAgC;AACvD,QAAO,OAAO,KAAK,OAAO,CACvB,MAAM,CACN,KAAK,QAAQ,OAAO,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwD9B,IAAa,oCAAb,MAAa,0CAEH,oBAAoB;CAC5B;CAEA;CAEA;CAEA,YAAY,MAAiD;AAC3D,QAAM,KAAK;AACX,OAAK,cAAc,KAAK;AACxB,OAAK,YAAY,KAAK;AACtB,MAAI,KAAK,gBAAgB,OACvB,MAAK,uBAAuB,KAAK,YAAY,YAAY;GACvD,GAAG,KAAK,KAAK;GACb,QAAQ,KAAK;GACd,CAAC;WACO,KAAK,qBACd,MAAK,uBAAuB,KAAK;MAEjC,OAAM,IAAI,MACR,oEACD;;;;;;;;CAUL,MAAM,WAAW,SAAiC;EAEhD,MAAM,gBAAgB,cADJ,KAAK,aAAa,OAAO,KAAK,QAAQ,EAE5C,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF,CAAC,KAAK,IAAI;AAEX,QAAM,KAAK,qBAAqB,aAAa,CAC3C,IAAI,SAAS;GACX,aAAa;GACb,UAAU;GACX,CAAC,CACH,CAAC;;;;;;;;;CAUJ,MAAM,eACJ,gBACoB;EAEpB,MAAM,QAAQ,cADI,KAAK,aAAa,OAAO,KAAK,eAAe,EAEnD,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,eAAe;GAAM,GACrD,EAAE,CACH,CACF,CAAC,KAAK,IAAI;EAIX,MAAM,YAFc,MAAM,KAAK,qBAAqB,OAAO,MAAM,EAEpC,KAAK,QAAQ,IAAI,SAAS;AACvD,MAAI,KAAK,YAEP,QAAO,SAAS,KAAK,YAClB,KAAK,YAAyB,QAC5B,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,CACF;AAEH,SAAO;;;;;;;;;;;;;;;;CAiBT,aAAa,aACX,UACA,YACA,gBACA,UAGoC,EAAE,EACM;EAC5C,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,iBAAiB,SAAS,KAAK,YACnC,aACE,YACI,UAAU,QACP,KAAK,SAAS;GAAE,GAAG;IAAM,MAAM,QAAQ;GAAM,GAC9C,EAAE,CACH,GACD,QACL,CAAC,KAAK,IAAI,CACZ;AASD,SAAO,IAAI,kCAAkC;GAC3C,aARkB,MAAM,eAAe,UACvC,gBACA,UACA,YACA,QACD;GAIC,GAAG,QAAQ,KAAK;GAChB,aAAa,QAAQ;GACrB,WAAW,QAAQ;GACpB,CAAC"}

package/dist/index.cjs

@@ -1,6 +1,8 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
+const require_runtime = require('./_virtual/_rolldown/runtime.cjs');
 
 //#region src/index.ts
-var src_exports = {};
+var src_exports = /* @__PURE__ */ require_runtime.__exportAll({});
 
 //#endregion
 Object.defineProperty(exports, 'src_exports', {

package/dist/index.js

@@ -1,5 +1,7 @@
+import { __exportAll } from "./_virtual/_rolldown/runtime.js";
+
 //#region src/index.ts
-var src_exports = {};
+var src_exports = /* @__PURE__ */ __exportAll({});
 
 //#endregion
 export { src_exports };

package/dist/indexing/base.cjs

@@ -1,9 +1,9 @@
-const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
 const require_hash = require('../utils/js-sha256/hash.cjs');
 require('../utils/hash.cjs');
 const require_document = require('../documents/document.cjs');
 const require_record_manager = require('./record_manager.cjs');
-const uuid = require_rolldown_runtime.__toESM(require("uuid"));
+let uuid = require("uuid");
 
 //#region src/indexing/base.ts
 /**
@@ -61,13 +61,11 @@ var _HashedDocument = class {
 		return doc;
 	}
 	_hashStringToUUID(inputString) {
-		const hash_value = this.keyEncoder(inputString);
-		return (0, uuid.v5)(hash_value, require_record_manager.UUIDV5_NAMESPACE);
+		return (0, uuid.v5)(this.keyEncoder(inputString), require_record_manager.UUIDV5_NAMESPACE);
 	}
 	_hashNestedDictToUUID(data) {
 		const serialized_data = JSON.stringify(data, Object.keys(data).sort());
-		const hash_value = this.keyEncoder(serialized_data);
-		return (0, uuid.v5)(hash_value, require_record_manager.UUIDV5_NAMESPACE);
+		return (0, uuid.v5)(this.keyEncoder(serialized_data), require_record_manager.UUIDV5_NAMESPACE);
 	}
 };
 function _batch(size, iterable) {
@@ -139,9 +137,8 @@ async function index(args) {
 	for (const batch of batches) {
 		const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
 		const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
-		if (cleanup === "incremental") hashedDocs.forEach((_hashedDoc, index$1) => {
-			const source = sourceIds[index$1];
-			if (source === null) throw new Error("sourceIdKey must be provided when cleanup is incremental");
+		if (cleanup === "incremental") hashedDocs.forEach((_hashedDoc, index) => {
+			if (sourceIds[index] === null) throw new Error("sourceIdKey must be provided when cleanup is incremental");
 		});
 		const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
 		const uids = [];
@@ -149,8 +146,7 @@ async function index(args) {
 		const docsToUpdate = [];
 		const seenDocs = /* @__PURE__ */ new Set();
 		hashedDocs.forEach((hashedDoc, i) => {
-			const docExists = batchExists[i];
-			if (docExists) if (forceUpdate) seenDocs.add(hashedDoc.uid);
+			if (batchExists[i]) if (forceUpdate) seenDocs.add(hashedDoc.uid);
 			else {
 				docsToUpdate.push(hashedDoc.uid);
 				return;

package/dist/indexing/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["sha256","fields: HashedDocumentArgs","keyEncoderFn: HashKeyEncoder","Document","document: DocumentInterface","uid?: string","inputString: string","UUIDV5_NAMESPACE","data: Record<string, unknown>","size: number","iterable: T[]","batches: T[][]","currentBatch: T[]","hashedDocuments: HashedDocumentInterface[]","deduplicated: HashedDocumentInterface[]","sourceIdKey: StringOrDocFunc | null","_doc: DocumentInterface","doc: DocumentInterface","arg: any","args: IndexArgs","index","uids: string[]","docsToIndex: DocumentInterface[]","docsToUpdate: string[]"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"uuid\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,AAAQ,aAA6BA;CAErC,YAAYC,QAA4B;EACtC,KAAK,MAAM,OAAO;EAClB,KAAK,cAAc,OAAO;EAC1B,KAAK,WAAW,OAAO;CACxB;CAED,sBAAsBC,cAAoC;EACxD,KAAK,aAAa;CACnB;CAED,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;EAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,CAAC,4BAA4B,EAAE,IAAI,uDAAuD,EAAE,cAAc,KACxG,KACD,CAAC,CAAC,CAAC;EAKV,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;GAC9D,KAAK,cAAc;GACnB,KAAK,eAAe;EACrB,SAAQ,GAAG;AACV,SAAM,IAAI,MACR,CAAC,yBAAyB,EAAE,EAAE,sDAAsD,CAAC;EAExF;EAED,KAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,KACR,KAAK,MAAM,KAAK;CAEnB;CAED,aAAgC;AAC9B,SAAO,IAAIC,0BAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;EAChB;CACF;CAED,OAAO,aACLC,UACAC,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;EAC/D;EACD,IAAI,iBAAiB;AACrB,SAAO;CACR;CAED,AAAQ,kBAAkBC,aAA6B;EACrD,MAAM,aAAa,KAAK,WAAW,YAAY;AAC/C,sBAAc,YAAYC,wCAAiB;CAC5C;CAED,AAAQ,sBAAsBC,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;EACtE,MAAM,aAAa,KAAK,WAAW,gBAAgB;AACnD,sBAAc,YAAYD,wCAAiB;CAC5C;AACF;AAyCD,SAAgB,OAAUE,MAAcC,UAAsB;CAC5D,MAAMC,UAAiB,CAAE;CACzB,IAAIC,eAAoB,CAAE;CAE1B,SAAS,QAAQ,CAAC,SAAS;EACzB,aAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;GAC/B,QAAQ,KAAK,aAAa;GAC1B,eAAe,CAAE;EAClB;CACF,EAAC;AAEF,KAAI,aAAa,SAAS,GACxB,QAAQ,KAAK,aAAa;AAG5B,QAAO;AACR;AAED,SAAgB,oBACdC,iBAC2B;CAC3B,MAAM,uBAAO,IAAI;CACjB,MAAMC,eAA0C,CAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM;AAGlB,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;GAC9B,KAAK,IAAI,UAAU,MAAM;GACzB,aAAa,KAAK,UAAU;EAC7B;CACF;AACD,QAAO;AACR;AAED,SAAgB,qBACdC,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,QAAO,CAACC,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,QAAO,CAACC,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,CAAC,wDAAwD,EAAE,OAAO,aAAa;AAGpF;AAGD,MAAa,wBAAwB,CAACC,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;AACR;;;;;;;;;;;;;;;;;;;;AA4BD,eAAsB,MAAMC,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,SAAS,GAAG;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,OACf,GAAG,WAAW,CAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR;CAIJ,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,IAAI,CAAC,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,IAAI,CAAC,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,eACd,WAAW,QAAQ,CAAC,YAAYC,YAAU;GACxC,MAAM,SAAS,UAAUA;AACzB,OAAI,WAAW,KACb,OAAM,IAAI,MACR;EAGL,EAAC;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,IAAI,CAAC,QAAQ,IAAI,IAAI,CACjC;EAED,MAAMC,OAAiB,CAAE;EACzB,MAAMC,cAAmC,CAAE;EAC3C,MAAMC,eAAyB,CAAE;EACjC,MAAM,2BAAW,IAAI;EACrB,WAAW,QAAQ,CAAC,WAAW,MAAM;GACnC,MAAM,YAAY,YAAY;AAC9B,OAAI,UACF,KAAI,aACF,SAAS,IAAI,UAAU,IAAI;QACtB;IACL,aAAa,KAAK,UAAU,IAAI;AAChC;GACD;GAEH,KAAK,KAAK,UAAU,IAAI;GACxB,YAAY,KAAK,UAAU,YAAY,CAAC;EACzC,EAAC;AAEF,MAAI,aAAa,SAAS,GAAG;GAC3B,MAAM,cAAc,OAAO,cAAc,EAAE,aAAa,aAAc,EAAC;GACvE,cAAc,aAAa;EAC5B;AAED,MAAI,YAAY,SAAS,GAAG;GAC1B,MAAM,YAAY,aAAa,aAAa,EAAE,KAAK,KAAM,EAAC;GAC1D,YAAY,YAAY,SAAS,SAAS;GAC1C,cAAc,SAAS;EACxB;EAED,MAAM,cAAc,OAClB,WAAW,IAAI,CAAC,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;EAAW,EACnD;AAED,MAAI,YAAY,eAAe;GAC7B,UAAU,QAAQ,CAAC,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM;GAChC,EAAC;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;GACX,EAAC;AAEF,OAAI,aAAa,SAAS,GAAG;IAC3B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAc,EAAC;IAC/C,MAAM,cAAc,WAAW,aAAa;IAC5C,cAAc,aAAa;GAC5B;EACF;CACF;AAED,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;EACR,EAAC;AACF,SAAO,aAAa,SAAS,GAAG;GAC9B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAc,EAAC;GAC/C,MAAM,cAAc,WAAW,aAAa;GAC5C,cAAc,aAAa;GAC3B,eAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;GACR,EAAC;EACH;CACF;AAED,QAAO;EACL;EACA;EACA;EACA;CACD;AACF"}
+{"version":3,"file":"base.cjs","names":["sha256","Document","UUIDV5_NAMESPACE"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"uuid\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,AAAQ,aAA6BA;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAIC,0BAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,AAAQ,kBAAkB,aAA6B;AAErD,sBADmB,KAAK,WAAW,YAAY,EACrBC,wCAAiB;;CAG7C,AAAQ,sBAAsB,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,sBADmB,KAAK,WAAW,gBAAgB,EACzBA,wCAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}

package/dist/indexing/base.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.cts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAMHJ;KAPTG,eAAAA,GASaL,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EATqBA,iBASrBA,EAAAA,GAAAA,MAAAA,CAAAA;AAR+BA,UAAhCM,uBAAAA,SAAgCN,iBAAAA,CAAAA;EAAiB,GAAA,EAAA,MAAA;EAUxDO,KAAAA,CAAAA,EAAAA,MAAAA;EAUWC,WAAAA,CAAAA,EAAAA,MAAe;EAMtBN,YAAAA,CAAAA,EAAAA,MAAAA;EAEUK,WAAAA,EAAAA,MAAAA;EACgBR,QAAAA,EAvB1BG,QAuB0BH;EAEtBC,eAAAA,EAAAA,EAAAA,IAAAA;EACgBA,UAAAA,EAAAA,EAxBhBA,iBAwBgBA;;UAtBxBO,kBAAAA,CAUsCD;EAAuB,WAAA,EAAA,MAAA;EAgB3DG,QAAAA,EAxBEP,QAwBS;EACXQ,GAAAA,EAAAA,MAAAA;AAoCZ;AACA;AACA;AACA;AAAoF;;AAE/CV,cA1DhBQ,eAAAA,YAA2BF,uBA0DXN,CAAAA;EAClBF,GAAAA,EAAAA,MAAAA;EACFD,KAAAA,CAAAA,EAAAA,MAAAA;EACHa,WAAAA,CAAAA,EAAAA,MAAAA;EAAY,YAAA,CAAA,EAAA,MAAA;EAqBFO,WAAK,EAAA,MAAA;EAAOD,QAAAA,EA5EtBd,QA4EsBc;EAAoBZ,QAAAA,UAAAA;EAARc,WAAAA,CAAAA,MAAAA,EA1ExBX,kBA0EwBW;EAAO,qBAAA,CAAA,YAAA,EAzEfnB,cAyEe,CAAA,EAAA,IAAA;;gBAvErCC;gCACgBA,kCAAkCQ;;;;KAIxDC,WAAAA;KACAC,YAAAA;;;;;;;;;;;;;;;;;;YAkBED;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;cACpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}
+{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/indexing/base.ts"],"mappings":";;;;;;;KAOK,QAAA,GAAW,MAAA;AAAA,KAEX,cAAA;EACH,QAAA;EACA,UAAA;EACA,UAAA;EACA,UAAA;AAAA;AAAA,KAGG,eAAA,cAA6B,GAAA,EAAK,iBAAA;AAAA,UAEtB,uBAAA,SAAgC,iBAAA;EAC/C,GAAA;EACA,KAAA;EACA,WAAA;EACA,YAAA;EACA,WAAA;EACA,QAAA,EAAU,QAAA;EACV,eAAA;EACA,UAAA,IAAc,iBAAA;AAAA;AAAA,UAGN,kBAAA;EACR,WAAA;EACA,QAAA,EAAU,QAAA;EACV,GAAA;AAAA;;;;;;cAQW,eAAA,YAA2B,uBAAA;EACtC,GAAA;EAEA,KAAA;EAEA,WAAA;EAEA,YAAA;EAEA,WAAA;EAEA,QAAA,EAAU,QAAA;EAAA,QAEF,UAAA;EAER,WAAA,CAAY,MAAA,EAAQ,kBAAA;EAMpB,qBAAA,CAAsB,YAAA,EAAc,cAAA;EAIpC,eAAA,CAAA;EAgCA,UAAA,CAAA,GAAc,iBAAA;EAAA,OAOP,YAAA,CACL,QAAA,EAAU,iBAAA,EACV,GAAA,YACC,eAAA;EAAA,QAUK,iBAAA;EAAA,QAKA,qBAAA;AAAA;AAAA,KAOE,WAAA;AAAA,KAEA,YAAA;EApGQ;;;EAwGlB,SAAA;EAvGA;;;AAQF;;;;;;;;;;EA6GE,OAAA,GAAU,WAAA;EA7GmD;;;;;;EAoH7D,WAAA,GAAc,eAAA;EAzGd;;;EA6GA,gBAAA;EAzGoB;;;;EA8GpB,WAAA;AAAA;AAAA,iBAGc,MAAA,GAAA,CAAU,IAAA,UAAc,QAAA,EAAU,CAAA,KAAM,CAAA;AAAA,iBAoBxC,mBAAA,CACd,eAAA,EAAiB,uBAAA,KAChB,uBAAA;AAAA,iBAiBa,oBAAA,CACd,WAAA,EAAa,eAAA,WACX,GAAA,EAAK,iBAAA;AAAA,cAeI,qBAAA,GAAqB,GAAA,UAAA,GAAA,IAAA,kBAAA;AAAA,UAYxB,SAAA;EACR,UAAA,EAAY,kBAAA,GAAqB,iBAAA;EACjC,aAAA,EAAe,sBAAA;EACf,WAAA,EAAa,WAAA;EACb,OAAA,GAAU,YAAA;AAAA;;AA/GZ;;;;;AAEA;;;;;;;;;;;;;iBAmIsB,KAAA,CAAM,IAAA,EAAM,SAAA,GAAY,OAAA,CAAQ,cAAA"}

package/dist/indexing/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n//# sourceMappingURL=base.d.ts.map"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAMHJ;KAPTG,eAAAA,GASaL,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EATqBA,iBASrBA,EAAAA,GAAAA,MAAAA,CAAAA;AAR+BA,UAAhCM,uBAAAA,SAAgCN,iBAAAA,CAAAA;EAAiB,GAAA,EAAA,MAAA;EAUxDO,KAAAA,CAAAA,EAAAA,MAAAA;EAUWC,WAAAA,CAAAA,EAAAA,MAAe;EAMtBN,YAAAA,CAAAA,EAAAA,MAAAA;EAEUK,WAAAA,EAAAA,MAAAA;EACgBR,QAAAA,EAvB1BG,QAuB0BH;EAEtBC,eAAAA,EAAAA,EAAAA,IAAAA;EACgBA,UAAAA,EAAAA,EAxBhBA,iBAwBgBA;;UAtBxBO,kBAAAA,CAUsCD;EAAuB,WAAA,EAAA,MAAA;EAgB3DG,QAAAA,EAxBEP,QAwBS;EACXQ,GAAAA,EAAAA,MAAAA;AAoCZ;AACA;AACA;AACA;AAAoF;;AAE/CV,cA1DhBQ,eAAAA,YAA2BF,uBA0DXN,CAAAA;EAClBF,GAAAA,EAAAA,MAAAA;EACFD,KAAAA,CAAAA,EAAAA,MAAAA;EACHa,WAAAA,CAAAA,EAAAA,MAAAA;EAAY,YAAA,CAAA,EAAA,MAAA;EAqBFO,WAAK,EAAA,MAAA;EAAOD,QAAAA,EA5EtBd,QA4EsBc;EAAoBZ,QAAAA,UAAAA;EAARc,WAAAA,CAAAA,MAAAA,EA1ExBX,kBA0EwBW;EAAO,qBAAA,CAAA,YAAA,EAzEfnB,cAyEe,CAAA,EAAA,IAAA;;gBAvErCC;gCACgBA,kCAAkCQ;;;;KAIxDC,WAAAA;KACAC,YAAAA;;;;;;;;;;;;;;;;;;YAkBED;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;cACpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}
+{"version":3,"file":"base.d.ts","names":[],"sources":["../../src/indexing/base.ts"],"mappings":";;;;;;;KAOK,QAAA,GAAW,MAAA;AAAA,KAEX,cAAA;EACH,QAAA;EACA,UAAA;EACA,UAAA;EACA,UAAA;AAAA;AAAA,KAGG,eAAA,cAA6B,GAAA,EAAK,iBAAA;AAAA,UAEtB,uBAAA,SAAgC,iBAAA;EAC/C,GAAA;EACA,KAAA;EACA,WAAA;EACA,YAAA;EACA,WAAA;EACA,QAAA,EAAU,QAAA;EACV,eAAA;EACA,UAAA,IAAc,iBAAA;AAAA;AAAA,UAGN,kBAAA;EACR,WAAA;EACA,QAAA,EAAU,QAAA;EACV,GAAA;AAAA;;;;;;cAQW,eAAA,YAA2B,uBAAA;EACtC,GAAA;EAEA,KAAA;EAEA,WAAA;EAEA,YAAA;EAEA,WAAA;EAEA,QAAA,EAAU,QAAA;EAAA,QAEF,UAAA;EAER,WAAA,CAAY,MAAA,EAAQ,kBAAA;EAMpB,qBAAA,CAAsB,YAAA,EAAc,cAAA;EAIpC,eAAA,CAAA;EAgCA,UAAA,CAAA,GAAc,iBAAA;EAAA,OAOP,YAAA,CACL,QAAA,EAAU,iBAAA,EACV,GAAA,YACC,eAAA;EAAA,QAUK,iBAAA;EAAA,QAKA,qBAAA;AAAA;AAAA,KAOE,WAAA;AAAA,KAEA,YAAA;EApGQ;;;EAwGlB,SAAA;EAvGA;;;AAQF;;;;;;;;;;EA6GE,OAAA,GAAU,WAAA;EA7GmD;;;;;;EAoH7D,WAAA,GAAc,eAAA;EAzGd;;;EA6GA,gBAAA;EAzGoB;;;;EA8GpB,WAAA;AAAA;AAAA,iBAGc,MAAA,GAAA,CAAU,IAAA,UAAc,QAAA,EAAU,CAAA,KAAM,CAAA;AAAA,iBAoBxC,mBAAA,CACd,eAAA,EAAiB,uBAAA,KAChB,uBAAA;AAAA,iBAiBa,oBAAA,CACd,WAAA,EAAa,eAAA,WACX,GAAA,EAAK,iBAAA;AAAA,cAeI,qBAAA,GAAqB,GAAA,UAAA,GAAA,IAAA,kBAAA;AAAA,UAYxB,SAAA;EACR,UAAA,EAAY,kBAAA,GAAqB,iBAAA;EACjC,aAAA,EAAe,sBAAA;EACf,WAAA,EAAa,WAAA;EACb,OAAA,GAAU,YAAA;AAAA;;AA/GZ;;;;;AAEA;;;;;;;;;;;;;iBAmIsB,KAAA,CAAM,IAAA,EAAM,SAAA,GAAY,OAAA,CAAQ,cAAA"}

package/dist/indexing/base.js

@@ -60,13 +60,11 @@ var _HashedDocument = class {
 		return doc;
 	}
 	_hashStringToUUID(inputString) {
-		const hash_value = this.keyEncoder(inputString);
-		return v5(hash_value, UUIDV5_NAMESPACE);
+		return v5(this.keyEncoder(inputString), UUIDV5_NAMESPACE);
 	}
 	_hashNestedDictToUUID(data) {
 		const serialized_data = JSON.stringify(data, Object.keys(data).sort());
-		const hash_value = this.keyEncoder(serialized_data);
-		return v5(hash_value, UUIDV5_NAMESPACE);
+		return v5(this.keyEncoder(serialized_data), UUIDV5_NAMESPACE);
 	}
 };
 function _batch(size, iterable) {
@@ -138,9 +136,8 @@ async function index(args) {
 	for (const batch of batches) {
 		const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
 		const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
-		if (cleanup === "incremental") hashedDocs.forEach((_hashedDoc, index$1) => {
-			const source = sourceIds[index$1];
-			if (source === null) throw new Error("sourceIdKey must be provided when cleanup is incremental");
+		if (cleanup === "incremental") hashedDocs.forEach((_hashedDoc, index) => {
+			if (sourceIds[index] === null) throw new Error("sourceIdKey must be provided when cleanup is incremental");
 		});
 		const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
 		const uids = [];
@@ -148,8 +145,7 @@ async function index(args) {
 		const docsToUpdate = [];
 		const seenDocs = /* @__PURE__ */ new Set();
 		hashedDocs.forEach((hashedDoc, i) => {
-			const docExists = batchExists[i];
-			if (docExists) if (forceUpdate) seenDocs.add(hashedDoc.uid);
+			if (batchExists[i]) if (forceUpdate) seenDocs.add(hashedDoc.uid);
 			else {
 				docsToUpdate.push(hashedDoc.uid);
 				return;

package/dist/indexing/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":["fields: HashedDocumentArgs","keyEncoderFn: HashKeyEncoder","document: DocumentInterface","uid?: string","inputString: string","uuidv5","data: Record<string, unknown>","size: number","iterable: T[]","batches: T[][]","currentBatch: T[]","hashedDocuments: HashedDocumentInterface[]","deduplicated: HashedDocumentInterface[]","sourceIdKey: StringOrDocFunc | null","_doc: DocumentInterface","doc: DocumentInterface","arg: any","args: IndexArgs","index","uids: string[]","docsToIndex: DocumentInterface[]","docsToUpdate: string[]"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"uuid\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,AAAQ,aAA6B;CAErC,YAAYA,QAA4B;EACtC,KAAK,MAAM,OAAO;EAClB,KAAK,cAAc,OAAO;EAC1B,KAAK,WAAW,OAAO;CACxB;CAED,sBAAsBC,cAAoC;EACxD,KAAK,aAAa;CACnB;CAED,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;EAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,CAAC,4BAA4B,EAAE,IAAI,uDAAuD,EAAE,cAAc,KACxG,KACD,CAAC,CAAC,CAAC;EAKV,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;GAC9D,KAAK,cAAc;GACnB,KAAK,eAAe;EACrB,SAAQ,GAAG;AACV,SAAM,IAAI,MACR,CAAC,yBAAyB,EAAE,EAAE,sDAAsD,CAAC;EAExF;EAED,KAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,KACR,KAAK,MAAM,KAAK;CAEnB;CAED,aAAgC;AAC9B,SAAO,IAAI,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;EAChB;CACF;CAED,OAAO,aACLC,UACAC,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;EAC/D;EACD,IAAI,iBAAiB;AACrB,SAAO;CACR;CAED,AAAQ,kBAAkBC,aAA6B;EACrD,MAAM,aAAa,KAAK,WAAW,YAAY;AAC/C,SAAOC,GAAO,YAAY,iBAAiB;CAC5C;CAED,AAAQ,sBAAsBC,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;EACtE,MAAM,aAAa,KAAK,WAAW,gBAAgB;AACnD,SAAOD,GAAO,YAAY,iBAAiB;CAC5C;AACF;AAyCD,SAAgB,OAAUE,MAAcC,UAAsB;CAC5D,MAAMC,UAAiB,CAAE;CACzB,IAAIC,eAAoB,CAAE;CAE1B,SAAS,QAAQ,CAAC,SAAS;EACzB,aAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;GAC/B,QAAQ,KAAK,aAAa;GAC1B,eAAe,CAAE;EAClB;CACF,EAAC;AAEF,KAAI,aAAa,SAAS,GACxB,QAAQ,KAAK,aAAa;AAG5B,QAAO;AACR;AAED,SAAgB,oBACdC,iBAC2B;CAC3B,MAAM,uBAAO,IAAI;CACjB,MAAMC,eAA0C,CAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM;AAGlB,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;GAC9B,KAAK,IAAI,UAAU,MAAM;GACzB,aAAa,KAAK,UAAU;EAC7B;CACF;AACD,QAAO;AACR;AAED,SAAgB,qBACdC,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,QAAO,CAACC,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,QAAO,CAACC,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,CAAC,wDAAwD,EAAE,OAAO,aAAa;AAGpF;AAGD,MAAa,wBAAwB,CAACC,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;AACR;;;;;;;;;;;;;;;;;;;;AA4BD,eAAsB,MAAMC,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,SAAS,GAAG;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,OACf,GAAG,WAAW,CAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR;CAIJ,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,IAAI,CAAC,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,IAAI,CAAC,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,eACd,WAAW,QAAQ,CAAC,YAAYC,YAAU;GACxC,MAAM,SAAS,UAAUA;AACzB,OAAI,WAAW,KACb,OAAM,IAAI,MACR;EAGL,EAAC;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,IAAI,CAAC,QAAQ,IAAI,IAAI,CACjC;EAED,MAAMC,OAAiB,CAAE;EACzB,MAAMC,cAAmC,CAAE;EAC3C,MAAMC,eAAyB,CAAE;EACjC,MAAM,2BAAW,IAAI;EACrB,WAAW,QAAQ,CAAC,WAAW,MAAM;GACnC,MAAM,YAAY,YAAY;AAC9B,OAAI,UACF,KAAI,aACF,SAAS,IAAI,UAAU,IAAI;QACtB;IACL,aAAa,KAAK,UAAU,IAAI;AAChC;GACD;GAEH,KAAK,KAAK,UAAU,IAAI;GACxB,YAAY,KAAK,UAAU,YAAY,CAAC;EACzC,EAAC;AAEF,MAAI,aAAa,SAAS,GAAG;GAC3B,MAAM,cAAc,OAAO,cAAc,EAAE,aAAa,aAAc,EAAC;GACvE,cAAc,aAAa;EAC5B;AAED,MAAI,YAAY,SAAS,GAAG;GAC1B,MAAM,YAAY,aAAa,aAAa,EAAE,KAAK,KAAM,EAAC;GAC1D,YAAY,YAAY,SAAS,SAAS;GAC1C,cAAc,SAAS;EACxB;EAED,MAAM,cAAc,OAClB,WAAW,IAAI,CAAC,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;EAAW,EACnD;AAED,MAAI,YAAY,eAAe;GAC7B,UAAU,QAAQ,CAAC,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM;GAChC,EAAC;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;GACX,EAAC;AAEF,OAAI,aAAa,SAAS,GAAG;IAC3B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAc,EAAC;IAC/C,MAAM,cAAc,WAAW,aAAa;IAC5C,cAAc,aAAa;GAC5B;EACF;CACF;AAED,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;EACR,EAAC;AACF,SAAO,aAAa,SAAS,GAAG;GAC9B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAc,EAAC;GAC/C,MAAM,cAAc,WAAW,aAAa;GAC5C,cAAc,aAAa;GAC3B,eAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;GACR,EAAC;EACH;CACF;AAED,QAAO;EACL;EACA;EACA;EACA;CACD;AACF"}
+{"version":3,"file":"base.js","names":["uuidv5"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"uuid\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,AAAQ,aAA6B;CAErC,YAAY,QAA4B;AACtC,OAAK,MAAM,OAAO;AAClB,OAAK,cAAc,OAAO;AAC1B,OAAK,WAAW,OAAO;;CAGzB,sBAAsB,cAAoC;AACxD,OAAK,aAAa;;CAGpB,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;GAAgB;AAEhE,OAAK,MAAM,OAAO,cAChB,KAAI,OAAO,KAAK,SACd,OAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,KACD,CAAC,GACH;EAIL,MAAM,cAAc,KAAK,kBAAkB,KAAK,YAAY;AAE5D,MAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,SAAS;AAC9D,QAAK,cAAc;AACnB,QAAK,eAAe;WACb,GAAG;AACV,SAAM,IAAI,MACR,4BAA4B,EAAE,wDAC/B;;AAGH,OAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,aAAa;AAEzE,MAAI,CAAC,KAAK,IACR,MAAK,MAAM,KAAK;;CAIpB,aAAgC;AAC9B,SAAO,IAAI,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;GAChB,CAAC;;CAGJ,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;GAC/D,CAAC;AACF,MAAI,iBAAiB;AACrB,SAAO;;CAGT,AAAQ,kBAAkB,aAA6B;AAErD,SAAOA,GADY,KAAK,WAAW,YAAY,EACrB,iBAAiB;;CAG7C,AAAQ,sBAAsB,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,KAAK,CAAC,MAAM,CAAC;AAEtE,SAAOA,GADY,KAAK,WAAW,gBAAgB,EACzB,iBAAiB;;;AA2C/C,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,EAAE;CACzB,IAAI,eAAoB,EAAE;AAE1B,UAAS,SAAS,SAAS;AACzB,eAAa,KAAK,KAAK;AAEvB,MAAI,aAAa,UAAU,MAAM;AAC/B,WAAQ,KAAK,aAAa;AAC1B,kBAAe,EAAE;;GAEnB;AAEF,KAAI,aAAa,SAAS,EACxB,SAAQ,KAAK,aAAa;AAG5B,QAAO;;AAGT,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,eAA0C,EAAE;AAElD,MAAK,MAAM,aAAa,iBAAiB;AACvC,MAAI,CAAC,UAAU,MACb,OAAM,IAAI,MAAM,uCAAuC;AAGzD,MAAI,CAAC,KAAK,IAAI,UAAU,MAAM,EAAE;AAC9B,QAAK,IAAI,UAAU,MAAM;AACzB,gBAAa,KAAK,UAAU;;;AAGhC,QAAO;;AAGT,SAAgB,qBACd,aAC2C;AAC3C,KAAI,gBAAgB,KAClB,SAAQ,SAA4B;UAC3B,OAAO,gBAAgB,SAChC,SAAQ,QAA2B,IAAI,SAAS;UACvC,OAAO,gBAAgB,WAChC,QAAO;KAEP,OAAM,IAAI,MACR,2DAA2D,OAAO,cACnE;;AAKL,MAAa,yBAAyB,QAAwC;AAC5E,KACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,WAE5B,QAAO;AAET,QAAO;;;;;;;;;;;;;;;;;;;;;AA6BT,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,EAAE;AAEjB,KAAI,YAAY,iBAAiB,CAAC,YAChC,OAAM,IAAI,MACR,0GACD;CAGH,MAAM,OAAO,sBAAsB,WAAW,GAC1C,MAAM,WAAW,MAAM,GACvB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,KAAK;CAElE,MAAM,eAAe,MAAM,cAAc,SAAS;CAClD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,KAAK;AAEjE,MAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,IAAI,CAAC,CACtD;EAED,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,IAAI,CAAC;AAEhE,MAAI,YAAY,cACd,YAAW,SAAS,YAAY,UAAU;AAExC,OADe,UAAU,WACV,KACb,OAAM,IAAI,MACR,2DACD;IAEH;EAGJ,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,IAAI,CACjC;EAED,MAAM,OAAiB,EAAE;EACzB,MAAM,cAAmC,EAAE;EAC3C,MAAM,eAAyB,EAAE;EACjC,MAAM,2BAAW,IAAI,KAAa;AAClC,aAAW,SAAS,WAAW,MAAM;AAEnC,OADkB,YAAY,GAE5B,KAAI,YACF,UAAS,IAAI,UAAU,IAAI;QACtB;AACL,iBAAa,KAAK,UAAU,IAAI;AAChC;;AAGJ,QAAK,KAAK,UAAU,IAAI;AACxB,eAAY,KAAK,UAAU,YAAY,CAAC;IACxC;AAEF,MAAI,aAAa,SAAS,GAAG;AAC3B,SAAM,cAAc,OAAO,cAAc,EAAE,aAAa,cAAc,CAAC;AACvE,iBAAc,aAAa;;AAG7B,MAAI,YAAY,SAAS,GAAG;AAC1B,SAAM,YAAY,aAAa,aAAa,EAAE,KAAK,MAAM,CAAC;AAC1D,eAAY,YAAY,SAAS,SAAS;AAC1C,iBAAc,SAAS;;AAGzB,QAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,IAAI,EAChC;GAAE,aAAa;GAAc,UAAU;GAAW,CACnD;AAED,MAAI,YAAY,eAAe;AAC7B,aAAU,SAAS,aAAa;AAC9B,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,2BAA2B;KAC1D;GACF,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;IACX,CAAC;AAEF,OAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,UAAM,cAAc,WAAW,aAAa;AAC5C,kBAAc,aAAa;;;;AAKjC,KAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;GACR,CAAC;AACF,SAAO,aAAa,SAAS,GAAG;AAC9B,SAAM,YAAY,OAAO,EAAE,KAAK,cAAc,CAAC;AAC/C,SAAM,cAAc,WAAW,aAAa;AAC5C,iBAAc,aAAa;AAC3B,kBAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;IACR,CAAC;;;AAIN,QAAO;EACL;EACA;EACA;EACA;EACD"}

package/dist/indexing/index.cjs

@@ -1,10 +1,10 @@
-const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
+const require_runtime = require('../_virtual/_rolldown/runtime.cjs');
 const require_record_manager = require('./record_manager.cjs');
 const require_base = require('./base.cjs');
 
 //#region src/indexing/index.ts
-var indexing_exports = {};
-require_rolldown_runtime.__export(indexing_exports, {
+var indexing_exports = /* @__PURE__ */ require_runtime.__exportAll({
 	RecordManager: () => require_record_manager.RecordManager,
 	UUIDV5_NAMESPACE: () => require_record_manager.UUIDV5_NAMESPACE,
 	_HashedDocument: () => require_base._HashedDocument,

package/dist/indexing/index.js

@@ -1,10 +1,9 @@
-import { __export } from "../_virtual/rolldown_runtime.js";
+import { __exportAll } from "../_virtual/_rolldown/runtime.js";
 import { RecordManager, UUIDV5_NAMESPACE } from "./record_manager.js";
 import { _HashedDocument, _batch, _deduplicateInOrder, _getSourceIdAssigner, _isBaseDocumentLoader, index } from "./base.js";
 
 //#region src/indexing/index.ts
-var indexing_exports = {};
-__export(indexing_exports, {
+var indexing_exports = /* @__PURE__ */ __exportAll({
 	RecordManager: () => RecordManager,
 	UUIDV5_NAMESPACE: () => UUIDV5_NAMESPACE,
 	_HashedDocument: () => _HashedDocument,

package/dist/indexing/record_manager.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"record_manager.cjs","names":["Serializable"],"sources":["../../src/indexing/record_manager.ts"],"sourcesContent":["import { Serializable } from \"../load/serializable.js\";\n\n// Arbitrary value, used for generating namespaced UUIDs.\nexport const UUIDV5_NAMESPACE = \"10f90ea3-90a4-4962-bf75-83a0f3c1c62a\";\n\nexport type UpdateOptions = {\n  groupIds?: (string | null)[];\n  timeAtLeast?: number;\n};\n\nexport type ListKeyOptions = {\n  before?: number;\n  after?: number;\n  groupIds?: (string | null)[];\n  limit?: number;\n};\n\nexport interface RecordManagerInterface {\n  /**\n   * Creates schema in the record manager.\n   * @returns Promise\n   */\n  createSchema(): Promise<void>;\n  /**\n   * Returns current time from the record manager.\n   * @returns Current time\n   */\n  getTime(): Promise<number>;\n  /**\n   * Updates keys in the record manager.\n   * @param keys List of keys to update\n   * @param groupIds List of groupIds to update\n   * @param timeAtLeast Update only if current time is at least this value\n   * @returns Promise\n   * @throws Error if timeAtLeast is provided and current time is less than timeAtLeast\n   * @throws Error if number of keys does not match number of groupIds\n   */\n  update(keys: string[], updateOptions: UpdateOptions): Promise<void>;\n  /**\n   * Checks if keys exist in the record manager.\n   * @param keys List of keys to check\n   * @returns List of booleans indicating if key exists in same order as provided keys\n   */\n  exists(keys: string[]): Promise<boolean[]>;\n  /**\n   * Lists keys from the record manager.\n   * @param before List keys before this timestamp\n   * @param after List keys after this timestamp\n   * @param groupIds List keys with these groupIds\n   * @param limit Limit the number of keys returned\n   * @returns List of keys\n   *\n   */\n  listKeys(options: ListKeyOptions): Promise<string[]>;\n  /**\n   * Deletes keys from the record manager.\n   * @param keys List of keys to delete\n   */\n  deleteKeys(keys: string[]): Promise<void>;\n}\n\nexport abstract class RecordManager\n  extends Serializable\n  implements RecordManagerInterface\n{\n  lc_namespace = [\"langchain\", \"recordmanagers\"];\n\n  abstract createSchema(): Promise<void>;\n\n  abstract getTime(): Promise<number>;\n\n  abstract update(keys: string[], updateOptions?: UpdateOptions): Promise<void>;\n\n  abstract exists(keys: string[]): Promise<boolean[]>;\n\n  abstract listKeys(options?: ListKeyOptions): Promise<string[]>;\n\n  abstract deleteKeys(keys: string[]): Promise<void>;\n}\n"],"mappings":";;;AAGA,MAAa,mBAAmB;AA0DhC,IAAsB,gBAAtB,cACUA,uCAEV;CACE,eAAe,CAAC,aAAa,gBAAiB;AAa/C"}
+{"version":3,"file":"record_manager.cjs","names":["Serializable"],"sources":["../../src/indexing/record_manager.ts"],"sourcesContent":["import { Serializable } from \"../load/serializable.js\";\n\n// Arbitrary value, used for generating namespaced UUIDs.\nexport const UUIDV5_NAMESPACE = \"10f90ea3-90a4-4962-bf75-83a0f3c1c62a\";\n\nexport type UpdateOptions = {\n  groupIds?: (string | null)[];\n  timeAtLeast?: number;\n};\n\nexport type ListKeyOptions = {\n  before?: number;\n  after?: number;\n  groupIds?: (string | null)[];\n  limit?: number;\n};\n\nexport interface RecordManagerInterface {\n  /**\n   * Creates schema in the record manager.\n   * @returns Promise\n   */\n  createSchema(): Promise<void>;\n  /**\n   * Returns current time from the record manager.\n   * @returns Current time\n   */\n  getTime(): Promise<number>;\n  /**\n   * Updates keys in the record manager.\n   * @param keys List of keys to update\n   * @param groupIds List of groupIds to update\n   * @param timeAtLeast Update only if current time is at least this value\n   * @returns Promise\n   * @throws Error if timeAtLeast is provided and current time is less than timeAtLeast\n   * @throws Error if number of keys does not match number of groupIds\n   */\n  update(keys: string[], updateOptions: UpdateOptions): Promise<void>;\n  /**\n   * Checks if keys exist in the record manager.\n   * @param keys List of keys to check\n   * @returns List of booleans indicating if key exists in same order as provided keys\n   */\n  exists(keys: string[]): Promise<boolean[]>;\n  /**\n   * Lists keys from the record manager.\n   * @param before List keys before this timestamp\n   * @param after List keys after this timestamp\n   * @param groupIds List keys with these groupIds\n   * @param limit Limit the number of keys returned\n   * @returns List of keys\n   *\n   */\n  listKeys(options: ListKeyOptions): Promise<string[]>;\n  /**\n   * Deletes keys from the record manager.\n   * @param keys List of keys to delete\n   */\n  deleteKeys(keys: string[]): Promise<void>;\n}\n\nexport abstract class RecordManager\n  extends Serializable\n  implements RecordManagerInterface\n{\n  lc_namespace = [\"langchain\", \"recordmanagers\"];\n\n  abstract createSchema(): Promise<void>;\n\n  abstract getTime(): Promise<number>;\n\n  abstract update(keys: string[], updateOptions?: UpdateOptions): Promise<void>;\n\n  abstract exists(keys: string[]): Promise<boolean[]>;\n\n  abstract listKeys(options?: ListKeyOptions): Promise<string[]>;\n\n  abstract deleteKeys(keys: string[]): Promise<void>;\n}\n"],"mappings":";;;AAGA,MAAa,mBAAmB;AA0DhC,IAAsB,gBAAtB,cACUA,uCAEV;CACE,eAAe,CAAC,aAAa,iBAAiB"}

package/dist/indexing/record_manager.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"record_manager.d.cts","names":["Serializable","UUIDV5_NAMESPACE","UpdateOptions","ListKeyOptions","RecordManagerInterface","Promise","RecordManager"],"sources":["../../src/indexing/record_manager.d.ts"],"sourcesContent":["import { Serializable } from \"../load/serializable.js\";\nexport declare const UUIDV5_NAMESPACE = \"10f90ea3-90a4-4962-bf75-83a0f3c1c62a\";\nexport type UpdateOptions = {\n    groupIds?: (string | null)[];\n    timeAtLeast?: number;\n};\nexport type ListKeyOptions = {\n    before?: number;\n    after?: number;\n    groupIds?: (string | null)[];\n    limit?: number;\n};\nexport interface RecordManagerInterface {\n    /**\n     * Creates schema in the record manager.\n     * @returns Promise\n     */\n    createSchema(): Promise<void>;\n    /**\n     * Returns current time from the record manager.\n     * @returns Current time\n     */\n    getTime(): Promise<number>;\n    /**\n     * Updates keys in the record manager.\n     * @param keys List of keys to update\n     * @param groupIds List of groupIds to update\n     * @param timeAtLeast Update only if current time is at least this value\n     * @returns Promise\n     * @throws Error if timeAtLeast is provided and current time is less than timeAtLeast\n     * @throws Error if number of keys does not match number of groupIds\n     */\n    update(keys: string[], updateOptions: UpdateOptions): Promise<void>;\n    /**\n     * Checks if keys exist in the record manager.\n     * @param keys List of keys to check\n     * @returns List of booleans indicating if key exists in same order as provided keys\n     */\n    exists(keys: string[]): Promise<boolean[]>;\n    /**\n     * Lists keys from the record manager.\n     * @param before List keys before this timestamp\n     * @param after List keys after this timestamp\n     * @param groupIds List keys with these groupIds\n     * @param limit Limit the number of keys returned\n     * @returns List of keys\n     *\n     */\n    listKeys(options: ListKeyOptions): Promise<string[]>;\n    /**\n     * Deletes keys from the record manager.\n     * @param keys List of keys to delete\n     */\n    deleteKeys(keys: string[]): Promise<void>;\n}\nexport declare abstract class RecordManager extends Serializable implements RecordManagerInterface {\n    lc_namespace: string[];\n    abstract createSchema(): Promise<void>;\n    abstract getTime(): Promise<number>;\n    abstract update(keys: string[], updateOptions?: UpdateOptions): Promise<void>;\n    abstract exists(keys: string[]): Promise<boolean[]>;\n    abstract listKeys(options?: ListKeyOptions): Promise<string[]>;\n    abstract deleteKeys(keys: string[]): Promise<void>;\n}\n//# sourceMappingURL=record_manager.d.ts.map"],"mappings":";;;cACqBC,gBAAAA;KACTC,aAAAA;EADSD,QAAAA,CAAAA,EAAAA,CAAAA,MAAAA,GAAgB,IAAA,CAAA,EAAA;EACzBC,WAAAA,CAAAA,EAAAA,MAAa;AAIzB,CAAA;AAMiBE,KANLD,cAAAA,GAM2B;EAKnBE,MAAAA,CAAAA,EAAAA,MAAAA;EAKLA,KAAAA,CAAAA,EAAAA,MAAAA;EAU2BH,QAAAA,CAAAA,EAAAA,CAAAA,MAAAA,GAAAA,IAAAA,CAAAA,EAAAA;EAAgBG,KAAAA,CAAAA,EAAAA,MAAAA;CAM9BA;AAUNF,UApCLC,sBAAAA,CAoCKD;EAAiBE;;AAKA;AAEvC;EAE6BA,YAAAA,EAAAA,EAxCTA,OAwCSA,CAAAA,IAAAA,CAAAA;EACLA;;;;EAGQF,OAAAA,EAAAA,EAvCjBE,OAuCiBF,CAAAA,MAAAA,CAAAA;EAAiBE;;;;AANiD;;;;;wCAvBxDH,gBAAgBG;;;;;;0BAM9BA;;;;;;;;;;oBAUNF,iBAAiBE;;;;;8BAKPA;;uBAEFC,aAAAA,SAAsBN,YAAAA,YAAwBI;;2BAE/CC;sBACLA;kDAC4BH,gBAAgBG;mCAC/BA;8BACLF,iBAAiBE;uCACRA"}
+{"version":3,"file":"record_manager.d.cts","names":[],"sources":["../../src/indexing/record_manager.ts"],"mappings":";;;cAGa,gBAAA;AAAA,KAED,aAAA;EACV,QAAA;EACA,WAAA;AAAA;AAAA,KAGU,cAAA;EACV,MAAA;EACA,KAAA;EACA,QAAA;EACA,KAAA;AAAA;AAAA,UAGe,sBAAA;EAPL;;;;EAYV,YAAA,IAAgB,OAAA;EAVhB;;;;EAeA,OAAA,IAAW,OAAA;EAVI;;;;;;;;;EAoBf,MAAA,CAAO,IAAA,YAAgB,aAAA,EAAe,aAAA,GAAgB,OAAA;EAqB1B;;;;;EAf5B,MAAA,CAAO,IAAA,aAAiB,OAAA;EAhBb;;;;;;;;;EA0BX,QAAA,CAAS,OAAA,EAAS,cAAA,GAAiB,OAAA;EAAjB;;;;EAKlB,UAAA,CAAW,IAAA,aAAiB,OAAA;AAAA;AAAA,uBAGR,aAAA,SACZ,YAAA,YACG,sBAAA;EAEX,YAAA;EAAA,SAES,YAAA,CAAA,GAAgB,OAAA;EAAA,SAEhB,OAAA,CAAA,GAAW,OAAA;EAAA,SAEX,MAAA,CAAO,IAAA,YAAgB,aAAA,GAAgB,aAAA,GAAgB,OAAA;EAAA,SAEvD,MAAA,CAAO,IAAA,aAAiB,OAAA;EAAA,SAExB,QAAA,CAAS,OAAA,GAAU,cAAA,GAAiB,OAAA;EAAA,SAEpC,UAAA,CAAW,IAAA,aAAiB,OAAA;AAAA"}