@langchain/core 0.3.72 → 1.0.0-alpha.2

package/dist/indexing/base.d.cts

@@ -0,0 +1,125 @@
+import { HashKeyEncoder } from "../utils/hash.cjs";
+import { DocumentInterface } from "../documents/document.cjs";
+import { BaseDocumentLoader } from "../document_loaders/base.cjs";
+import { VectorStore } from "../vectorstores.cjs";
+import { RecordManagerInterface } from "./record_manager.cjs";
+
+//#region src/indexing/base.d.ts
+type Metadata = Record<string, unknown>;
+type IndexingResult = {
+  numAdded: number;
+  numDeleted: number;
+  numUpdated: number;
+  numSkipped: number;
+};
+type StringOrDocFunc = string | ((doc: DocumentInterface) => string);
+interface HashedDocumentInterface extends DocumentInterface {
+  uid: string;
+  hash_?: string;
+  contentHash?: string;
+  metadataHash?: string;
+  pageContent: string;
+  metadata: Metadata;
+  calculateHashes(): void;
+  toDocument(): DocumentInterface;
+}
+interface HashedDocumentArgs {
+  pageContent: string;
+  metadata: Metadata;
+  uid: string;
+}
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+declare class _HashedDocument implements HashedDocumentInterface {
+  uid: string;
+  hash_?: string;
+  contentHash?: string;
+  metadataHash?: string;
+  pageContent: string;
+  metadata: Metadata;
+  // For backwards compatibility, we use a default key encoder
+  // that uses SHA-1 to hash the prompt and LLM key. This will also print a warning
+  // about the security implications of using SHA-1 as a key encoder.
+  private keyEncoder;
+  constructor(fields: HashedDocumentArgs);
+  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;
+  calculateHashes(): void;
+  toDocument(): DocumentInterface;
+  static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;
+  private _hashStringToUUID;
+  private _hashNestedDictToUUID;
+}
+type CleanupMode = "full" | "incremental";
+type IndexOptions = {
+  /**
+   * The number of documents to index in one batch.
+   */
+  batchSize?: number;
+  /**
+   * The cleanup mode to use. Can be "full", "incremental" or undefined.
+   * - **Incremental**: Cleans up all documents that haven't been updated AND
+   *   that are associated with source ids that were seen
+   *   during indexing.
+   *   Clean up is done continuously during indexing helping
+   *   to minimize the probability of users seeing duplicated
+   *   content.
+   * - **Full**: Delete all documents that haven to been returned by the loader.
+   *   Clean up runs after all documents have been indexed.
+   *   This means that users may see duplicated content during indexing.
+   * - **undefined**: Do not delete any documents.
+   */
+  cleanup?: CleanupMode;
+  /**
+   * Optional key that helps identify the original source of the document.
+   * Must either be a string representing the key of the source in the metadata
+   * or a function that takes a document and returns a string representing the source.
+   * **Required when cleanup is incremental**.
+   */
+  sourceIdKey?: StringOrDocFunc;
+  /**
+   * Batch size to use when cleaning up documents.
+   */
+  cleanupBatchSize?: number;
+  /**
+   * Force update documents even if they are present in the
+   * record manager. Useful if you are re-indexing with updated embeddings.
+   */
+  forceUpdate?: boolean;
+};
+declare function _batch<T>(size: number, iterable: T[]): T[][];
+declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];
+declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;
+interface IndexArgs {
+  docsSource: BaseDocumentLoader | DocumentInterface[];
+  recordManager: RecordManagerInterface;
+  vectorStore: VectorStore;
+  options?: IndexOptions;
+}
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+declare function index(args: IndexArgs): Promise<IndexingResult>;
+//#endregion
+export { CleanupMode, HashedDocumentInterface, IndexOptions, _HashedDocument, _batch, _deduplicateInOrder, _getSourceIdAssigner, _isBaseDocumentLoader, index };
+//# sourceMappingURL=base.d.cts.map

package/dist/indexing/base.d.cts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.cts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    // For backwards compatibility, we use a default key encoder\n    // that uses SHA-1 to hash the prompt and LLM key. This will also print a warning\n    // about the security implications of using SHA-1 as a key encoder.\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAAuB;KADnCD,eAAAA,GAOSH,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EAPyBF,iBAOzBE,EAAAA,GAAAA,MAAAA,CAAAA;AAEIF,UARDM,uBAAAA,SAAgCN,iBAQ/BA,CAAAA;EAAiB,GARcA,EAAAA,MAAAA;EAAiB,KAAA,CAAA,EAAA,MAAA;EAUxDO,WAAAA,CAAAA,EAAAA,MAAAA;EAUWC,YAAAA,CAAAA,EAAAA,MAAe;EAAA,WAAA,EAAA,MAAA;EAAA,QAMtBN,EApBAA,QAoBAA;EAAQ,eAKEK,EAAAA,EAAAA,IAAAA;EAAkB,UACFR,EAAAA,EAxBtBC,iBAwBsBD;;UAtB9BQ,kBAAAA,CAyBwBP;EAAiB,WAAiBQ,EAAAA,MAAAA;EAAe,QAfnCF,EARlCJ,QAQkCI;EAAuB,GAAA,EAAA,MAAA;AAmBvE;AACA;;;;AAyBiC;AAWTK,cAxDHH,eAAAA,YAA2BF,uBAwDlB,CAAA;EAAA,GAAA,EAAA,MAAA;EAAA,KAA4BM,CAAAA,EAAAA,MAAAA;EAAC,WAAKA,CAAAA,EAAAA,MAAAA;EAAC,YAAA,CAAA,EAAA,MAAA;EACzCC,WAAAA,EAAAA,MAAAA;EAAmB,QAAA,EAnD7BX,QAmD6B;EAAA;EAAyC;EAA4B;EACxFY,QAAAA,UAAAA;EAAoB,WAAA,CAAA,MAAA,EA/CpBP,kBA+CoB;EAAA,qBAAcF,CAAAA,YAAAA,EA9ClBN,cA8CkBM,CAAAA,EAAAA,IAAAA;EAAe,eAAgBL,CAAAA,CAAAA,EAAAA,IAAAA;EAAiB,UAAA,CAAA,CAAA,EA5CxFA,iBA4CwF;EAErFe,OAAAA,YAAAA,CAAAA,QAA8D,EA7CjDf,iBA6C+BC,EAAAA,GAAkB,CAAlBA,EAAkB,MAAA,CAAA,EA7CfO,eA6Ce;EACzEQ,QAAAA,iBAAS;EAAA,QAAA,qBAAA;;AACkBhB,KA3CzBS,WAAAA,GA2CyBT,MAAAA,GAAAA,aAAAA;AAClBF,KA3CPY,YAAAA,GA2COZ;EAAsB;;AAEf;EAqBFmB,SAAK,CAAA,EAAA,MAAA;EAAA;;;;AAA0B;;;;;;;;;YAhDzCR;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;;cAEpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}

package/dist/indexing/base.d.ts

@@ -1,98 +1,104 @@
-import { VectorStore } from "../vectorstores.js";
-import { RecordManagerInterface } from "./record_manager.js";
-import { type HashKeyEncoder } from "../utils/hash.js";
+import { HashKeyEncoder } from "../utils/hash.js";
 import { DocumentInterface } from "../documents/document.js";
 import { BaseDocumentLoader } from "../document_loaders/base.js";
+import { VectorStore } from "../vectorstores.js";
+import { RecordManagerInterface } from "./record_manager.js";
+
+//#region src/indexing/base.d.ts
 type Metadata = Record<string, unknown>;
 type IndexingResult = {
-    numAdded: number;
-    numDeleted: number;
-    numUpdated: number;
-    numSkipped: number;
+  numAdded: number;
+  numDeleted: number;
+  numUpdated: number;
+  numSkipped: number;
 };
 type StringOrDocFunc = string | ((doc: DocumentInterface) => string);
-export interface HashedDocumentInterface extends DocumentInterface {
-    uid: string;
-    hash_?: string;
-    contentHash?: string;
-    metadataHash?: string;
-    pageContent: string;
-    metadata: Metadata;
-    calculateHashes(): void;
-    toDocument(): DocumentInterface;
+interface HashedDocumentInterface extends DocumentInterface {
+  uid: string;
+  hash_?: string;
+  contentHash?: string;
+  metadataHash?: string;
+  pageContent: string;
+  metadata: Metadata;
+  calculateHashes(): void;
+  toDocument(): DocumentInterface;
 }
 interface HashedDocumentArgs {
-    pageContent: string;
-    metadata: Metadata;
-    uid: string;
+  pageContent: string;
+  metadata: Metadata;
+  uid: string;
 }
 /**
  * HashedDocument is a Document with hashes calculated.
  * Hashes are calculated based on page content and metadata.
  * It is used for indexing.
  */
-export declare class _HashedDocument implements HashedDocumentInterface {
-    uid: string;
-    hash_?: string;
-    contentHash?: string;
-    metadataHash?: string;
-    pageContent: string;
-    metadata: Metadata;
-    private keyEncoder;
-    constructor(fields: HashedDocumentArgs);
-    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;
-    calculateHashes(): void;
-    toDocument(): DocumentInterface;
-    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;
-    private _hashStringToUUID;
-    private _hashNestedDictToUUID;
+declare class _HashedDocument implements HashedDocumentInterface {
+  uid: string;
+  hash_?: string;
+  contentHash?: string;
+  metadataHash?: string;
+  pageContent: string;
+  metadata: Metadata;
+  // For backwards compatibility, we use a default key encoder
+  // that uses SHA-1 to hash the prompt and LLM key. This will also print a warning
+  // about the security implications of using SHA-1 as a key encoder.
+  private keyEncoder;
+  constructor(fields: HashedDocumentArgs);
+  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;
+  calculateHashes(): void;
+  toDocument(): DocumentInterface;
+  static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;
+  private _hashStringToUUID;
+  private _hashNestedDictToUUID;
 }
-export type CleanupMode = "full" | "incremental";
-export type IndexOptions = {
-    /**
-     * The number of documents to index in one batch.
-     */
-    batchSize?: number;
-    /**
-     * The cleanup mode to use. Can be "full", "incremental" or undefined.
-     * - **Incremental**: Cleans up all documents that haven't been updated AND
-     *   that are associated with source ids that were seen
-     *   during indexing.
-     *   Clean up is done continuously during indexing helping
-     *   to minimize the probability of users seeing duplicated
-     *   content.
-     * - **Full**: Delete all documents that haven to been returned by the loader.
-     *   Clean up runs after all documents have been indexed.
-     *   This means that users may see duplicated content during indexing.
-     * - **undefined**: Do not delete any documents.
-     */
-    cleanup?: CleanupMode;
-    /**
-     * Optional key that helps identify the original source of the document.
-     * Must either be a string representing the key of the source in the metadata
-     * or a function that takes a document and returns a string representing the source.
-     * **Required when cleanup is incremental**.
-     */
-    sourceIdKey?: StringOrDocFunc;
-    /**
-     * Batch size to use when cleaning up documents.
-     */
-    cleanupBatchSize?: number;
-    /**
-     * Force update documents even if they are present in the
-     * record manager. Useful if you are re-indexing with updated embeddings.
-     */
-    forceUpdate?: boolean;
+type CleanupMode = "full" | "incremental";
+type IndexOptions = {
+  /**
+   * The number of documents to index in one batch.
+   */
+  batchSize?: number;
+  /**
+   * The cleanup mode to use. Can be "full", "incremental" or undefined.
+   * - **Incremental**: Cleans up all documents that haven't been updated AND
+   *   that are associated with source ids that were seen
+   *   during indexing.
+   *   Clean up is done continuously during indexing helping
+   *   to minimize the probability of users seeing duplicated
+   *   content.
+   * - **Full**: Delete all documents that haven to been returned by the loader.
+   *   Clean up runs after all documents have been indexed.
+   *   This means that users may see duplicated content during indexing.
+   * - **undefined**: Do not delete any documents.
+   */
+  cleanup?: CleanupMode;
+  /**
+   * Optional key that helps identify the original source of the document.
+   * Must either be a string representing the key of the source in the metadata
+   * or a function that takes a document and returns a string representing the source.
+   * **Required when cleanup is incremental**.
+   */
+  sourceIdKey?: StringOrDocFunc;
+  /**
+   * Batch size to use when cleaning up documents.
+   */
+  cleanupBatchSize?: number;
+  /**
+   * Force update documents even if they are present in the
+   * record manager. Useful if you are re-indexing with updated embeddings.
+   */
+  forceUpdate?: boolean;
 };
-export declare function _batch<T>(size: number, iterable: T[]): T[][];
-export declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];
-export declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;
-export declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;
+declare function _batch<T>(size: number, iterable: T[]): T[][];
+declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];
+declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;
 interface IndexArgs {
-    docsSource: BaseDocumentLoader | DocumentInterface[];
-    recordManager: RecordManagerInterface;
-    vectorStore: VectorStore;
-    options?: IndexOptions;
+  docsSource: BaseDocumentLoader | DocumentInterface[];
+  recordManager: RecordManagerInterface;
+  vectorStore: VectorStore;
+  options?: IndexOptions;
 }
 /**
  * Index data from the doc source into the vector store.
@@ -113,5 +119,7 @@ interface IndexArgs {
  * @param {IndexOptions | undefined} args.options Options for indexing.
  * @returns {Promise<IndexingResult>}
  */
-export declare function index(args: IndexArgs): Promise<IndexingResult>;
-export {};
+declare function index(args: IndexArgs): Promise<IndexingResult>;
+//#endregion
+export { CleanupMode, HashedDocumentInterface, IndexOptions, _HashedDocument, _batch, _deduplicateInOrder, _getSourceIdAssigner, _isBaseDocumentLoader, index };
+//# sourceMappingURL=base.d.ts.map

package/dist/indexing/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","names":["VectorStore","RecordManagerInterface","HashKeyEncoder","DocumentInterface","BaseDocumentLoader","Metadata","Record","IndexingResult","StringOrDocFunc","HashedDocumentInterface","HashedDocumentArgs","_HashedDocument","CleanupMode","IndexOptions","_batch","T","_deduplicateInOrder","_getSourceIdAssigner","_isBaseDocumentLoader","IndexArgs","index","Promise"],"sources":["../../src/indexing/base.d.ts"],"sourcesContent":["import { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface } from \"./record_manager.js\";\nimport { type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\ntype Metadata = Record<string, unknown>;\ntype IndexingResult = {\n    numAdded: number;\n    numDeleted: number;\n    numUpdated: number;\n    numSkipped: number;\n};\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\nexport interface HashedDocumentInterface extends DocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n}\ninterface HashedDocumentArgs {\n    pageContent: string;\n    metadata: Metadata;\n    uid: string;\n}\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport declare class _HashedDocument implements HashedDocumentInterface {\n    uid: string;\n    hash_?: string;\n    contentHash?: string;\n    metadataHash?: string;\n    pageContent: string;\n    metadata: Metadata;\n    // For backwards compatibility, we use a default key encoder\n    // that uses SHA-1 to hash the prompt and LLM key. This will also print a warning\n    // about the security implications of using SHA-1 as a key encoder.\n    private keyEncoder;\n    constructor(fields: HashedDocumentArgs);\n    makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void;\n    calculateHashes(): void;\n    toDocument(): DocumentInterface;\n    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;\n    private _hashStringToUUID;\n    private _hashNestedDictToUUID;\n}\nexport type CleanupMode = \"full\" | \"incremental\";\nexport type IndexOptions = {\n    /**\n     * The number of documents to index in one batch.\n     */\n    batchSize?: number;\n    /**\n     * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n     * - **Incremental**: Cleans up all documents that haven't been updated AND\n     *   that are associated with source ids that were seen\n     *   during indexing.\n     *   Clean up is done continuously during indexing helping\n     *   to minimize the probability of users seeing duplicated\n     *   content.\n     * - **Full**: Delete all documents that haven to been returned by the loader.\n     *   Clean up runs after all documents have been indexed.\n     *   This means that users may see duplicated content during indexing.\n     * - **undefined**: Do not delete any documents.\n     */\n    cleanup?: CleanupMode;\n    /**\n     * Optional key that helps identify the original source of the document.\n     * Must either be a string representing the key of the source in the metadata\n     * or a function that takes a document and returns a string representing the source.\n     * **Required when cleanup is incremental**.\n     */\n    sourceIdKey?: StringOrDocFunc;\n    /**\n     * Batch size to use when cleaning up documents.\n     */\n    cleanupBatchSize?: number;\n    /**\n     * Force update documents even if they are present in the\n     * record manager. Useful if you are re-indexing with updated embeddings.\n     */\n    forceUpdate?: boolean;\n};\nexport declare function _batch<T>(size: number, iterable: T[]): T[][];\nexport declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];\nexport declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;\ninterface IndexArgs {\n    docsSource: BaseDocumentLoader | DocumentInterface[];\n    recordManager: RecordManagerInterface;\n    vectorStore: VectorStore;\n    options?: IndexOptions;\n}\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport declare function index(args: IndexArgs): Promise<IndexingResult>;\nexport {};\n"],"mappings":";;;;;;;KAKKK,QAAAA,GAAWC;KACXC,cAAAA;EADAF,QAAAA,EAAAA,MAAQ;EACRE,UAAAA,EAAAA,MAAc;EAMdC,UAAAA,EAAAA,MAAe;EACHC,UAAAA,EAAAA,MAAAA;CAAuB;KADnCD,eAAAA,GAOSH,MAAAA,GAAAA,CAAAA,CAAAA,GAAAA,EAPyBF,iBAOzBE,EAAAA,GAAAA,MAAAA,CAAAA;AAEIF,UARDM,uBAAAA,SAAgCN,iBAQ/BA,CAAAA;EAAiB,GARcA,EAAAA,MAAAA;EAAiB,KAAA,CAAA,EAAA,MAAA;EAUxDO,WAAAA,CAAAA,EAAAA,MAAAA;EAUWC,YAAAA,CAAAA,EAAAA,MAAe;EAAA,WAAA,EAAA,MAAA;EAAA,QAMtBN,EApBAA,QAoBAA;EAAQ,eAKEK,EAAAA,EAAAA,IAAAA;EAAkB,UACFR,EAAAA,EAxBtBC,iBAwBsBD;;UAtB9BQ,kBAAAA,CAyBwBP;EAAiB,WAAiBQ,EAAAA,MAAAA;EAAe,QAfnCF,EARlCJ,QAQkCI;EAAuB,GAAA,EAAA,MAAA;AAmBvE;AACA;;;;AAyBiC;AAWTK,cAxDHH,eAAAA,YAA2BF,uBAwDlB,CAAA;EAAA,GAAA,EAAA,MAAA;EAAA,KAA4BM,CAAAA,EAAAA,MAAAA;EAAC,WAAKA,CAAAA,EAAAA,MAAAA;EAAC,YAAA,CAAA,EAAA,MAAA;EACzCC,WAAAA,EAAAA,MAAAA;EAAmB,QAAA,EAnD7BX,QAmD6B;EAAA;EAAyC;EAA4B;EACxFY,QAAAA,UAAAA;EAAoB,WAAA,CAAA,MAAA,EA/CpBP,kBA+CoB;EAAA,qBAAcF,CAAAA,YAAAA,EA9ClBN,cA8CkBM,CAAAA,EAAAA,IAAAA;EAAe,eAAgBL,CAAAA,CAAAA,EAAAA,IAAAA;EAAiB,UAAA,CAAA,CAAA,EA5CxFA,iBA4CwF;EAErFe,OAAAA,YAAAA,CAAAA,QAA8D,EA7CjDf,iBA6C+BC,EAAAA,GAAkB,CAAlBA,EAAkB,MAAA,CAAA,EA7CfO,eA6Ce;EACzEQ,QAAAA,iBAAS;EAAA,QAAA,qBAAA;;AACkBhB,KA3CzBS,WAAAA,GA2CyBT,MAAAA,GAAAA,aAAAA;AAClBF,KA3CPY,YAAAA,GA2COZ;EAAsB;;AAEf;EAqBFmB,SAAK,CAAA,EAAA,MAAA;EAAA;;;;AAA0B;;;;;;;;;YAhDzCR;;;;;;;gBAOIJ;;;;;;;;;;;iBAWMM,kCAAkCC,MAAMA;iBACxCC,mBAAAA,kBAAqCP,4BAA4BA;iBACjEQ,oBAAAA,cAAkCT,+BAA+BL;;cAEpEe,4CAA4Cd;UACvDe,SAAAA;cACMf,qBAAqBD;iBAClBF;eACFD;YACHa;;;;;;;;;;;;;;;;;;;;;iBAqBUO,KAAAA,OAAYD,YAAYE,QAAQd"}