@langchain/core 0.1.61 → 0.2.0-rc.0

package/dist/document_loaders/base.cjs

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseDocumentLoader = void 0;
+/**
+ * Abstract class that provides a default implementation for the
+ * loadAndSplit() method from the DocumentLoader interface. The load()
+ * method is left abstract and needs to be implemented by subclasses.
+ */
+class BaseDocumentLoader {
+    /**
+     * @deprecated Use `this.load()` and `splitter.splitDocuments()` individually.
+     * Loads the documents and splits them using a specified text splitter.
+     * @param textSplitter The TextSplitter instance to use for splitting the loaded documents. Defaults to a RecursiveCharacterTextSplitter instance.
+     * @returns A Promise that resolves with an array of Document instances, each split according to the provided TextSplitter.
+     */
+    async loadAndSplit(splitter) {
+        if (splitter === undefined) {
+            throw new Error("You must pass a text splitter to use this method.");
+        }
+        const docs = await this.load();
+        return splitter.invoke(docs);
+    }
+}
+exports.BaseDocumentLoader = BaseDocumentLoader;

package/dist/document_loaders/base.d.ts

@@ -0,0 +1,28 @@
+import { Document } from "../documents/document.js";
+import { BaseDocumentTransformer } from "../documents/transformers.js";
+/**
+ * Interface that defines the methods for loading and splitting documents.
+ */
+export interface DocumentLoader {
+    load(): Promise<Document[]>;
+    loadAndSplit(textSplitter?: BaseDocumentTransformer): Promise<Document[]>;
+}
+/**
+ * Abstract class that provides a default implementation for the
+ * loadAndSplit() method from the DocumentLoader interface. The load()
+ * method is left abstract and needs to be implemented by subclasses.
+ */
+export declare abstract class BaseDocumentLoader implements DocumentLoader {
+    /**
+     * Loads the documents.
+     * @returns A Promise that resolves with an array of Document instances.
+     */
+    abstract load(): Promise<Document[]>;
+    /**
+     * @deprecated Use `this.load()` and `splitter.splitDocuments()` individually.
+     * Loads the documents and splits them using a specified text splitter.
+     * @param textSplitter The TextSplitter instance to use for splitting the loaded documents. Defaults to a RecursiveCharacterTextSplitter instance.
+     * @returns A Promise that resolves with an array of Document instances, each split according to the provided TextSplitter.
+     */
+    loadAndSplit(splitter?: BaseDocumentTransformer): Promise<Document[]>;
+}

package/dist/document_loaders/base.js

@@ -0,0 +1,20 @@
+/**
+ * Abstract class that provides a default implementation for the
+ * loadAndSplit() method from the DocumentLoader interface. The load()
+ * method is left abstract and needs to be implemented by subclasses.
+ */
+export class BaseDocumentLoader {
+    /**
+     * @deprecated Use `this.load()` and `splitter.splitDocuments()` individually.
+     * Loads the documents and splits them using a specified text splitter.
+     * @param textSplitter The TextSplitter instance to use for splitting the loaded documents. Defaults to a RecursiveCharacterTextSplitter instance.
+     * @returns A Promise that resolves with an array of Document instances, each split according to the provided TextSplitter.
+     */
+    async loadAndSplit(splitter) {
+        if (splitter === undefined) {
+            throw new Error("You must pass a text splitter to use this method.");
+        }
+        const docs = await this.load();
+        return splitter.invoke(docs);
+    }
+}

package/dist/documents/transformers.d.ts

@@ -26,7 +26,7 @@ export declare abstract class BaseDocumentTransformer<RunInput extends DocumentI
      * @param _options Optional configuration object to customize the behavior of callbacks.
      * @returns A Promise that resolves to the transformed documents.
      */
-    invoke(input: RunInput, _options: BaseCallbackConfig): Promise<RunOutput>;
+    invoke(input: RunInput, _options?: BaseCallbackConfig): Promise<RunOutput>;
 }
 /**
  * Class for document transformers that return exactly one transformed document

package/dist/indexing/base.cjs

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.index = exports._isBaseDocumentLoader = exports._getSourceIdAssigner = exports._deduplicateInOrder = exports._batch = exports._HashedDocument = void 0;
+const uuid_1 = require("uuid");
+const record_manager_js_1 = require("./record_manager.cjs");
+const hash_js_1 = require("../utils/hash.cjs");
+const document_js_1 = require("../documents/document.cjs");
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+class _HashedDocument {
+    constructor(fields) {
+        Object.defineProperty(this, "uid", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "hash_", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contentHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.uid = fields.uid;
+        this.pageContent = fields.pageContent;
+        this.metadata = fields.metadata;
+    }
+    calculateHashes() {
+        const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
+        for (const key of forbiddenKeys) {
+            if (key in this.metadata) {
+                throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
+            }
+        }
+        const contentHash = this._hashStringToUUID(this.pageContent);
+        try {
+            const metadataHash = this._hashNestedDictToUUID(this.metadata);
+            this.contentHash = contentHash;
+            this.metadataHash = metadataHash;
+        }
+        catch (e) {
+            throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
+        }
+        this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
+        if (!this.uid) {
+            this.uid = this.hash_;
+        }
+    }
+    toDocument() {
+        return new document_js_1.Document({
+            pageContent: this.pageContent,
+            metadata: this.metadata,
+        });
+    }
+    static fromDocument(document, uid) {
+        const doc = new this({
+            pageContent: document.pageContent,
+            metadata: document.metadata,
+            uid: uid || document.uid,
+        });
+        doc.calculateHashes();
+        return doc;
+    }
+    _hashStringToUUID(inputString) {
+        const hash_value = (0, hash_js_1.insecureHash)(inputString);
+        return (0, uuid_1.v5)(hash_value, record_manager_js_1.UUIDV5_NAMESPACE);
+    }
+    _hashNestedDictToUUID(data) {
+        const serialized_data = JSON.stringify(data, Object.keys(data).sort());
+        const hash_value = (0, hash_js_1.insecureHash)(serialized_data);
+        return (0, uuid_1.v5)(hash_value, record_manager_js_1.UUIDV5_NAMESPACE);
+    }
+}
+exports._HashedDocument = _HashedDocument;
+function _batch(size, iterable) {
+    const batches = [];
+    let currentBatch = [];
+    iterable.forEach((item) => {
+        currentBatch.push(item);
+        if (currentBatch.length >= size) {
+            batches.push(currentBatch);
+            currentBatch = [];
+        }
+    });
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+}
+exports._batch = _batch;
+function _deduplicateInOrder(hashedDocuments) {
+    const seen = new Set();
+    const deduplicated = [];
+    for (const hashedDoc of hashedDocuments) {
+        if (!hashedDoc.hash_) {
+            throw new Error("Hashed document does not have a hash");
+        }
+        if (!seen.has(hashedDoc.hash_)) {
+            seen.add(hashedDoc.hash_);
+            deduplicated.push(hashedDoc);
+        }
+    }
+    return deduplicated;
+}
+exports._deduplicateInOrder = _deduplicateInOrder;
+function _getSourceIdAssigner(sourceIdKey) {
+    if (sourceIdKey === null) {
+        return (_doc) => null;
+    }
+    else if (typeof sourceIdKey === "string") {
+        return (doc) => doc.metadata[sourceIdKey];
+    }
+    else if (typeof sourceIdKey === "function") {
+        return sourceIdKey;
+    }
+    else {
+        throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
+    }
+}
+exports._getSourceIdAssigner = _getSourceIdAssigner;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const _isBaseDocumentLoader = (arg) => {
+    if ("load" in arg &&
+        typeof arg.load === "function" &&
+        "loadAndSplit" in arg &&
+        typeof arg.loadAndSplit === "function") {
+        return true;
+    }
+    return false;
+};
+exports._isBaseDocumentLoader = _isBaseDocumentLoader;
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+async function index(args) {
+    const { docsSource, recordManager, vectorStore, options } = args;
+    const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
+    if (cleanup === "incremental" && !sourceIdKey) {
+        throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
+    }
+    const docs = (0, exports._isBaseDocumentLoader)(docsSource)
+        ? await docsSource.load()
+        : docsSource;
+    const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
+    const indexStartDt = await recordManager.getTime();
+    let numAdded = 0;
+    let numDeleted = 0;
+    let numUpdated = 0;
+    let numSkipped = 0;
+    const batches = _batch(batchSize ?? 100, docs);
+    for (const batch of batches) {
+        const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
+        const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
+        if (cleanup === "incremental") {
+            hashedDocs.forEach((_hashedDoc, index) => {
+                const source = sourceIds[index];
+                if (source === null) {
+                    throw new Error("sourceIdKey must be provided when cleanup is incremental");
+                }
+            });
+        }
+        const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
+        const uids = [];
+        const docsToIndex = [];
+        const docsToUpdate = [];
+        const seenDocs = new Set();
+        hashedDocs.forEach((hashedDoc, i) => {
+            const docExists = batchExists[i];
+            if (docExists) {
+                if (forceUpdate) {
+                    seenDocs.add(hashedDoc.uid);
+                }
+                else {
+                    docsToUpdate.push(hashedDoc.uid);
+                    return;
+                }
+            }
+            uids.push(hashedDoc.uid);
+            docsToIndex.push(hashedDoc.toDocument());
+        });
+        if (docsToUpdate.length > 0) {
+            await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
+            numSkipped += docsToUpdate.length;
+        }
+        if (docsToIndex.length > 0) {
+            await vectorStore.addDocuments(docsToIndex, { ids: uids });
+            numAdded += docsToIndex.length - seenDocs.size;
+            numUpdated += seenDocs.size;
+        }
+        await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
+        if (cleanup === "incremental") {
+            sourceIds.forEach((sourceId) => {
+                if (!sourceId)
+                    throw new Error("Source id cannot be null");
+            });
+            const uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                groupIds: sourceIds,
+            });
+            await vectorStore.delete({ ids: uidsToDelete });
+            await recordManager.deleteKeys(uidsToDelete);
+            numDeleted += uidsToDelete.length;
+        }
+    }
+    if (cleanup === "full") {
+        let uidsToDelete = await recordManager.listKeys({
+            before: indexStartDt,
+            limit: cleanupBatchSize,
+        });
+        while (uidsToDelete.length > 0) {
+            await vectorStore.delete({ ids: uidsToDelete });
+            await recordManager.deleteKeys(uidsToDelete);
+            numDeleted += uidsToDelete.length;
+            uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                limit: cleanupBatchSize,
+            });
+        }
+    }
+    return {
+        numAdded,
+        numDeleted,
+        numUpdated,
+        numSkipped,
+    };
+}
+exports.index = index;

package/dist/indexing/base.d.ts

@@ -0,0 +1,114 @@
+import { VectorStore } from "../vectorstores.js";
+import { RecordManagerInterface } from "./record_manager.js";
+import { DocumentInterface } from "../documents/document.js";
+import { BaseDocumentLoader } from "../document_loaders/base.js";
+type Metadata = Record<string, unknown>;
+type IndexingResult = {
+    numAdded: number;
+    numDeleted: number;
+    numUpdated: number;
+    numSkipped: number;
+};
+type StringOrDocFunc = string | ((doc: DocumentInterface) => string);
+export interface HashedDocumentInterface extends DocumentInterface {
+    uid: string;
+    hash_?: string;
+    contentHash?: string;
+    metadataHash?: string;
+    pageContent: string;
+    metadata: Metadata;
+    calculateHashes(): void;
+    toDocument(): DocumentInterface;
+}
+interface HashedDocumentArgs {
+    pageContent: string;
+    metadata: Metadata;
+    uid: string;
+}
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+export declare class _HashedDocument implements HashedDocumentInterface {
+    uid: string;
+    hash_?: string;
+    contentHash?: string;
+    metadataHash?: string;
+    pageContent: string;
+    metadata: Metadata;
+    constructor(fields: HashedDocumentArgs);
+    calculateHashes(): void;
+    toDocument(): DocumentInterface;
+    static fromDocument(document: DocumentInterface, uid?: string): _HashedDocument;
+    private _hashStringToUUID;
+    private _hashNestedDictToUUID;
+}
+export type CleanupMode = "full" | "incremental";
+export type IndexOptions = {
+    /**
+     * The number of documents to index in one batch.
+     */
+    batchSize?: number;
+    /**
+     * The cleanup mode to use. Can be "full", "incremental" or undefined.
+     * - **Incremental**: Cleans up all documents that haven't been updated AND
+     *   that are associated with source ids that were seen
+     *   during indexing.
+     *   Clean up is done continuously during indexing helping
+     *   to minimize the probability of users seeing duplicated
+     *   content.
+     * - **Full**: Delete all documents that haven to been returned by the loader.
+     *   Clean up runs after all documents have been indexed.
+     *   This means that users may see duplicated content during indexing.
+     * - **undefined**: Do not delete any documents.
+     */
+    cleanup?: CleanupMode;
+    /**
+     * Optional key that helps identify the original source of the document.
+     * Must either be a string representing the key of the source in the metadata
+     * or a function that takes a document and returns a string representing the source.
+     * **Required when cleanup is incremental**.
+     */
+    sourceIdKey?: StringOrDocFunc;
+    /**
+     * Batch size to use when cleaning up documents.
+     */
+    cleanupBatchSize?: number;
+    /**
+     * Force update documents even if they are present in the
+     * record manager. Useful if you are re-indexing with updated embeddings.
+     */
+    forceUpdate?: boolean;
+};
+export declare function _batch<T>(size: number, iterable: T[]): T[][];
+export declare function _deduplicateInOrder(hashedDocuments: HashedDocumentInterface[]): HashedDocumentInterface[];
+export declare function _getSourceIdAssigner(sourceIdKey: StringOrDocFunc | null): (doc: DocumentInterface) => string | null;
+export declare const _isBaseDocumentLoader: (arg: any) => arg is BaseDocumentLoader;
+interface IndexArgs {
+    docsSource: BaseDocumentLoader | DocumentInterface[];
+    recordManager: RecordManagerInterface;
+    vectorStore: VectorStore;
+    options?: IndexOptions;
+}
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+export declare function index(args: IndexArgs): Promise<IndexingResult>;
+export {};

package/dist/indexing/base.js

@@ -0,0 +1,261 @@
+import { v5 as uuidv5 } from "uuid";
+import { UUIDV5_NAMESPACE } from "./record_manager.js";
+import { insecureHash } from "../utils/hash.js";
+import { Document } from "../documents/document.js";
+/**
+ * HashedDocument is a Document with hashes calculated.
+ * Hashes are calculated based on page content and metadata.
+ * It is used for indexing.
+ */
+export class _HashedDocument {
+    constructor(fields) {
+        Object.defineProperty(this, "uid", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "hash_", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contentHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataHash", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.uid = fields.uid;
+        this.pageContent = fields.pageContent;
+        this.metadata = fields.metadata;
+    }
+    calculateHashes() {
+        const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
+        for (const key of forbiddenKeys) {
+            if (key in this.metadata) {
+                throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
+            }
+        }
+        const contentHash = this._hashStringToUUID(this.pageContent);
+        try {
+            const metadataHash = this._hashNestedDictToUUID(this.metadata);
+            this.contentHash = contentHash;
+            this.metadataHash = metadataHash;
+        }
+        catch (e) {
+            throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
+        }
+        this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
+        if (!this.uid) {
+            this.uid = this.hash_;
+        }
+    }
+    toDocument() {
+        return new Document({
+            pageContent: this.pageContent,
+            metadata: this.metadata,
+        });
+    }
+    static fromDocument(document, uid) {
+        const doc = new this({
+            pageContent: document.pageContent,
+            metadata: document.metadata,
+            uid: uid || document.uid,
+        });
+        doc.calculateHashes();
+        return doc;
+    }
+    _hashStringToUUID(inputString) {
+        const hash_value = insecureHash(inputString);
+        return uuidv5(hash_value, UUIDV5_NAMESPACE);
+    }
+    _hashNestedDictToUUID(data) {
+        const serialized_data = JSON.stringify(data, Object.keys(data).sort());
+        const hash_value = insecureHash(serialized_data);
+        return uuidv5(hash_value, UUIDV5_NAMESPACE);
+    }
+}
+export function _batch(size, iterable) {
+    const batches = [];
+    let currentBatch = [];
+    iterable.forEach((item) => {
+        currentBatch.push(item);
+        if (currentBatch.length >= size) {
+            batches.push(currentBatch);
+            currentBatch = [];
+        }
+    });
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+}
+export function _deduplicateInOrder(hashedDocuments) {
+    const seen = new Set();
+    const deduplicated = [];
+    for (const hashedDoc of hashedDocuments) {
+        if (!hashedDoc.hash_) {
+            throw new Error("Hashed document does not have a hash");
+        }
+        if (!seen.has(hashedDoc.hash_)) {
+            seen.add(hashedDoc.hash_);
+            deduplicated.push(hashedDoc);
+        }
+    }
+    return deduplicated;
+}
+export function _getSourceIdAssigner(sourceIdKey) {
+    if (sourceIdKey === null) {
+        return (_doc) => null;
+    }
+    else if (typeof sourceIdKey === "string") {
+        return (doc) => doc.metadata[sourceIdKey];
+    }
+    else if (typeof sourceIdKey === "function") {
+        return sourceIdKey;
+    }
+    else {
+        throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
+    }
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const _isBaseDocumentLoader = (arg) => {
+    if ("load" in arg &&
+        typeof arg.load === "function" &&
+        "loadAndSplit" in arg &&
+        typeof arg.loadAndSplit === "function") {
+        return true;
+    }
+    return false;
+};
+/**
+ * Index data from the doc source into the vector store.
+ *
+ * Indexing functionality uses a manager to keep track of which documents
+ * are in the vector store.
+ *
+ * This allows us to keep track of which documents were updated, and which
+ * documents were deleted, which documents should be skipped.
+ *
+ * For the time being, documents are indexed using their hashes, and users
+ *  are not able to specify the uid of the document.
+ *
+ * @param {IndexArgs} args
+ * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
+ * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
+ * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
+ * @param {IndexOptions | undefined} args.options Options for indexing.
+ * @returns {Promise<IndexingResult>}
+ */
+export async function index(args) {
+    const { docsSource, recordManager, vectorStore, options } = args;
+    const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
+    if (cleanup === "incremental" && !sourceIdKey) {
+        throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
+    }
+    const docs = _isBaseDocumentLoader(docsSource)
+        ? await docsSource.load()
+        : docsSource;
+    const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
+    const indexStartDt = await recordManager.getTime();
+    let numAdded = 0;
+    let numDeleted = 0;
+    let numUpdated = 0;
+    let numSkipped = 0;
+    const batches = _batch(batchSize ?? 100, docs);
+    for (const batch of batches) {
+        const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
+        const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
+        if (cleanup === "incremental") {
+            hashedDocs.forEach((_hashedDoc, index) => {
+                const source = sourceIds[index];
+                if (source === null) {
+                    throw new Error("sourceIdKey must be provided when cleanup is incremental");
+                }
+            });
+        }
+        const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
+        const uids = [];
+        const docsToIndex = [];
+        const docsToUpdate = [];
+        const seenDocs = new Set();
+        hashedDocs.forEach((hashedDoc, i) => {
+            const docExists = batchExists[i];
+            if (docExists) {
+                if (forceUpdate) {
+                    seenDocs.add(hashedDoc.uid);
+                }
+                else {
+                    docsToUpdate.push(hashedDoc.uid);
+                    return;
+                }
+            }
+            uids.push(hashedDoc.uid);
+            docsToIndex.push(hashedDoc.toDocument());
+        });
+        if (docsToUpdate.length > 0) {
+            await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
+            numSkipped += docsToUpdate.length;
+        }
+        if (docsToIndex.length > 0) {
+            await vectorStore.addDocuments(docsToIndex, { ids: uids });
+            numAdded += docsToIndex.length - seenDocs.size;
+            numUpdated += seenDocs.size;
+        }
+        await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
+        if (cleanup === "incremental") {
+            sourceIds.forEach((sourceId) => {
+                if (!sourceId)
+                    throw new Error("Source id cannot be null");
+            });
+            const uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                groupIds: sourceIds,
+            });
+            await vectorStore.delete({ ids: uidsToDelete });
+            await recordManager.deleteKeys(uidsToDelete);
+            numDeleted += uidsToDelete.length;
+        }
+    }
+    if (cleanup === "full") {
+        let uidsToDelete = await recordManager.listKeys({
+            before: indexStartDt,
+            limit: cleanupBatchSize,
+        });
+        while (uidsToDelete.length > 0) {
+            await vectorStore.delete({ ids: uidsToDelete });
+            await recordManager.deleteKeys(uidsToDelete);
+            numDeleted += uidsToDelete.length;
+            uidsToDelete = await recordManager.listKeys({
+                before: indexStartDt,
+                limit: cleanupBatchSize,
+            });
+        }
+    }
+    return {
+        numAdded,
+        numDeleted,
+        numUpdated,
+        numSkipped,
+    };
+}

package/dist/indexing/index.cjs

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./record_manager.cjs"), exports);
+__exportStar(require("./base.cjs"), exports);

package/dist/indexing/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./record_manager.js";
+export * from "./base.js";

package/dist/indexing/index.js

@@ -0,0 +1,2 @@
+export * from "./record_manager.js";
+export * from "./base.js";

package/dist/indexing/record_manager.cjs

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordManager = exports.UUIDV5_NAMESPACE = void 0;
+const serializable_js_1 = require("../load/serializable.cjs");
+// Arbitrary value, used for generating namespaced UUIDs.
+exports.UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+class RecordManager extends serializable_js_1.Serializable {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "recordmanagers"]
+        });
+    }
+}
+exports.RecordManager = RecordManager;

package/dist/indexing/record_manager.d.ts

@@ -0,0 +1,64 @@
+import { Serializable } from "../load/serializable.js";
+export declare const UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+export type UpdateOptions = {
+    groupIds?: (string | null)[];
+    timeAtLeast?: number;
+};
+export type ListKeyOptions = {
+    before?: number;
+    after?: number;
+    groupIds?: (string | null)[];
+    limit?: number;
+};
+export interface RecordManagerInterface {
+    /**
+     * Creates schema in the record manager.
+     * @returns Promise
+     */
+    createSchema(): Promise<void>;
+    /**
+     * Returns current time from the record manager.
+     * @returns Current time
+     */
+    getTime(): Promise<number>;
+    /**
+     * Updates keys in the record manager.
+     * @param keys List of keys to update
+     * @param groupIds List of groupIds to update
+     * @param timeAtLeast Update only if current time is at least this value
+     * @returns Promise
+     * @throws Error if timeAtLeast is provided and current time is less than timeAtLeast
+     * @throws Error if number of keys does not match number of groupIds
+     */
+    update(keys: string[], updateOptions: UpdateOptions): Promise<void>;
+    /**
+     * Checks if keys exist in the record manager.
+     * @param keys List of keys to check
+     * @returns List of booleans indicating if key exists in same order as provided keys
+     */
+    exists(keys: string[]): Promise<boolean[]>;
+    /**
+     * Lists keys from the record manager.
+     * @param before List keys before this timestamp
+     * @param after List keys after this timestamp
+     * @param groupIds List keys with these groupIds
+     * @param limit Limit the number of keys returned
+     * @returns List of keys
+     *
+     */
+    listKeys(options: ListKeyOptions): Promise<string[]>;
+    /**
+     * Deletes keys from the record manager.
+     * @param keys List of keys to delete
+     */
+    deleteKeys(keys: string[]): Promise<void>;
+}
+export declare abstract class RecordManager extends Serializable implements RecordManagerInterface {
+    lc_namespace: string[];
+    abstract createSchema(): Promise<void>;
+    abstract getTime(): Promise<number>;
+    abstract update(keys: string[], updateOptions?: UpdateOptions): Promise<void>;
+    abstract exists(keys: string[]): Promise<boolean[]>;
+    abstract listKeys(options?: ListKeyOptions): Promise<string[]>;
+    abstract deleteKeys(keys: string[]): Promise<void>;
+}

package/dist/indexing/record_manager.js

@@ -0,0 +1,14 @@
+import { Serializable } from "../load/serializable.js";
+// Arbitrary value, used for generating namespaced UUIDs.
+export const UUIDV5_NAMESPACE = "10f90ea3-90a4-4962-bf75-83a0f3c1c62a";
+export class RecordManager extends Serializable {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "recordmanagers"]
+        });
+    }
+}

package/dist/runnables/base.cjs

@@ -16,6 +16,7 @@ const root_listener_js_1 = require("../tracers/root_listener.cjs");
 const utils_js_1 = require("./utils.cjs");
 const index_js_1 = require("../singletons/index.cjs");
 const graph_js_1 = require("./graph.cjs");
+const wrappers_js_1 = require("./wrappers.cjs");
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function _coerceToDict(value, defaultKey) {
     return value &&
@@ -448,46 +449,16 @@ class Runnable extends serializable_js_1.Serializable {
             await runnableStreamConsumePromise;
         }
     }
-    /**
-     * Generate a stream of events emitted by the internal steps of the runnable.
-     *
-     * Use to create an iterator over StreamEvents that provide real-time information
-     * about the progress of the runnable, including StreamEvents from intermediate
-     * results.
-     *
-     * A StreamEvent is a dictionary with the following schema:
-     *
-     * - `event`: string - Event names are of the format: on_[runnable_type]_(start|stream|end).
-     * - `name`: string - The name of the runnable that generated the event.
-     * - `run_id`: string - Randomly generated ID associated with the given execution of
-     *   the runnable that emitted the event. A child runnable that gets invoked as part of the execution of a
-     *   parent runnable is assigned its own unique ID.
-     * - `tags`: string[] - The tags of the runnable that generated the event.
-     * - `metadata`: Record<string, any> - The metadata of the runnable that generated the event.
-     * - `data`: Record<string, any>
-     *
-     * Below is a table that illustrates some events that might be emitted by various
-     * chains. Metadata fields have been omitted from the table for brevity.
-     * Chain definitions have been included after the table.
-     *
-     * | event                | name             | chunk                              | input                                         | output                                          |
-     * |----------------------|------------------|------------------------------------|-----------------------------------------------|-------------------------------------------------|
-     * | on_llm_start         | [model name]     |                                    | {'input': 'hello'}                            |                                                 |
-     * | on_llm_stream        | [model name]     | 'Hello' OR AIMessageChunk("hello") |                                               |                                                 |
-     * | on_llm_end           | [model name]     |                                    | 'Hello human!'                                |
-     * | on_chain_start       | format_docs      |                                    |                                               |                                                 |
-     * | on_chain_stream      | format_docs      | "hello world!, goodbye world!"     |                                               |                                                 |
-     * | on_chain_end         | format_docs      |                                    | [Document(...)]                               | "hello world!, goodbye world!"                  |
-     * | on_tool_start        | some_tool        |                                    | {"x": 1, "y": "2"}                            |                                                 |
-     * | on_tool_stream       | some_tool        |   {"x": 1, "y": "2"}               |                                               |                                                 |
-     * | on_tool_end          | some_tool        |                                    |                                               | {"x": 1, "y": "2"}                              |
-     * | on_retriever_start   | [retriever name] |                                    | {"query": "hello"}                            |                                                 |
-     * | on_retriever_chunk   | [retriever name] |  {documents: [...]}                |                                               |                                                 |
-     * | on_retriever_end     | [retriever name] |                                    | {"query": "hello"}                            | {documents: [...]}                              |
-     * | on_prompt_start      | [template_name]  |                                    | {"question": "hello"}                         |                                                 |
-     * | on_prompt_end        | [template_name]  |                                    | {"question": "hello"}                         | ChatPromptValue(messages: [SystemMessage, ...]) |
-     */
     async *streamEvents(input, options, streamOptions) {
+        if (options.encoding === "text/event-stream") {
+            const stream = await this._streamEvents(input, options, streamOptions);
+            yield* (0, wrappers_js_1.convertToHttpEventStream)(stream);
+        }
+        else {
+            yield* this._streamEvents(input, options, streamOptions);
+        }
+    }
+    async *_streamEvents(input, options, streamOptions) {
         if (options.version !== "v1") {
             throw new Error(`Only version "v1" of the events schema is currently supported.`);
         }

package/dist/runnables/base.d.ts

@@ -199,6 +199,13 @@ export declare abstract class Runnable<RunInput = any, RunOutput = any, CallOpti
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
+    _streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
     static isRunnable(thing: any): thing is Runnable;
     /**
      * Bind lifecycle listeners to a Runnable, returning a new Runnable.
@@ -257,6 +264,10 @@ export declare class RunnableBinding<RunInput, RunOutput, CallOptions extends Ru
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
     static isRunnableBinding(thing: any): thing is RunnableBinding<any, any, any>;
     /**
      * Bind lifecycle listeners to a Runnable, returning a new Runnable.

package/dist/runnables/base.js

@@ -10,6 +10,7 @@ import { RootListenersTracer } from "../tracers/root_listener.js";
 import { _RootEventFilter, isRunnableInterface } from "./utils.js";
 import { AsyncLocalStorageProviderSingleton } from "../singletons/index.js";
 import { Graph } from "./graph.js";
+import { convertToHttpEventStream } from "./wrappers.js";
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function _coerceToDict(value, defaultKey) {
     return value &&
@@ -441,46 +442,16 @@ export class Runnable extends Serializable {
             await runnableStreamConsumePromise;
         }
     }
-    /**
-     * Generate a stream of events emitted by the internal steps of the runnable.
-     *
-     * Use to create an iterator over StreamEvents that provide real-time information
-     * about the progress of the runnable, including StreamEvents from intermediate
-     * results.
-     *
-     * A StreamEvent is a dictionary with the following schema:
-     *
-     * - `event`: string - Event names are of the format: on_[runnable_type]_(start|stream|end).
-     * - `name`: string - The name of the runnable that generated the event.
-     * - `run_id`: string - Randomly generated ID associated with the given execution of
-     *   the runnable that emitted the event. A child runnable that gets invoked as part of the execution of a
-     *   parent runnable is assigned its own unique ID.
-     * - `tags`: string[] - The tags of the runnable that generated the event.
-     * - `metadata`: Record<string, any> - The metadata of the runnable that generated the event.
-     * - `data`: Record<string, any>
-     *
-     * Below is a table that illustrates some events that might be emitted by various
-     * chains. Metadata fields have been omitted from the table for brevity.
-     * Chain definitions have been included after the table.
-     *
-     * | event                | name             | chunk                              | input                                         | output                                          |
-     * |----------------------|------------------|------------------------------------|-----------------------------------------------|-------------------------------------------------|
-     * | on_llm_start         | [model name]     |                                    | {'input': 'hello'}                            |                                                 |
-     * | on_llm_stream        | [model name]     | 'Hello' OR AIMessageChunk("hello") |                                               |                                                 |
-     * | on_llm_end           | [model name]     |                                    | 'Hello human!'                                |
-     * | on_chain_start       | format_docs      |                                    |                                               |                                                 |
-     * | on_chain_stream      | format_docs      | "hello world!, goodbye world!"     |                                               |                                                 |
-     * | on_chain_end         | format_docs      |                                    | [Document(...)]                               | "hello world!, goodbye world!"                  |
-     * | on_tool_start        | some_tool        |                                    | {"x": 1, "y": "2"}                            |                                                 |
-     * | on_tool_stream       | some_tool        |   {"x": 1, "y": "2"}               |                                               |                                                 |
-     * | on_tool_end          | some_tool        |                                    |                                               | {"x": 1, "y": "2"}                              |
-     * | on_retriever_start   | [retriever name] |                                    | {"query": "hello"}                            |                                                 |
-     * | on_retriever_chunk   | [retriever name] |  {documents: [...]}                |                                               |                                                 |
-     * | on_retriever_end     | [retriever name] |                                    | {"query": "hello"}                            | {documents: [...]}                              |
-     * | on_prompt_start      | [template_name]  |                                    | {"question": "hello"}                         |                                                 |
-     * | on_prompt_end        | [template_name]  |                                    | {"question": "hello"}                         | ChatPromptValue(messages: [SystemMessage, ...]) |
-     */
     async *streamEvents(input, options, streamOptions) {
+        if (options.encoding === "text/event-stream") {
+            const stream = await this._streamEvents(input, options, streamOptions);
+            yield* convertToHttpEventStream(stream);
+        }
+        else {
+            yield* this._streamEvents(input, options, streamOptions);
+        }
+    }
+    async *_streamEvents(input, options, streamOptions) {
         if (options.version !== "v1") {
             throw new Error(`Only version "v1" of the events schema is currently supported.`);
         }

package/dist/runnables/remote.d.ts

@@ -30,5 +30,9 @@ export declare class RemoteRunnable<RunInput, RunOutput, CallOptions extends Run
     streamEvents(input: RunInput, options: Partial<CallOptions> & {
         version: "v1";
     }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<StreamEvent>;
+    streamEvents(input: RunInput, options: Partial<CallOptions> & {
+        version: "v1";
+        encoding: "text/event-stream";
+    }, streamOptions?: Omit<LogStreamCallbackHandlerInput, "autoClose">): AsyncGenerator<Uint8Array>;
 }
 export {};

package/dist/runnables/wrappers.cjs

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToHttpEventStream = void 0;
+const stream_js_1 = require("../utils/stream.cjs");
+function convertToHttpEventStream(stream) {
+    const encoder = new TextEncoder();
+    const finalStream = new ReadableStream({
+        async start(controller) {
+            for await (const chunk of stream) {
+                controller.enqueue(encoder.encode(`event: data\ndata: ${JSON.stringify(chunk)}\n\n`));
+            }
+            controller.enqueue(encoder.encode("event: end\n\n"));
+            controller.close();
+        },
+    });
+    return stream_js_1.IterableReadableStream.fromReadableStream(finalStream);
+}
+exports.convertToHttpEventStream = convertToHttpEventStream;

package/dist/runnables/wrappers.d.ts

@@ -0,0 +1,2 @@
+import { IterableReadableStream } from "../utils/stream.js";
+export declare function convertToHttpEventStream(stream: AsyncGenerator): IterableReadableStream<Uint8Array>;

package/dist/runnables/wrappers.js

@@ -0,0 +1,14 @@
+import { IterableReadableStream } from "../utils/stream.js";
+export function convertToHttpEventStream(stream) {
+    const encoder = new TextEncoder();
+    const finalStream = new ReadableStream({
+        async start(controller) {
+            for await (const chunk of stream) {
+                controller.enqueue(encoder.encode(`event: data\ndata: ${JSON.stringify(chunk)}\n\n`));
+            }
+            controller.enqueue(encoder.encode("event: end\n\n"));
+            controller.close();
+        },
+    });
+    return IterableReadableStream.fromReadableStream(finalStream);
+}

package/dist/tracers/base.cjs

@@ -11,9 +11,9 @@ function _coerceToDict(value, defaultKey) {
 function stripNonAlphanumeric(input) {
     return input.replace(/[-:.]/g, "");
 }
-function convertToDottedOrderFormat(epoch, runId) {
-    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}000Z`) +
-        runId);
+function convertToDottedOrderFormat(epoch, runId, executionOrder) {
+    const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
+    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
 class BaseTracer extends base_js_1.BaseCallbackHandler {
     constructor(_fields) {
@@ -42,7 +42,7 @@ class BaseTracer extends base_js_1.BaseCallbackHandler {
         parentRun.child_runs.push(childRun);
     }
     async _startTrace(run) {
-        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id);
+        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id, run.execution_order);
         const storedRun = { ...run };
         if (storedRun.parent_run_id !== undefined) {
             const parentRun = this.runMap.get(storedRun.parent_run_id);

package/dist/tracers/base.js

@@ -8,9 +8,9 @@ function _coerceToDict(value, defaultKey) {
 function stripNonAlphanumeric(input) {
     return input.replace(/[-:.]/g, "");
 }
-function convertToDottedOrderFormat(epoch, runId) {
-    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}000Z`) +
-        runId);
+function convertToDottedOrderFormat(epoch, runId, executionOrder) {
+    const paddedOrder = executionOrder.toFixed(0).slice(0, 3).padStart(3, "0");
+    return (stripNonAlphanumeric(`${new Date(epoch).toISOString().slice(0, -1)}${paddedOrder}Z`) + runId);
 }
 export class BaseTracer extends BaseCallbackHandler {
     constructor(_fields) {
@@ -39,7 +39,7 @@ export class BaseTracer extends BaseCallbackHandler {
         parentRun.child_runs.push(childRun);
     }
     async _startTrace(run) {
-        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id);
+        const currentDottedOrder = convertToDottedOrderFormat(run.start_time, run.id, run.execution_order);
         const storedRun = { ...run };
         if (storedRun.parent_run_id !== undefined) {
             const parentRun = this.runMap.get(storedRun.parent_run_id);

package/document_loaders/base.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/document_loaders/base.cjs');

package/document_loaders/base.d.cts

@@ -0,0 +1 @@
+export * from '../dist/document_loaders/base.js'

package/document_loaders/base.d.ts

@@ -0,0 +1 @@
+export * from '../dist/document_loaders/base.js'

package/document_loaders/base.js

@@ -0,0 +1 @@
+export * from '../dist/document_loaders/base.js'

package/indexing.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/indexing/index.cjs');

package/indexing.d.cts

@@ -0,0 +1 @@
+export * from './dist/indexing/index.js'

package/indexing.d.ts

@@ -0,0 +1 @@
+export * from './dist/indexing/index.js'

package/indexing.js

@@ -0,0 +1 @@
+export * from './dist/indexing/index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/core",
-  "version": "0.1.61",
+  "version": "0.2.0-rc.0",
   "description": "Core LangChain.js abstractions and schemas",
   "type": "module",
   "engines": {
@@ -158,6 +158,15 @@
       "import": "./documents.js",
       "require": "./documents.cjs"
     },
+    "./document_loaders/base": {
+      "types": {
+        "import": "./document_loaders/base.d.ts",
+        "require": "./document_loaders/base.d.cts",
+        "default": "./document_loaders/base.d.ts"
+      },
+      "import": "./document_loaders/base.js",
+      "require": "./document_loaders/base.cjs"
+    },
     "./embeddings": {
       "types": {
         "import": "./embeddings.d.ts",
@@ -176,6 +185,15 @@
       "import": "./example_selectors.js",
       "require": "./example_selectors.cjs"
     },
+    "./indexing": {
+      "types": {
+        "import": "./indexing.d.ts",
+        "require": "./indexing.d.cts",
+        "default": "./indexing.d.ts"
+      },
+      "import": "./indexing.js",
+      "require": "./indexing.cjs"
+    },
     "./language_models/base": {
       "types": {
         "import": "./language_models/base.d.ts",
@@ -604,6 +622,10 @@
     "documents.js",
     "documents.d.ts",
     "documents.d.cts",
+    "document_loaders/base.cjs",
+    "document_loaders/base.js",
+    "document_loaders/base.d.ts",
+    "document_loaders/base.d.cts",
     "embeddings.cjs",
     "embeddings.js",
     "embeddings.d.ts",
@@ -612,6 +634,10 @@
     "example_selectors.js",
     "example_selectors.d.ts",
     "example_selectors.d.cts",
+    "indexing.cjs",
+    "indexing.js",
+    "indexing.d.ts",
+    "indexing.d.cts",
     "language_models/base.cjs",
     "language_models/base.js",
     "language_models/base.d.ts",