langchain 0.3.30 → 1.0.0-alpha.1

package/dist/document_loaders/fs/text.d.ts

@@ -1,6 +1,9 @@
-import type { readFile as ReadFileT } from "node:fs/promises";
 import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+import { readFile } from "node:fs/promises";
+
+//#region src/document_loaders/fs/text.d.ts
+
 /**
  * A class that extends the `BaseDocumentLoader` class. It represents a
  * document loader that loads documents from a text file. The `load()`
@@ -14,38 +17,41 @@ import { BaseDocumentLoader } from "../base.js";
  * const docs = await loader.load();
  * ```
  */
-export declare class TextLoader extends BaseDocumentLoader {
-    filePathOrBlob: string | Blob;
-    constructor(filePathOrBlob: string | Blob);
-    /**
-     * A protected method that takes a `raw` string as a parameter and returns
-     * a promise that resolves to an array containing the raw text as a single
-     * element.
-     * @param raw The raw text to be parsed.
-     * @returns A promise that resolves to an array containing the raw text as a single element.
-     */
-    protected parse(raw: string): Promise<string[]>;
-    /**
-     * A method that loads the text file or blob and returns a promise that
-     * resolves to an array of `Document` instances. It reads the text from
-     * the file or blob using the `readFile` function from the
-     * `node:fs/promises` module or the `text()` method of the blob. It then
-     * parses the text using the `parse()` method and creates a `Document`
-     * instance for each parsed page. The metadata includes the source of the
-     * text (file path or blob) and, if there are multiple pages, the line
-     * number of each page.
-     * @returns A promise that resolves to an array of `Document` instances.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * A static method that imports the `readFile` function from the
-     * `node:fs/promises` module. It is used to dynamically import the
-     * function when needed. If the import fails, it throws an error
-     * indicating that the `fs/promises` module is not available in the
-     * current environment.
-     * @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.
-     */
-    static imports(): Promise<{
-        readFile: typeof ReadFileT;
-    }>;
+declare class TextLoader extends BaseDocumentLoader {
+  filePathOrBlob: string | Blob;
+  constructor(filePathOrBlob: string | Blob);
+  /**
+   * A protected method that takes a `raw` string as a parameter and returns
+   * a promise that resolves to an array containing the raw text as a single
+   * element.
+   * @param raw The raw text to be parsed.
+   * @returns A promise that resolves to an array containing the raw text as a single element.
+   */
+  protected parse(raw: string): Promise<string[]>;
+  /**
+   * A method that loads the text file or blob and returns a promise that
+   * resolves to an array of `Document` instances. It reads the text from
+   * the file or blob using the `readFile` function from the
+   * `node:fs/promises` module or the `text()` method of the blob. It then
+   * parses the text using the `parse()` method and creates a `Document`
+   * instance for each parsed page. The metadata includes the source of the
+   * text (file path or blob) and, if there are multiple pages, the line
+   * number of each page.
+   * @returns A promise that resolves to an array of `Document` instances.
+   */
+  load(): Promise<Document[]>;
+  /**
+   * A static method that imports the `readFile` function from the
+   * `node:fs/promises` module. It is used to dynamically import the
+   * function when needed. If the import fails, it throws an error
+   * indicating that the `fs/promises` module is not available in the
+   * current environment.
+   * @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.
+   */
+  static imports(): Promise<{
+    readFile: typeof readFile;
+  }>;
 }
+//#endregion
+export { TextLoader };
+//# sourceMappingURL=text.d.ts.map

package/dist/document_loaders/fs/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","names":["readFile","ReadFileT","Document","BaseDocumentLoader","TextLoader","Blob","Promise"],"sources":["../../../src/document_loaders/fs/text.d.ts"],"sourcesContent":["import type { readFile as ReadFileT } from \"node:fs/promises\";\nimport { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n/**\n * A class that extends the `BaseDocumentLoader` class. It represents a\n * document loader that loads documents from a text file. The `load()`\n * method is implemented to read the text from the file or blob, parse it\n * using the `parse()` method, and create a `Document` instance for each\n * parsed page. The metadata includes the source of the text (file path or\n * blob) and, if there are multiple pages, the line number of each page.\n * @example\n * ```typescript\n * const loader = new TextLoader(\"src/document_loaders/example_data/example.txt\");\n * const docs = await loader.load();\n * ```\n */\nexport declare class TextLoader extends BaseDocumentLoader {\n    filePathOrBlob: string | Blob;\n    constructor(filePathOrBlob: string | Blob);\n    /**\n     * A protected method that takes a `raw` string as a parameter and returns\n     * a promise that resolves to an array containing the raw text as a single\n     * element.\n     * @param raw The raw text to be parsed.\n     * @returns A promise that resolves to an array containing the raw text as a single element.\n     */\n    protected parse(raw: string): Promise<string[]>;\n    /**\n     * A method that loads the text file or blob and returns a promise that\n     * resolves to an array of `Document` instances. It reads the text from\n     * the file or blob using the `readFile` function from the\n     * `node:fs/promises` module or the `text()` method of the blob. It then\n     * parses the text using the `parse()` method and creates a `Document`\n     * instance for each parsed page. The metadata includes the source of the\n     * text (file path or blob) and, if there are multiple pages, the line\n     * number of each page.\n     * @returns A promise that resolves to an array of `Document` instances.\n     */\n    load(): Promise<Document[]>;\n    /**\n     * A static method that imports the `readFile` function from the\n     * `node:fs/promises` module. It is used to dynamically import the\n     * function when needed. If the import fails, it throws an error\n     * indicating that the `fs/promises` module is not available in the\n     * current environment.\n     * @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.\n     */\n    static imports(): Promise<{\n        readFile: typeof ReadFileT;\n    }>;\n}\n"],"mappings":";;;;;;;;AAgBA;;;;;;;;;;AAA0D;cAArCI,UAAAA,SAAmBD,kBAAAA;2BACXE;uCACYA;;;;;;;;gCAQPC;;;;;;;;;;;;UAYtBA,QAAQJ;;;;;;;;;oBASEI;qBACGL"}

package/dist/document_loaders/fs/text.js

@@ -1,94 +1,95 @@
+import { __export } from "../../_virtual/rolldown_runtime.js";
 import { Document } from "@langchain/core/documents";
 import { getEnv } from "@langchain/core/utils/env";
-import { BaseDocumentLoader } from "../base.js";
+import { BaseDocumentLoader } from "@langchain/core/document_loaders/base";
+
+//#region src/document_loaders/fs/text.ts
+var text_exports = {};
+__export(text_exports, { TextLoader: () => TextLoader });
 /**
- * A class that extends the `BaseDocumentLoader` class. It represents a
- * document loader that loads documents from a text file. The `load()`
- * method is implemented to read the text from the file or blob, parse it
- * using the `parse()` method, and create a `Document` instance for each
- * parsed page. The metadata includes the source of the text (file path or
- * blob) and, if there are multiple pages, the line number of each page.
- * @example
- * ```typescript
- * const loader = new TextLoader("src/document_loaders/example_data/example.txt");
- * const docs = await loader.load();
- * ```
- */
-export class TextLoader extends BaseDocumentLoader {
-    constructor(filePathOrBlob) {
-        super();
-        Object.defineProperty(this, "filePathOrBlob", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: filePathOrBlob
-        });
-    }
-    /**
-     * A protected method that takes a `raw` string as a parameter and returns
-     * a promise that resolves to an array containing the raw text as a single
-     * element.
-     * @param raw The raw text to be parsed.
-     * @returns A promise that resolves to an array containing the raw text as a single element.
-     */
-    async parse(raw) {
-        return [raw];
-    }
-    /**
-     * A method that loads the text file or blob and returns a promise that
-     * resolves to an array of `Document` instances. It reads the text from
-     * the file or blob using the `readFile` function from the
-     * `node:fs/promises` module or the `text()` method of the blob. It then
-     * parses the text using the `parse()` method and creates a `Document`
-     * instance for each parsed page. The metadata includes the source of the
-     * text (file path or blob) and, if there are multiple pages, the line
-     * number of each page.
-     * @returns A promise that resolves to an array of `Document` instances.
-     */
-    async load() {
-        let text;
-        let metadata;
-        if (typeof this.filePathOrBlob === "string") {
-            const { readFile } = await TextLoader.imports();
-            text = await readFile(this.filePathOrBlob, "utf8");
-            metadata = { source: this.filePathOrBlob };
-        }
-        else {
-            text = await this.filePathOrBlob.text();
-            metadata = { source: "blob", blobType: this.filePathOrBlob.type };
-        }
-        const parsed = await this.parse(text);
-        parsed.forEach((pageContent, i) => {
-            if (typeof pageContent !== "string") {
-                throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`);
-            }
-        });
-        return parsed.map((pageContent, i) => new Document({
-            pageContent,
-            metadata: parsed.length === 1
-                ? metadata
-                : {
-                    ...metadata,
-                    line: i + 1,
-                },
-        }));
-    }
-    /**
-     * A static method that imports the `readFile` function from the
-     * `node:fs/promises` module. It is used to dynamically import the
-     * function when needed. If the import fails, it throws an error
-     * indicating that the `fs/promises` module is not available in the
-     * current environment.
-     * @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.
-     */
-    static async imports() {
-        try {
-            const { readFile } = await import("node:fs/promises");
-            return { readFile };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error(`Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`);
-        }
-    }
-}
+* A class that extends the `BaseDocumentLoader` class. It represents a
+* document loader that loads documents from a text file. The `load()`
+* method is implemented to read the text from the file or blob, parse it
+* using the `parse()` method, and create a `Document` instance for each
+* parsed page. The metadata includes the source of the text (file path or
+* blob) and, if there are multiple pages, the line number of each page.
+* @example
+* ```typescript
+* const loader = new TextLoader("src/document_loaders/example_data/example.txt");
+* const docs = await loader.load();
+* ```
+*/
+var TextLoader = class TextLoader extends BaseDocumentLoader {
+	constructor(filePathOrBlob) {
+		super();
+		this.filePathOrBlob = filePathOrBlob;
+	}
+	/**
+	* A protected method that takes a `raw` string as a parameter and returns
+	* a promise that resolves to an array containing the raw text as a single
+	* element.
+	* @param raw The raw text to be parsed.
+	* @returns A promise that resolves to an array containing the raw text as a single element.
+	*/
+	async parse(raw) {
+		return [raw];
+	}
+	/**
+	* A method that loads the text file or blob and returns a promise that
+	* resolves to an array of `Document` instances. It reads the text from
+	* the file or blob using the `readFile` function from the
+	* `node:fs/promises` module or the `text()` method of the blob. It then
+	* parses the text using the `parse()` method and creates a `Document`
+	* instance for each parsed page. The metadata includes the source of the
+	* text (file path or blob) and, if there are multiple pages, the line
+	* number of each page.
+	* @returns A promise that resolves to an array of `Document` instances.
+	*/
+	async load() {
+		let text;
+		let metadata;
+		if (typeof this.filePathOrBlob === "string") {
+			const { readFile } = await TextLoader.imports();
+			text = await readFile(this.filePathOrBlob, "utf8");
+			metadata = { source: this.filePathOrBlob };
+		} else {
+			text = await this.filePathOrBlob.text();
+			metadata = {
+				source: "blob",
+				blobType: this.filePathOrBlob.type
+			};
+		}
+		const parsed = await this.parse(text);
+		parsed.forEach((pageContent, i) => {
+			if (typeof pageContent !== "string") throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`);
+		});
+		return parsed.map((pageContent, i) => new Document({
+			pageContent,
+			metadata: parsed.length === 1 ? metadata : {
+				...metadata,
+				line: i + 1
+			}
+		}));
+	}
+	/**
+	* A static method that imports the `readFile` function from the
+	* `node:fs/promises` module. It is used to dynamically import the
+	* function when needed. If the import fails, it throws an error
+	* indicating that the `fs/promises` module is not available in the
+	* current environment.
+	* @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.
+	*/
+	static async imports() {
+		try {
+			const { readFile } = await import("node:fs/promises");
+			return { readFile };
+		} catch (e) {
+			console.error(e);
+			throw new Error(`Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`);
+		}
+	}
+};
+
+//#endregion
+export { TextLoader, text_exports };
+//# sourceMappingURL=text.js.map

package/dist/document_loaders/fs/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","names":["filePathOrBlob: string | Blob","raw: string","text: string","metadata: Record<string, string>"],"sources":["../../../src/document_loaders/fs/text.ts"],"sourcesContent":["import type { readFile as ReadFileT } from \"node:fs/promises\";\nimport { Document } from \"@langchain/core/documents\";\nimport { getEnv } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\n/**\n * A class that extends the `BaseDocumentLoader` class. It represents a\n * document loader that loads documents from a text file. The `load()`\n * method is implemented to read the text from the file or blob, parse it\n * using the `parse()` method, and create a `Document` instance for each\n * parsed page. The metadata includes the source of the text (file path or\n * blob) and, if there are multiple pages, the line number of each page.\n * @example\n * ```typescript\n * const loader = new TextLoader(\"src/document_loaders/example_data/example.txt\");\n * const docs = await loader.load();\n * ```\n */\nexport class TextLoader extends BaseDocumentLoader {\n  constructor(public filePathOrBlob: string | Blob) {\n    super();\n  }\n\n  /**\n   * A protected method that takes a `raw` string as a parameter and returns\n   * a promise that resolves to an array containing the raw text as a single\n   * element.\n   * @param raw The raw text to be parsed.\n   * @returns A promise that resolves to an array containing the raw text as a single element.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    return [raw];\n  }\n\n  /**\n   * A method that loads the text file or blob and returns a promise that\n   * resolves to an array of `Document` instances. It reads the text from\n   * the file or blob using the `readFile` function from the\n   * `node:fs/promises` module or the `text()` method of the blob. It then\n   * parses the text using the `parse()` method and creates a `Document`\n   * instance for each parsed page. The metadata includes the source of the\n   * text (file path or blob) and, if there are multiple pages, the line\n   * number of each page.\n   * @returns A promise that resolves to an array of `Document` instances.\n   */\n  public async load(): Promise<Document[]> {\n    let text: string;\n    let metadata: Record<string, string>;\n    if (typeof this.filePathOrBlob === \"string\") {\n      const { readFile } = await TextLoader.imports();\n      text = await readFile(this.filePathOrBlob, \"utf8\");\n      metadata = { source: this.filePathOrBlob };\n    } else {\n      text = await this.filePathOrBlob.text();\n      metadata = { source: \"blob\", blobType: this.filePathOrBlob.type };\n    }\n    const parsed = await this.parse(text);\n    parsed.forEach((pageContent, i) => {\n      if (typeof pageContent !== \"string\") {\n        throw new Error(\n          `Expected string, at position ${i} got ${typeof pageContent}`\n        );\n      }\n    });\n    return parsed.map(\n      (pageContent, i) =>\n        new Document({\n          pageContent,\n          metadata:\n            parsed.length === 1\n              ? metadata\n              : {\n                  ...metadata,\n                  line: i + 1,\n                },\n        })\n    );\n  }\n\n  /**\n   * A static method that imports the `readFile` function from the\n   * `node:fs/promises` module. It is used to dynamically import the\n   * function when needed. If the import fails, it throws an error\n   * indicating that the `fs/promises` module is not available in the\n   * current environment.\n   * @returns A promise that resolves to an object containing the `readFile` function from the `node:fs/promises` module.\n   */\n  static async imports(): Promise<{\n    readFile: typeof ReadFileT;\n  }> {\n    try {\n      const { readFile } = await import(\"node:fs/promises\");\n      return { readFile };\n    } catch (e) {\n      console.error(e);\n      throw new Error(\n        `Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkBA,IAAa,aAAb,MAAa,mBAAmB,mBAAmB;CACjD,YAAmBA,gBAA+B;EAChD,OAAO;EADU;CAElB;;;;;;;;CASD,MAAgB,MAAMC,KAAgC;AACpD,SAAO,CAAC,GAAI;CACb;;;;;;;;;;;;CAaD,MAAa,OAA4B;EACvC,IAAIC;EACJ,IAAIC;AACJ,MAAI,OAAO,KAAK,mBAAmB,UAAU;GAC3C,MAAM,EAAE,UAAU,GAAG,MAAM,WAAW,SAAS;GAC/C,OAAO,MAAM,SAAS,KAAK,gBAAgB,OAAO;GAClD,WAAW,EAAE,QAAQ,KAAK,eAAgB;EAC3C,OAAM;GACL,OAAO,MAAM,KAAK,eAAe,MAAM;GACvC,WAAW;IAAE,QAAQ;IAAQ,UAAU,KAAK,eAAe;GAAM;EAClE;EACD,MAAM,SAAS,MAAM,KAAK,MAAM,KAAK;EACrC,OAAO,QAAQ,CAAC,aAAa,MAAM;AACjC,OAAI,OAAO,gBAAgB,SACzB,OAAM,IAAI,MACR,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,OAAO,aAAa;EAGlE,EAAC;AACF,SAAO,OAAO,IACZ,CAAC,aAAa,MACZ,IAAI,SAAS;GACX;GACA,UACE,OAAO,WAAW,IACd,WACA;IACE,GAAG;IACH,MAAM,IAAI;GACX;EACR,GACJ;CACF;;;;;;;;;CAUD,aAAa,UAEV;AACD,MAAI;GACF,MAAM,EAAE,UAAU,GAAG,MAAM,OAAO;AAClC,UAAO,EAAE,SAAU;EACpB,SAAQ,GAAG;GACV,QAAQ,MAAM,EAAE;AAChB,SAAM,IAAI,MACR,CAAC,qHAAqH,EAAE,QAAQ,CAAC,+CAA+C,CAAC;EAEpL;CACF;AACF"}

package/dist/embeddings/cache_backed.cjs

@@ -1,143 +1,140 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CacheBackedEmbeddings = void 0;
-const hash_1 = require("@langchain/core/utils/hash");
-const embeddings_1 = require("@langchain/core/embeddings");
-const encoder_backed_js_1 = require("../storage/encoder_backed.cjs");
+const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_storage_encoder_backed = require('../storage/encoder_backed.cjs');
+const __langchain_core_utils_hash = require_rolldown_runtime.__toESM(require("@langchain/core/utils/hash"));
+const __langchain_core_embeddings = require_rolldown_runtime.__toESM(require("@langchain/core/embeddings"));
+
+//#region src/embeddings/cache_backed.ts
+var cache_backed_exports = {};
+require_rolldown_runtime.__export(cache_backed_exports, { CacheBackedEmbeddings: () => CacheBackedEmbeddings });
 /**
- * Interface for caching results from embedding models.
- *
- * The interface allows works with any store that implements
- * the abstract store interface accepting keys of type str and values of list of
- * floats.
- *
- * If need be, the interface can be extended to accept other implementations
- * of the value serializer and deserializer, as well as the key encoder.
- * @example
- * ```typescript
- * const underlyingEmbeddings = new OpenAIEmbeddings();
- *
- * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(
- *   underlyingEmbeddings,
- *   new ConvexKVStore({ ctx }),
- *   {
- *     namespace: underlyingEmbeddings.modelName,
- *   },
- * );
- *
- * const loader = new TextLoader("./state_of_the_union.txt");
- * const rawDocuments = await loader.load();
- * const splitter = new RecursiveCharacterTextSplitter({
- *   chunkSize: 1000,
- *   chunkOverlap: 0,
- * });
- * const documents = await splitter.splitDocuments(rawDocuments);
- *
- * let time = Date.now();
- * const vectorstore = await ConvexVectorStore.fromDocuments(
- *   documents,
- *   cacheBackedEmbeddings,
- *   { ctx },
- * );
- * console.log(`Initial creation time: ${Date.now() - time}ms`);
- *
- * time = Date.now();
- * const vectorstore2 = await ConvexVectorStore.fromDocuments(
- *   documents,
- *   cacheBackedEmbeddings,
- *   { ctx },
- * );
- * console.log(`Cached creation time: ${Date.now() - time}ms`);
- *
- * ```
- */
-class CacheBackedEmbeddings extends embeddings_1.Embeddings {
-    constructor(fields) {
-        super(fields);
-        Object.defineProperty(this, "underlyingEmbeddings", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "documentEmbeddingStore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.underlyingEmbeddings = fields.underlyingEmbeddings;
-        this.documentEmbeddingStore = fields.documentEmbeddingStore;
-    }
-    /**
-     * Embed query text.
-     *
-     * This method does not support caching at the moment.
-     *
-     * Support for caching queries is easy to implement, but might make
-     * sense to hold off to see the most common patterns.
-     *
-     * If the cache has an eviction policy, we may need to be a bit more careful
-     * about sharing the cache between documents and queries. Generally,
-     * one is OK evicting query caches, but document caches should be kept.
-     *
-     * @param document The text to embed.
-     * @returns The embedding for the given text.
-     */
-    async embedQuery(document) {
-        return this.underlyingEmbeddings.embedQuery(document);
-    }
-    /**
-     * Embed a list of texts.
-     *
-     * The method first checks the cache for the embeddings.
-     * If the embeddings are not found, the method uses the underlying embedder
-     * to embed the documents and stores the results in the cache.
-     *
-     * @param documents
-     * @returns A list of embeddings for the given texts.
-     */
-    async embedDocuments(documents) {
-        const vectors = await this.documentEmbeddingStore.mget(documents);
-        const missingIndicies = [];
-        const missingDocuments = [];
-        for (let i = 0; i < vectors.length; i += 1) {
-            if (vectors[i] === undefined) {
-                missingIndicies.push(i);
-                missingDocuments.push(documents[i]);
-            }
-        }
-        if (missingDocuments.length) {
-            const missingVectors = await this.underlyingEmbeddings.embedDocuments(missingDocuments);
-            const keyValuePairs = missingDocuments.map((document, i) => [document, missingVectors[i]]);
-            await this.documentEmbeddingStore.mset(keyValuePairs);
-            for (let i = 0; i < missingIndicies.length; i += 1) {
-                vectors[missingIndicies[i]] = missingVectors[i];
-            }
-        }
-        return vectors;
-    }
-    /**
-     * Create a new CacheBackedEmbeddings instance from another embeddings instance
-     * and a storage instance.
-     * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.
-     * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.
-     * @param options.namespace Optional namespace for store keys.
-     * @returns A new CacheBackedEmbeddings instance.
-     */
-    static fromBytesStore(underlyingEmbeddings, documentEmbeddingStore, options) {
-        const encoder = new TextEncoder();
-        const decoder = new TextDecoder();
-        const encoderBackedStore = new encoder_backed_js_1.EncoderBackedStore({
-            store: documentEmbeddingStore,
-            keyEncoder: (key) => (options?.namespace ?? "") + (0, hash_1.insecureHash)(key),
-            valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
-            valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue)),
-        });
-        return new this({
-            underlyingEmbeddings,
-            documentEmbeddingStore: encoderBackedStore,
-        });
-    }
-}
+* Interface for caching results from embedding models.
+*
+* The interface allows works with any store that implements
+* the abstract store interface accepting keys of type str and values of list of
+* floats.
+*
+* If need be, the interface can be extended to accept other implementations
+* of the value serializer and deserializer, as well as the key encoder.
+* @example
+* ```typescript
+* const underlyingEmbeddings = new OpenAIEmbeddings();
+*
+* const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(
+*   underlyingEmbeddings,
+*   new ConvexKVStore({ ctx }),
+*   {
+*     namespace: underlyingEmbeddings.modelName,
+*   },
+* );
+*
+* const loader = new TextLoader("./state_of_the_union.txt");
+* const rawDocuments = await loader.load();
+* const splitter = new RecursiveCharacterTextSplitter({
+*   chunkSize: 1000,
+*   chunkOverlap: 0,
+* });
+* const documents = await splitter.splitDocuments(rawDocuments);
+*
+* let time = Date.now();
+* const vectorstore = await ConvexVectorStore.fromDocuments(
+*   documents,
+*   cacheBackedEmbeddings,
+*   { ctx },
+* );
+* console.log(`Initial creation time: ${Date.now() - time}ms`);
+*
+* time = Date.now();
+* const vectorstore2 = await ConvexVectorStore.fromDocuments(
+*   documents,
+*   cacheBackedEmbeddings,
+*   { ctx },
+* );
+* console.log(`Cached creation time: ${Date.now() - time}ms`);
+*
+* ```
+*/
+var CacheBackedEmbeddings = class extends __langchain_core_embeddings.Embeddings {
+	underlyingEmbeddings;
+	documentEmbeddingStore;
+	constructor(fields) {
+		super(fields);
+		this.underlyingEmbeddings = fields.underlyingEmbeddings;
+		this.documentEmbeddingStore = fields.documentEmbeddingStore;
+	}
+	/**
+	* Embed query text.
+	*
+	* This method does not support caching at the moment.
+	*
+	* Support for caching queries is easy to implement, but might make
+	* sense to hold off to see the most common patterns.
+	*
+	* If the cache has an eviction policy, we may need to be a bit more careful
+	* about sharing the cache between documents and queries. Generally,
+	* one is OK evicting query caches, but document caches should be kept.
+	*
+	* @param document The text to embed.
+	* @returns The embedding for the given text.
+	*/
+	async embedQuery(document) {
+		return this.underlyingEmbeddings.embedQuery(document);
+	}
+	/**
+	* Embed a list of texts.
+	*
+	* The method first checks the cache for the embeddings.
+	* If the embeddings are not found, the method uses the underlying embedder
+	* to embed the documents and stores the results in the cache.
+	*
+	* @param documents
+	* @returns A list of embeddings for the given texts.
+	*/
+	async embedDocuments(documents) {
+		const vectors = await this.documentEmbeddingStore.mget(documents);
+		const missingIndicies = [];
+		const missingDocuments = [];
+		for (let i = 0; i < vectors.length; i += 1) if (vectors[i] === void 0) {
+			missingIndicies.push(i);
+			missingDocuments.push(documents[i]);
+		}
+		if (missingDocuments.length) {
+			const missingVectors = await this.underlyingEmbeddings.embedDocuments(missingDocuments);
+			const keyValuePairs = missingDocuments.map((document, i) => [document, missingVectors[i]]);
+			await this.documentEmbeddingStore.mset(keyValuePairs);
+			for (let i = 0; i < missingIndicies.length; i += 1) vectors[missingIndicies[i]] = missingVectors[i];
+		}
+		return vectors;
+	}
+	/**
+	* Create a new CacheBackedEmbeddings instance from another embeddings instance
+	* and a storage instance.
+	* @param underlyingEmbeddings Embeddings used to populate the cache for new documents.
+	* @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.
+	* @param options.namespace Optional namespace for store keys.
+	* @returns A new CacheBackedEmbeddings instance.
+	*/
+	static fromBytesStore(underlyingEmbeddings, documentEmbeddingStore, options) {
+		const encoder = new TextEncoder();
+		const decoder = new TextDecoder();
+		const encoderBackedStore = new require_storage_encoder_backed.EncoderBackedStore({
+			store: documentEmbeddingStore,
+			keyEncoder: (key) => (options?.namespace ?? "") + (0, __langchain_core_utils_hash.insecureHash)(key),
+			valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
+			valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue))
+		});
+		return new this({
+			underlyingEmbeddings,
+			documentEmbeddingStore: encoderBackedStore
+		});
+	}
+};
+
+//#endregion
 exports.CacheBackedEmbeddings = CacheBackedEmbeddings;
+Object.defineProperty(exports, 'cache_backed_exports', {
+  enumerable: true,
+  get: function () {
+    return cache_backed_exports;
+  }
+});
+//# sourceMappingURL=cache_backed.cjs.map

package/dist/embeddings/cache_backed.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache_backed.cjs","names":["Embeddings","fields: CacheBackedEmbeddingsFields","document: string","documents: string[]","keyValuePairs: [string, number[]][]","underlyingEmbeddings: EmbeddingsInterface","documentEmbeddingStore: BaseStore<string, Uint8Array>","options?: {\n      namespace?: string;\n    }","EncoderBackedStore"],"sources":["../../src/embeddings/cache_backed.ts"],"sourcesContent":["import { insecureHash } from \"@langchain/core/utils/hash\";\nimport {\n  type EmbeddingsInterface,\n  Embeddings,\n} from \"@langchain/core/embeddings\";\nimport { BaseStore } from \"@langchain/core/stores\";\n\nimport { AsyncCallerParams } from \"@langchain/core/utils/async_caller\";\nimport { EncoderBackedStore } from \"../storage/encoder_backed.js\";\n\n/**\n * Interface for the fields required to initialize an instance of the\n * CacheBackedEmbeddings class.\n */\nexport interface CacheBackedEmbeddingsFields extends AsyncCallerParams {\n  underlyingEmbeddings: EmbeddingsInterface;\n  documentEmbeddingStore: BaseStore<string, number[]>;\n}\n\n/**\n * Interface for caching results from embedding models.\n *\n * The interface allows works with any store that implements\n * the abstract store interface accepting keys of type str and values of list of\n * floats.\n *\n * If need be, the interface can be extended to accept other implementations\n * of the value serializer and deserializer, as well as the key encoder.\n * @example\n * ```typescript\n * const underlyingEmbeddings = new OpenAIEmbeddings();\n *\n * const cacheBackedEmbeddings = CacheBackedEmbeddings.fromBytesStore(\n *   underlyingEmbeddings,\n *   new ConvexKVStore({ ctx }),\n *   {\n *     namespace: underlyingEmbeddings.modelName,\n *   },\n * );\n *\n * const loader = new TextLoader(\"./state_of_the_union.txt\");\n * const rawDocuments = await loader.load();\n * const splitter = new RecursiveCharacterTextSplitter({\n *   chunkSize: 1000,\n *   chunkOverlap: 0,\n * });\n * const documents = await splitter.splitDocuments(rawDocuments);\n *\n * let time = Date.now();\n * const vectorstore = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Initial creation time: ${Date.now() - time}ms`);\n *\n * time = Date.now();\n * const vectorstore2 = await ConvexVectorStore.fromDocuments(\n *   documents,\n *   cacheBackedEmbeddings,\n *   { ctx },\n * );\n * console.log(`Cached creation time: ${Date.now() - time}ms`);\n *\n * ```\n */\nexport class CacheBackedEmbeddings extends Embeddings {\n  protected underlyingEmbeddings: EmbeddingsInterface;\n\n  protected documentEmbeddingStore: BaseStore<string, number[]>;\n\n  constructor(fields: CacheBackedEmbeddingsFields) {\n    super(fields);\n    this.underlyingEmbeddings = fields.underlyingEmbeddings;\n    this.documentEmbeddingStore = fields.documentEmbeddingStore;\n  }\n\n  /**\n   * Embed query text.\n   *\n   * This method does not support caching at the moment.\n   *\n   * Support for caching queries is easy to implement, but might make\n   * sense to hold off to see the most common patterns.\n   *\n   * If the cache has an eviction policy, we may need to be a bit more careful\n   * about sharing the cache between documents and queries. Generally,\n   * one is OK evicting query caches, but document caches should be kept.\n   *\n   * @param document The text to embed.\n   * @returns The embedding for the given text.\n   */\n  async embedQuery(document: string): Promise<number[]> {\n    return this.underlyingEmbeddings.embedQuery(document);\n  }\n\n  /**\n   * Embed a list of texts.\n   *\n   * The method first checks the cache for the embeddings.\n   * If the embeddings are not found, the method uses the underlying embedder\n   * to embed the documents and stores the results in the cache.\n   *\n   * @param documents\n   * @returns A list of embeddings for the given texts.\n   */\n  async embedDocuments(documents: string[]): Promise<number[][]> {\n    const vectors = await this.documentEmbeddingStore.mget(documents);\n    const missingIndicies = [];\n    const missingDocuments = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      if (vectors[i] === undefined) {\n        missingIndicies.push(i);\n        missingDocuments.push(documents[i]);\n      }\n    }\n    if (missingDocuments.length) {\n      const missingVectors = await this.underlyingEmbeddings.embedDocuments(\n        missingDocuments\n      );\n      const keyValuePairs: [string, number[]][] = missingDocuments.map(\n        (document, i) => [document, missingVectors[i]]\n      );\n      await this.documentEmbeddingStore.mset(keyValuePairs);\n      for (let i = 0; i < missingIndicies.length; i += 1) {\n        vectors[missingIndicies[i]] = missingVectors[i];\n      }\n    }\n    return vectors as number[][];\n  }\n\n  /**\n   * Create a new CacheBackedEmbeddings instance from another embeddings instance\n   * and a storage instance.\n   * @param underlyingEmbeddings Embeddings used to populate the cache for new documents.\n   * @param documentEmbeddingStore Stores raw document embedding values. Keys are hashes of the document content.\n   * @param options.namespace Optional namespace for store keys.\n   * @returns A new CacheBackedEmbeddings instance.\n   */\n  static fromBytesStore(\n    underlyingEmbeddings: EmbeddingsInterface,\n    documentEmbeddingStore: BaseStore<string, Uint8Array>,\n    options?: {\n      namespace?: string;\n    }\n  ) {\n    const encoder = new TextEncoder();\n    const decoder = new TextDecoder();\n    const encoderBackedStore = new EncoderBackedStore<\n      string,\n      number[],\n      Uint8Array\n    >({\n      store: documentEmbeddingStore,\n      keyEncoder: (key) => (options?.namespace ?? \"\") + insecureHash(key),\n      valueSerializer: (value) => encoder.encode(JSON.stringify(value)),\n      valueDeserializer: (serializedValue) =>\n        JSON.parse(decoder.decode(serializedValue)),\n    });\n    return new this({\n      underlyingEmbeddings,\n      documentEmbeddingStore: encoderBackedStore,\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkEA,IAAa,wBAAb,cAA2CA,uCAAW;CACpD,AAAU;CAEV,AAAU;CAEV,YAAYC,QAAqC;EAC/C,MAAM,OAAO;EACb,KAAK,uBAAuB,OAAO;EACnC,KAAK,yBAAyB,OAAO;CACtC;;;;;;;;;;;;;;;;CAiBD,MAAM,WAAWC,UAAqC;AACpD,SAAO,KAAK,qBAAqB,WAAW,SAAS;CACtD;;;;;;;;;;;CAYD,MAAM,eAAeC,WAA0C;EAC7D,MAAM,UAAU,MAAM,KAAK,uBAAuB,KAAK,UAAU;EACjE,MAAM,kBAAkB,CAAE;EAC1B,MAAM,mBAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,EACvC,KAAI,QAAQ,OAAO,QAAW;GAC5B,gBAAgB,KAAK,EAAE;GACvB,iBAAiB,KAAK,UAAU,GAAG;EACpC;AAEH,MAAI,iBAAiB,QAAQ;GAC3B,MAAM,iBAAiB,MAAM,KAAK,qBAAqB,eACrD,iBACD;GACD,MAAMC,gBAAsC,iBAAiB,IAC3D,CAAC,UAAU,MAAM,CAAC,UAAU,eAAe,EAAG,EAC/C;GACD,MAAM,KAAK,uBAAuB,KAAK,cAAc;AACrD,QAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,QAAQ,KAAK,GAC/C,QAAQ,gBAAgB,MAAM,eAAe;EAEhD;AACD,SAAO;CACR;;;;;;;;;CAUD,OAAO,eACLC,sBACAC,wBACAC,SAGA;EACA,MAAM,UAAU,IAAI;EACpB,MAAM,UAAU,IAAI;EACpB,MAAM,qBAAqB,IAAIC,kDAI7B;GACA,OAAO;GACP,YAAY,CAAC,SAAS,SAAS,aAAa,oDAAmB,IAAI;GACnE,iBAAiB,CAAC,UAAU,QAAQ,OAAO,KAAK,UAAU,MAAM,CAAC;GACjE,mBAAmB,CAAC,oBAClB,KAAK,MAAM,QAAQ,OAAO,gBAAgB,CAAC;EAC9C;AACD,SAAO,IAAI,KAAK;GACd;GACA,wBAAwB;EACzB;CACF;AACF"}