langchain 0.0.203 → 0.0.204-rc.0

package/dist/vectorstores/elasticsearch.js

@@ -1,274 +1 @@
-import * as uuid from "uuid";
-import { Document } from "../document.js";
-import { VectorStore } from "./base.js";
-/**
- * Class for interacting with an Elasticsearch database. It extends the
- * VectorStore base class and provides methods for adding documents and
- * vectors to the Elasticsearch database, performing similarity searches,
- * deleting documents, and more.
- */
-export class ElasticVectorSearch extends VectorStore {
-    _vectorstoreType() {
-        return "elasticsearch";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "engine", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "similarity", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "efConstruction", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "m", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "candidates", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.engine = args.vectorSearchOptions?.engine ?? "hnsw";
-        this.similarity = args.vectorSearchOptions?.similarity ?? "l2_norm";
-        this.m = args.vectorSearchOptions?.m ?? 16;
-        this.efConstruction = args.vectorSearchOptions?.efConstruction ?? 100;
-        this.candidates = args.vectorSearchOptions?.candidates ?? 200;
-        this.client = args.client.child({
-            headers: { "user-agent": "langchain-js-vs/0.0.1" },
-        });
-        this.indexName = args.indexName ?? "documents";
-    }
-    /**
-     * Method to add documents to the Elasticsearch database. It first
-     * converts the documents to vectors using the embeddings, then adds the
-     * vectors to the database.
-     * @param documents The documents to add to the database.
-     * @param options Optional parameter that can contain the IDs for the documents.
-     * @returns A promise that resolves with the IDs of the added documents.
-     */
-    async addDocuments(documents, options) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
-    }
-    /**
-     * Method to add vectors to the Elasticsearch database. It ensures the
-     * index exists, then adds the vectors and their corresponding documents
-     * to the database.
-     * @param vectors The vectors to add to the database.
-     * @param documents The documents corresponding to the vectors.
-     * @param options Optional parameter that can contain the IDs for the documents.
-     * @returns A promise that resolves with the IDs of the added documents.
-     */
-    async addVectors(vectors, documents, options) {
-        await this.ensureIndexExists(vectors[0].length, this.engine, this.similarity, this.efConstruction, this.m);
-        const documentIds = options?.ids ?? Array.from({ length: vectors.length }, () => uuid.v4());
-        const operations = vectors.flatMap((embedding, idx) => [
-            {
-                index: {
-                    _id: documentIds[idx],
-                    _index: this.indexName,
-                },
-            },
-            {
-                embedding,
-                metadata: documents[idx].metadata,
-                text: documents[idx].pageContent,
-            },
-        ]);
-        await this.client.bulk({ refresh: true, operations });
-        return documentIds;
-    }
-    /**
-     * Method to perform a similarity search in the Elasticsearch database
-     * using a vector. It returns the k most similar documents along with
-     * their similarity scores.
-     * @param query The query vector.
-     * @param k The number of most similar documents to return.
-     * @param filter Optional filter to apply to the search.
-     * @returns A promise that resolves with an array of tuples, where each tuple contains a Document and its similarity score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        const result = await this.client.search({
-            index: this.indexName,
-            size: k,
-            knn: {
-                field: "embedding",
-                query_vector: query,
-                filter: this.buildMetadataTerms(filter),
-                k,
-                num_candidates: this.candidates,
-            },
-        });
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return result.hits.hits.map((hit) => [
-            new Document({
-                pageContent: hit._source.text,
-                metadata: hit._source.metadata,
-            }),
-            hit._score,
-        ]);
-    }
-    /**
-     * Method to delete documents from the Elasticsearch database.
-     * @param params Object containing the IDs of the documents to delete.
-     * @returns A promise that resolves when the deletion is complete.
-     */
-    async delete(params) {
-        const operations = params.ids.map((id) => ({
-            delete: {
-                _id: id,
-                _index: this.indexName,
-            },
-        }));
-        await this.client.bulk({ refresh: true, operations });
-    }
-    /**
-     * Static method to create an ElasticVectorSearch instance from texts. It
-     * creates Document instances from the texts and their corresponding
-     * metadata, then calls the fromDocuments method to create the
-     * ElasticVectorSearch instance.
-     * @param texts The texts to create the ElasticVectorSearch instance from.
-     * @param metadatas The metadata corresponding to the texts.
-     * @param embeddings The embeddings to use for the documents.
-     * @param args The arguments to create the Elasticsearch client.
-     * @returns A promise that resolves with the created ElasticVectorSearch instance.
-     */
-    static fromTexts(texts, metadatas, embeddings, args) {
-        const documents = texts.map((text, idx) => {
-            const metadata = Array.isArray(metadatas) ? metadatas[idx] : metadatas;
-            return new Document({ pageContent: text, metadata });
-        });
-        return ElasticVectorSearch.fromDocuments(documents, embeddings, args);
-    }
-    /**
-     * Static method to create an ElasticVectorSearch instance from Document
-     * instances. It adds the documents to the Elasticsearch database, then
-     * returns the ElasticVectorSearch instance.
-     * @param docs The Document instances to create the ElasticVectorSearch instance from.
-     * @param embeddings The embeddings to use for the documents.
-     * @param dbConfig The configuration for the Elasticsearch database.
-     * @returns A promise that resolves with the created ElasticVectorSearch instance.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const store = new ElasticVectorSearch(embeddings, dbConfig);
-        await store.addDocuments(docs).then(() => store);
-        return store;
-    }
-    /**
-     * Static method to create an ElasticVectorSearch instance from an
-     * existing index in the Elasticsearch database. It checks if the index
-     * exists, then returns the ElasticVectorSearch instance if it does.
-     * @param embeddings The embeddings to use for the documents.
-     * @param dbConfig The configuration for the Elasticsearch database.
-     * @returns A promise that resolves with the created ElasticVectorSearch instance if the index exists, otherwise it throws an error.
-     */
-    static async fromExistingIndex(embeddings, dbConfig) {
-        const store = new ElasticVectorSearch(embeddings, dbConfig);
-        const exists = await store.doesIndexExist();
-        if (exists) {
-            return store;
-        }
-        throw new Error(`The index ${store.indexName} does not exist.`);
-    }
-    async ensureIndexExists(dimension, engine = "hnsw", similarity = "l2_norm", efConstruction = 100, m = 16) {
-        const request = {
-            index: this.indexName,
-            mappings: {
-                dynamic_templates: [
-                    {
-                        // map all metadata properties to be keyword
-                        "metadata.*": {
-                            match_mapping_type: "*",
-                            mapping: { type: "keyword" },
-                        },
-                    },
-                ],
-                properties: {
-                    text: { type: "text" },
-                    metadata: { type: "object" },
-                    embedding: {
-                        type: "dense_vector",
-                        dims: dimension,
-                        index: true,
-                        similarity,
-                        index_options: {
-                            type: engine,
-                            m,
-                            ef_construction: efConstruction,
-                        },
-                    },
-                },
-            },
-        };
-        const indexExists = await this.doesIndexExist();
-        if (indexExists)
-            return;
-        await this.client.indices.create(request);
-    }
-    buildMetadataTerms(filter
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    ) {
-        if (filter == null)
-            return [];
-        const result = [];
-        const filters = Array.isArray(filter)
-            ? filter
-            : Object.entries(filter).map(([key, value]) => ({
-                operator: "term",
-                field: key,
-                value,
-            }));
-        for (const condition of filters) {
-            result.push({
-                [condition.operator]: {
-                    [`metadata.${condition.field}`]: condition.value,
-                },
-            });
-        }
-        return result;
-    }
-    /**
-     * Method to check if an index exists in the Elasticsearch database.
-     * @returns A promise that resolves with a boolean indicating whether the index exists.
-     */
-    async doesIndexExist() {
-        return await this.client.indices.exists({ index: this.indexName });
-    }
-    /**
-     * Method to delete an index from the Elasticsearch database if it exists.
-     * @returns A promise that resolves when the deletion is complete.
-     */
-    async deleteIfExists() {
-        const indexExists = await this.doesIndexExist();
-        if (!indexExists)
-            return;
-        await this.client.indices.delete({ index: this.indexName });
-    }
-}
+export * from "@langchain/community/vectorstores/elasticsearch";

package/dist/vectorstores/faiss.cjs

@@ -10,383 +10,8 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FaissStore = void 0;
-const uuid = __importStar(require("uuid"));
-const base_js_1 = require("./base.cjs");
-const document_js_1 = require("../document.cjs");
-const in_memory_js_1 = require("../stores/doc/in_memory.cjs");
-/**
- * A class that wraps the FAISS (Facebook AI Similarity Search) vector
- * database for efficient similarity search and clustering of dense
- * vectors.
- */
-class FaissStore extends base_js_1.SaveableVectorStore {
-    _vectorstoreType() {
-        return "faiss";
-    }
-    getMapping() {
-        return this._mapping;
-    }
-    getDocstore() {
-        return this.docstore;
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "_index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_mapping", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "docstore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "args", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.args = args;
-        this._index = args.index;
-        this._mapping = args.mapping ?? {};
-        this.embeddings = embeddings;
-        this.docstore = args?.docstore ?? new in_memory_js_1.SynchronousInMemoryDocstore();
-    }
-    /**
-     * Adds an array of Document objects to the store.
-     * @param documents An array of Document objects.
-     * @returns A Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents, options) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
-    }
-    get index() {
-        if (!this._index) {
-            throw new Error("Vector store not initialised yet. Try calling `fromTexts`, `fromDocuments` or `fromIndex` first.");
-        }
-        return this._index;
-    }
-    set index(index) {
-        this._index = index;
-    }
-    /**
-     * Adds an array of vectors and their corresponding Document objects to
-     * the store.
-     * @param vectors An array of vectors.
-     * @param documents An array of Document objects corresponding to the vectors.
-     * @returns A Promise that resolves with an array of document IDs when the vectors and documents have been added.
-     */
-    async addVectors(vectors, documents, options) {
-        if (vectors.length === 0) {
-            return [];
-        }
-        if (vectors.length !== documents.length) {
-            throw new Error(`Vectors and documents must have the same length`);
-        }
-        const dv = vectors[0].length;
-        if (!this._index) {
-            const { IndexFlatL2 } = await FaissStore.importFaiss();
-            this._index = new IndexFlatL2(dv);
-        }
-        const d = this.index.getDimension();
-        if (dv !== d) {
-            throw new Error(`Vectors must have the same length as the number of dimensions (${d})`);
-        }
-        const docstoreSize = this.index.ntotal();
-        const documentIds = options?.ids ?? documents.map(() => uuid.v4());
-        for (let i = 0; i < vectors.length; i += 1) {
-            const documentId = documentIds[i];
-            const id = docstoreSize + i;
-            this.index.add(vectors[i]);
-            this._mapping[id] = documentId;
-            this.docstore.add({ [documentId]: documents[i] });
-        }
-        return documentIds;
-    }
-    /**
-     * Performs a similarity search in the vector store using a query vector
-     * and returns the top k results along with their scores.
-     * @param query A query vector.
-     * @param k The number of top results to return.
-     * @returns A Promise that resolves with an array of tuples, each containing a Document and its corresponding score.
-     */
-    async similaritySearchVectorWithScore(query, k) {
-        const d = this.index.getDimension();
-        if (query.length !== d) {
-            throw new Error(`Query vector must have the same length as the number of dimensions (${d})`);
-        }
-        if (k > this.index.ntotal()) {
-            const total = this.index.ntotal();
-            console.warn(`k (${k}) is greater than the number of elements in the index (${total}), setting k to ${total}`);
-            // eslint-disable-next-line no-param-reassign
-            k = total;
-        }
-        const result = this.index.search(query, k);
-        return result.labels.map((id, index) => {
-            const uuid = this._mapping[id];
-            return [this.docstore.search(uuid), result.distances[index]];
-        });
-    }
-    /**
-     * Saves the current state of the FaissStore to a specified directory.
-     * @param directory The directory to save the state to.
-     * @returns A Promise that resolves when the state has been saved.
-     */
-    async save(directory) {
-        const fs = await import("node:fs/promises");
-        const path = await import("node:path");
-        await fs.mkdir(directory, { recursive: true });
-        await Promise.all([
-            this.index.write(path.join(directory, "faiss.index")),
-            await fs.writeFile(path.join(directory, "docstore.json"), JSON.stringify([
-                Array.from(this.docstore._docs.entries()),
-                this._mapping,
-            ])),
-        ]);
-    }
-    /**
-     * Method to delete documents.
-     * @param params Object containing the IDs of the documents to delete.
-     * @returns A promise that resolves when the deletion is complete.
-     */
-    async delete(params) {
-        const documentIds = params.ids;
-        if (documentIds == null) {
-            throw new Error("No documentIds provided to delete.");
-        }
-        const mappings = new Map(Object.entries(this._mapping).map(([key, value]) => [
-            parseInt(key, 10),
-            value,
-        ]));
-        const reversedMappings = new Map(Array.from(mappings, (entry) => [entry[1], entry[0]]));
-        const missingIds = new Set(documentIds.filter((id) => !reversedMappings.has(id)));
-        if (missingIds.size > 0) {
-            throw new Error(`Some specified documentIds do not exist in the current store. DocumentIds not found: ${Array.from(missingIds).join(", ")}`);
-        }
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const indexIdToDelete = documentIds.map((id) => reversedMappings.get(id));
-        // remove from index
-        this.index.removeIds(indexIdToDelete);
-        // remove from docstore
-        documentIds.forEach((id) => {
-            this.docstore._docs.delete(id);
-        });
-        // remove from mappings
-        indexIdToDelete.forEach((id) => {
-            mappings.delete(id);
-        });
-        this._mapping = { ...Array.from(mappings.values()) };
-    }
-    /**
-     * Merges the current FaissStore with another FaissStore.
-     * @param targetIndex The FaissStore to merge with.
-     * @returns A Promise that resolves with an array of document IDs when the merge is complete.
-     */
-    async mergeFrom(targetIndex) {
-        const targetIndexDimensions = targetIndex.index.getDimension();
-        if (!this._index) {
-            const { IndexFlatL2 } = await FaissStore.importFaiss();
-            this._index = new IndexFlatL2(targetIndexDimensions);
-        }
-        const d = this.index.getDimension();
-        if (targetIndexDimensions !== d) {
-            throw new Error("Cannot merge indexes with different dimensions.");
-        }
-        const targetMapping = targetIndex.getMapping();
-        const targetDocstore = targetIndex.getDocstore();
-        const targetSize = targetIndex.index.ntotal();
-        const documentIds = [];
-        const currentDocstoreSize = this.index.ntotal();
-        for (let i = 0; i < targetSize; i += 1) {
-            const targetId = targetMapping[i];
-            documentIds.push(targetId);
-            const targetDocument = targetDocstore.search(targetId);
-            const id = currentDocstoreSize + i;
-            this._mapping[id] = targetId;
-            this.docstore.add({ [targetId]: targetDocument });
-        }
-        this.index.mergeFrom(targetIndex.index);
-        return documentIds;
-    }
-    /**
-     * Loads a FaissStore from a specified directory.
-     * @param directory The directory to load the FaissStore from.
-     * @param embeddings An Embeddings object.
-     * @returns A Promise that resolves with a new FaissStore instance.
-     */
-    static async load(directory, embeddings) {
-        const fs = await import("node:fs/promises");
-        const path = await import("node:path");
-        const readStore = (directory) => fs
-            .readFile(path.join(directory, "docstore.json"), "utf8")
-            .then(JSON.parse);
-        const readIndex = async (directory) => {
-            const { IndexFlatL2 } = await this.importFaiss();
-            return IndexFlatL2.read(path.join(directory, "faiss.index"));
-        };
-        const [[docstoreFiles, mapping], index] = await Promise.all([
-            readStore(directory),
-            readIndex(directory),
-        ]);
-        const docstore = new in_memory_js_1.SynchronousInMemoryDocstore(new Map(docstoreFiles));
-        return new this(embeddings, { docstore, index, mapping });
-    }
-    static async loadFromPython(directory, embeddings) {
-        const fs = await import("node:fs/promises");
-        const path = await import("node:path");
-        const { Parser, NameRegistry } = await this.importPickleparser();
-        class PyDocument extends Map {
-            toDocument() {
-                return new document_js_1.Document({
-                    pageContent: this.get("page_content"),
-                    metadata: this.get("metadata"),
-                });
-            }
-        }
-        class PyInMemoryDocstore {
-            constructor() {
-                Object.defineProperty(this, "_dict", {
-                    enumerable: true,
-                    configurable: true,
-                    writable: true,
-                    value: void 0
-                });
-            }
-            toInMemoryDocstore() {
-                const s = new in_memory_js_1.SynchronousInMemoryDocstore();
-                for (const [key, value] of Object.entries(this._dict)) {
-                    s._docs.set(key, value.toDocument());
-                }
-                return s;
-            }
-        }
-        const readStore = async (directory) => {
-            const pkl = await fs.readFile(path.join(directory, "index.pkl"), "binary");
-            const buffer = Buffer.from(pkl, "binary");
-            const registry = new NameRegistry()
-                .register("langchain.docstore.in_memory", "InMemoryDocstore", PyInMemoryDocstore)
-                .register("langchain.schema", "Document", PyDocument)
-                .register("langchain.docstore.document", "Document", PyDocument)
-                .register("langchain.schema.document", "Document", PyDocument)
-                .register("pathlib", "WindowsPath", (...args) => args.join("\\"))
-                .register("pathlib", "PosixPath", (...args) => args.join("/"));
-            const pickleparser = new Parser({
-                nameResolver: registry,
-            });
-            const [rawStore, mapping] = pickleparser.parse(buffer);
-            const store = rawStore.toInMemoryDocstore();
-            return { store, mapping };
-        };
-        const readIndex = async (directory) => {
-            const { IndexFlatL2 } = await this.importFaiss();
-            return IndexFlatL2.read(path.join(directory, "index.faiss"));
-        };
-        const [store, index] = await Promise.all([
-            readStore(directory),
-            readIndex(directory),
-        ]);
-        return new this(embeddings, {
-            docstore: store.store,
-            index,
-            mapping: store.mapping,
-        });
-    }
-    /**
-     * Creates a new FaissStore from an array of texts, their corresponding
-     * metadata, and an Embeddings object.
-     * @param texts An array of texts.
-     * @param metadatas An array of metadata corresponding to the texts, or a single metadata object to be used for all texts.
-     * @param embeddings An Embeddings object.
-     * @param dbConfig An optional configuration object for the document store.
-     * @returns A Promise that resolves with a new FaissStore instance.
-     */
-    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new document_js_1.Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return this.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Creates a new FaissStore from an array of Document objects and an
-     * Embeddings object.
-     * @param docs An array of Document objects.
-     * @param embeddings An Embeddings object.
-     * @param dbConfig An optional configuration object for the document store.
-     * @returns A Promise that resolves with a new FaissStore instance.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const args = {
-            docstore: dbConfig?.docstore,
-        };
-        const instance = new this(embeddings, args);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Creates a new FaissStore from an existing FaissStore and an Embeddings
-     * object.
-     * @param targetIndex An existing FaissStore.
-     * @param embeddings An Embeddings object.
-     * @param dbConfig An optional configuration object for the document store.
-     * @returns A Promise that resolves with a new FaissStore instance.
-     */
-    static async fromIndex(targetIndex, embeddings, dbConfig) {
-        const args = {
-            docstore: dbConfig?.docstore,
-        };
-        const instance = new this(embeddings, args);
-        await instance.mergeFrom(targetIndex);
-        return instance;
-    }
-    static async importFaiss() {
-        try {
-            const { default: { IndexFlatL2 }, } = await import("faiss-node");
-            return { IndexFlatL2 };
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (err) {
-            throw new Error(`Could not import faiss-node. Please install faiss-node as a dependency with, e.g. \`npm install -S faiss-node\`.\n\nError: ${err?.message}`);
-        }
-    }
-    static async importPickleparser() {
-        try {
-            const { default: { Parser, NameRegistry }, } = await import("pickleparser");
-            return { Parser, NameRegistry };
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (err) {
-            throw new Error(`Could not import pickleparser. Please install pickleparser as a dependency with, e.g. \`npm install -S pickleparser\`.\n\nError: ${err?.message}`);
-        }
-    }
-}
-exports.FaissStore = FaissStore;
+__exportStar(require("@langchain/community/vectorstores/faiss"), exports);