langchain 0.0.203 → 0.0.204-rc.0

package/dist/vectorstores/qdrant.js

@@ -1,193 +1 @@
-import { QdrantClient } from "@qdrant/js-client-rest";
-import { v4 as uuid } from "uuid";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-import { getEnvironmentVariable } from "../util/env.js";
-/**
- * Class that extends the `VectorStore` base class to interact with a
- * Qdrant database. It includes methods for adding documents and vectors
- * to the Qdrant database, searching for similar vectors, and ensuring the
- * existence of a collection in the database.
- */
-export class QdrantVectorStore extends VectorStore {
-    get lc_secrets() {
-        return {
-            apiKey: "QDRANT_API_KEY",
-            url: "QDRANT_URL",
-        };
-    }
-    _vectorstoreType() {
-        return "qdrant";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collectionName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collectionConfig", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const url = args.url ?? getEnvironmentVariable("QDRANT_URL");
-        const apiKey = args.apiKey ?? getEnvironmentVariable("QDRANT_API_KEY");
-        if (!args.client && !url) {
-            throw new Error("Qdrant client or url address must be set.");
-        }
-        this.client =
-            args.client ||
-                new QdrantClient({
-                    url,
-                    apiKey,
-                });
-        this.collectionName = args.collectionName ?? "documents";
-        this.collectionConfig = args.collectionConfig;
-    }
-    /**
-     * Method to add documents to the Qdrant database. It generates vectors
-     * from the documents using the `Embeddings` instance and then adds the
-     * vectors to the database.
-     * @param documents Array of `Document` instances to be added to the Qdrant database.
-     * @returns Promise that resolves when the documents have been added to the database.
-     */
-    async addDocuments(documents) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        await this.addVectors(await this.embeddings.embedDocuments(texts), documents);
-    }
-    /**
-     * Method to add vectors to the Qdrant database. Each vector is associated
-     * with a document, which is stored as the payload for a point in the
-     * database.
-     * @param vectors Array of vectors to be added to the Qdrant database.
-     * @param documents Array of `Document` instances associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added to the database.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        await this.ensureCollection();
-        const points = vectors.map((embedding, idx) => ({
-            id: uuid(),
-            vector: embedding,
-            payload: {
-                content: documents[idx].pageContent,
-                metadata: documents[idx].metadata,
-            },
-        }));
-        try {
-            await this.client.upsert(this.collectionName, {
-                wait: true,
-                points,
-            });
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (e) {
-            const error = new Error(`${e?.status ?? "Undefined error code"} ${e?.message}: ${e?.data?.status?.error}`);
-            throw error;
-        }
-    }
-    /**
-     * Method to search for vectors in the Qdrant database that are similar to
-     * a given query vector. The search results include the score and payload
-     * (metadata and content) for each similar vector.
-     * @param query Query vector to search for similar vectors in the Qdrant database.
-     * @param k Optional number of similar vectors to return. If not specified, all similar vectors are returned.
-     * @param filter Optional filter to apply to the search results.
-     * @returns Promise that resolves with an array of tuples, where each tuple includes a `Document` instance and a score for a similar vector.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!query) {
-            return [];
-        }
-        await this.ensureCollection();
-        const results = await this.client.search(this.collectionName, {
-            vector: query,
-            limit: k,
-            filter,
-        });
-        const result = results.map((res) => [
-            new Document({
-                metadata: res.payload.metadata,
-                pageContent: res.payload.content,
-            }),
-            res.score,
-        ]);
-        return result;
-    }
-    /**
-     * Method to ensure the existence of a collection in the Qdrant database.
-     * If the collection does not exist, it is created.
-     * @returns Promise that resolves when the existence of the collection has been ensured.
-     */
-    async ensureCollection() {
-        const response = await this.client.getCollections();
-        const collectionNames = response.collections.map((collection) => collection.name);
-        if (!collectionNames.includes(this.collectionName)) {
-            const collectionConfig = this.collectionConfig ?? {
-                vectors: {
-                    size: (await this.embeddings.embedQuery("test")).length,
-                    distance: "Cosine",
-                },
-            };
-            await this.client.createCollection(this.collectionName, collectionConfig);
-        }
-    }
-    /**
-     * Static method to create a `QdrantVectorStore` instance from texts. Each
-     * text is associated with metadata and converted to a `Document`
-     * instance, which is then added to the Qdrant database.
-     * @param texts Array of texts to be converted to `Document` instances and added to the Qdrant database.
-     * @param metadatas Array or single object of metadata to be associated with the texts.
-     * @param embeddings `Embeddings` instance used to generate vectors from the texts.
-     * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database.
-     * @returns Promise that resolves with a new `QdrantVectorStore` instance.
-     */
-    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return QdrantVectorStore.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Static method to create a `QdrantVectorStore` instance from `Document`
-     * instances. The documents are added to the Qdrant database.
-     * @param docs Array of `Document` instances to be added to the Qdrant database.
-     * @param embeddings `Embeddings` instance used to generate vectors from the documents.
-     * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database.
-     * @returns Promise that resolves with a new `QdrantVectorStore` instance.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to create a `QdrantVectorStore` instance from an existing
-     * collection in the Qdrant database.
-     * @param embeddings `Embeddings` instance used to generate vectors from the documents in the collection.
-     * @param dbConfig `QdrantLibArgs` instance specifying the configuration for the Qdrant database.
-     * @returns Promise that resolves with a new `QdrantVectorStore` instance.
-     */
-    static async fromExistingCollection(embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.ensureCollection();
-        return instance;
-    }
-}
+export * from "@langchain/community/vectorstores/qdrant";

package/dist/vectorstores/redis.cjs

@@ -1,335 +1,17 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RedisVectorStore = void 0;
-const redis_1 = require("redis");
-const base_js_1 = require("./base.cjs");
-const document_js_1 = require("../document.cjs");
-/**
- * Class representing a RedisVectorStore. It extends the VectorStore class
- * and includes methods for adding documents and vectors, performing
- * similarity searches, managing the index, and more.
- */
-class RedisVectorStore extends base_js_1.VectorStore {
-    _vectorstoreType() {
-        return "redis";
-    }
-    constructor(embeddings, _dbConfig) {
-        super(embeddings, _dbConfig);
-        Object.defineProperty(this, "redisClient", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexOptions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "createIndexOptions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "keyPrefix", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "contentKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "metadataKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "vectorKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "filter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.redisClient = _dbConfig.redisClient;
-        this.indexName = _dbConfig.indexName;
-        this.indexOptions = _dbConfig.indexOptions ?? {
-            ALGORITHM: redis_1.VectorAlgorithms.HNSW,
-            DISTANCE_METRIC: "COSINE",
-        };
-        this.keyPrefix = _dbConfig.keyPrefix ?? `doc:${this.indexName}:`;
-        this.contentKey = _dbConfig.contentKey ?? "content";
-        this.metadataKey = _dbConfig.metadataKey ?? "metadata";
-        this.vectorKey = _dbConfig.vectorKey ?? "content_vector";
-        this.filter = _dbConfig.filter;
-        this.createIndexOptions = {
-            ON: "HASH",
-            PREFIX: this.keyPrefix,
-            ..._dbConfig.createIndexOptions,
-        };
-    }
-    /**
-     * Method for adding documents to the RedisVectorStore. It first converts
-     * the documents to texts and then adds them as vectors.
-     * @param documents The documents to add.
-     * @param options Optional parameters for adding the documents.
-     * @returns A promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents, options) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
-    }
-    /**
-     * Method for adding vectors to the RedisVectorStore. It checks if the
-     * index exists and creates it if it doesn't, then adds the vectors in
-     * batches.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @param keys Optional keys for the vectors.
-     * @param batchSize The size of the batches in which to add the vectors. Defaults to 1000.
-     * @returns A promise that resolves when the vectors have been added.
-     */
-    async addVectors(vectors, documents, { keys, batchSize = 1000 } = {}) {
-        if (!vectors.length || !vectors[0].length) {
-            throw new Error("No vectors provided");
-        }
-        // check if the index exists and create it if it doesn't
-        await this.createIndex(vectors[0].length);
-        const info = await this.redisClient.ft.info(this.indexName);
-        const lastKeyCount = parseInt(info.numDocs, 10) || 0;
-        const multi = this.redisClient.multi();
-        vectors.map(async (vector, idx) => {
-            const key = keys && keys.length
-                ? keys[idx]
-                : `${this.keyPrefix}${idx + lastKeyCount}`;
-            const metadata = documents[idx] && documents[idx].metadata
-                ? documents[idx].metadata
-                : {};
-            multi.hSet(key, {
-                [this.vectorKey]: this.getFloat32Buffer(vector),
-                [this.contentKey]: documents[idx].pageContent,
-                [this.metadataKey]: this.escapeSpecialChars(JSON.stringify(metadata)),
-            });
-            // write batch
-            if (idx % batchSize === 0) {
-                await multi.exec();
-            }
-        });
-        // insert final batch
-        await multi.exec();
-    }
-    /**
-     * Method for performing a similarity search in the RedisVectorStore. It
-     * returns the documents and their scores.
-     * @param query The query vector.
-     * @param k The number of nearest neighbors to return.
-     * @param filter Optional filter to apply to the search.
-     * @returns A promise that resolves to an array of documents and their scores.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (filter && this.filter) {
-            throw new Error("cannot provide both `filter` and `this.filter`");
-        }
-        const _filter = filter ?? this.filter;
-        const results = await this.redisClient.ft.search(this.indexName, ...this.buildQuery(query, k, _filter));
-        const result = [];
-        if (results.total) {
-            for (const res of results.documents) {
-                if (res.value) {
-                    const document = res.value;
-                    if (document.vector_score) {
-                        result.push([
-                            new document_js_1.Document({
-                                pageContent: document[this.contentKey],
-                                metadata: JSON.parse(this.unEscapeSpecialChars(document.metadata)),
-                            }),
-                            Number(document.vector_score),
-                        ]);
-                    }
-                }
-            }
-        }
-        return result;
-    }
-    /**
-     * Static method for creating a new instance of RedisVectorStore from
-     * texts. It creates documents from the texts and metadata, then adds them
-     * to the RedisVectorStore.
-     * @param texts The texts to add.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param dbConfig The configuration for the RedisVectorStore.
-     * @returns A promise that resolves to a new instance of RedisVectorStore.
-     */
-    static fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new document_js_1.Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return RedisVectorStore.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Static method for creating a new instance of RedisVectorStore from
-     * documents. It adds the documents to the RedisVectorStore.
-     * @param docs The documents to add.
-     * @param embeddings The embeddings to use.
-     * @param dbConfig The configuration for the RedisVectorStore.
-     * @returns A promise that resolves to a new instance of RedisVectorStore.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Method for checking if an index exists in the RedisVectorStore.
-     * @returns A promise that resolves to a boolean indicating whether the index exists.
-     */
-    async checkIndexExists() {
-        try {
-            await this.redisClient.ft.info(this.indexName);
-        }
-        catch (err) {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            if (err?.message.includes("unknown command")) {
-                throw new Error("Failed to run FT.INFO command. Please ensure that you are running a RediSearch-capable Redis instance: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/redis#setup");
-            }
-            // index doesn't exist
-            return false;
-        }
-        return true;
-    }
-    /**
-     * Method for creating an index in the RedisVectorStore. If the index
-     * already exists, it does nothing.
-     * @param dimensions The dimensions of the index
-     * @returns A promise that resolves when the index has been created.
-     */
-    async createIndex(dimensions = 1536) {
-        if (await this.checkIndexExists()) {
-            return;
-        }
-        const schema = {
-            [this.vectorKey]: {
-                type: redis_1.SchemaFieldTypes.VECTOR,
-                TYPE: "FLOAT32",
-                DIM: dimensions,
-                ...this.indexOptions,
-            },
-            [this.contentKey]: redis_1.SchemaFieldTypes.TEXT,
-            [this.metadataKey]: redis_1.SchemaFieldTypes.TEXT,
-        };
-        await this.redisClient.ft.create(this.indexName, schema, this.createIndexOptions);
-    }
-    /**
-     * Method for dropping an index from the RedisVectorStore.
-     * @param deleteDocuments Optional boolean indicating whether to drop the associated documents.
-     * @returns A promise that resolves to a boolean indicating whether the index was dropped.
-     */
-    async dropIndex(deleteDocuments) {
-        try {
-            const options = deleteDocuments ? { DD: deleteDocuments } : undefined;
-            await this.redisClient.ft.dropIndex(this.indexName, options);
-            return true;
-        }
-        catch (err) {
-            return false;
-        }
-    }
-    /**
-     * Deletes vectors from the vector store.
-     * @param params The parameters for deleting vectors.
-     * @returns A promise that resolves when the vectors have been deleted.
-     */
-    async delete(params) {
-        if (params.deleteAll) {
-            await this.dropIndex(true);
-        }
-        else {
-            throw new Error(`Invalid parameters passed to "delete".`);
-        }
-    }
-    buildQuery(query, k, filter) {
-        const vectorScoreField = "vector_score";
-        let hybridFields = "*";
-        // if a filter is set, modify the hybrid query
-        if (filter && filter.length) {
-            // `filter` is a list of strings, then it's applied using the OR operator in the metadata key
-            // for example: filter = ['foo', 'bar'] => this will filter all metadata containing either 'foo' OR 'bar'
-            hybridFields = `@${this.metadataKey}:(${this.prepareFilter(filter)})`;
-        }
-        const baseQuery = `${hybridFields} => [KNN ${k} @${this.vectorKey} $vector AS ${vectorScoreField}]`;
-        const returnFields = [this.metadataKey, this.contentKey, vectorScoreField];
-        const options = {
-            PARAMS: {
-                vector: this.getFloat32Buffer(query),
-            },
-            RETURN: returnFields,
-            SORTBY: vectorScoreField,
-            DIALECT: 2,
-            LIMIT: {
-                from: 0,
-                size: k,
-            },
-        };
-        return [baseQuery, options];
-    }
-    prepareFilter(filter) {
-        return filter.map(this.escapeSpecialChars).join("|");
-    }
-    /**
-     * Escapes all '-' characters.
-     * RediSearch considers '-' as a negative operator, hence we need
-     * to escape it
-     * @see https://redis.io/docs/stack/search/reference/query_syntax
-     *
-     * @param str
-     * @returns
-     */
-    escapeSpecialChars(str) {
-        return str.replaceAll("-", "\\-");
-    }
-    /**
-     * Unescapes all '-' characters, returning the original string
-     *
-     * @param str
-     * @returns
-     */
-    unEscapeSpecialChars(str) {
-        return str.replaceAll("\\-", "-");
-    }
-    /**
-     * Converts the vector to the buffer Redis needs to
-     * correctly store an embedding
-     *
-     * @param vector
-     * @returns Buffer
-     */
-    getFloat32Buffer(vector) {
-        return Buffer.from(new Float32Array(vector).buffer);
-    }
-}
-exports.RedisVectorStore = RedisVectorStore;
+__exportStar(require("@langchain/community/vectorstores/redis"), exports);