langchain 0.0.202 → 0.0.204-rc.0

package/dist/vectorstores/analyticdb.js

@@ -1,317 +1 @@
-import * as uuid from "uuid";
-import pg from "pg";
-import { from as copyFrom } from "pg-copy-streams";
-import { pipeline } from "node:stream/promises";
-import { Readable } from "node:stream";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-const _LANGCHAIN_DEFAULT_COLLECTION_NAME = "langchain_document";
-/**
- * Class that provides methods for creating and managing a collection of
- * documents in an AnalyticDB, adding documents or vectors to the
- * collection, performing similarity search on vectors, and creating an
- * instance of `AnalyticDBVectorStore` from texts or documents.
- */
-export class AnalyticDBVectorStore extends VectorStore {
-    _vectorstoreType() {
-        return "analyticdb";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "pool", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingDimension", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collectionName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "preDeleteCollection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isCreateCollection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        this.pool = new pg.Pool({
-            host: args.connectionOptions.host,
-            port: args.connectionOptions.port,
-            database: args.connectionOptions.database,
-            user: args.connectionOptions.user,
-            password: args.connectionOptions.password,
-        });
-        this.embeddingDimension = args.embeddingDimension;
-        this.collectionName =
-            args.collectionName || _LANGCHAIN_DEFAULT_COLLECTION_NAME;
-        this.preDeleteCollection = args.preDeleteCollection || false;
-    }
-    /**
-     * Closes all the clients in the pool and terminates the pool.
-     * @returns Promise that resolves when all clients are closed and the pool is terminated.
-     */
-    async end() {
-        return this.pool.end();
-    }
-    /**
-     * Creates a new table in the database if it does not already exist. The
-     * table is created with columns for id, embedding, document, and
-     * metadata. An index is also created on the embedding column if it does
-     * not already exist.
-     * @returns Promise that resolves when the table and index are created.
-     */
-    async createTableIfNotExists() {
-        if (!this.embeddingDimension) {
-            this.embeddingDimension = (await this.embeddings.embedQuery("test")).length;
-        }
-        const client = await this.pool.connect();
-        try {
-            await client.query("BEGIN");
-            // Create the table if it doesn't exist
-            await client.query(`
-        CREATE TABLE IF NOT EXISTS ${this.collectionName} (
-          id TEXT PRIMARY KEY DEFAULT NULL,
-          embedding REAL[],
-          document TEXT,
-          metadata JSON
-        );
-      `);
-            // Check if the index exists
-            const indexName = `${this.collectionName}_embedding_idx`;
-            const indexQuery = `
-        SELECT 1
-        FROM pg_indexes
-        WHERE indexname = '${indexName}';
-      `;
-            const result = await client.query(indexQuery);
-            // Create the index if it doesn't exist
-            if (result.rowCount === 0) {
-                const indexStatement = `
-          CREATE INDEX ${indexName}
-          ON ${this.collectionName} USING ann(embedding)
-          WITH (
-            "dim" = ${this.embeddingDimension},
-            "hnsw_m" = 100
-          );
-        `;
-                await client.query(indexStatement);
-            }
-            await client.query("COMMIT");
-        }
-        catch (err) {
-            await client.query("ROLLBACK");
-            throw err;
-        }
-        finally {
-            client.release();
-        }
-    }
-    /**
-     * Deletes the collection from the database if it exists.
-     * @returns Promise that resolves when the collection is deleted.
-     */
-    async deleteCollection() {
-        const dropStatement = `DROP TABLE IF EXISTS ${this.collectionName};`;
-        await this.pool.query(dropStatement);
-    }
-    /**
-     * Creates a new collection in the database. If `preDeleteCollection` is
-     * true, any existing collection with the same name is deleted before the
-     * new collection is created.
-     * @returns Promise that resolves when the collection is created.
-     */
-    async createCollection() {
-        if (this.preDeleteCollection) {
-            await this.deleteCollection();
-        }
-        await this.createTableIfNotExists();
-        this.isCreateCollection = true;
-    }
-    /**
-     * Adds an array of documents to the collection. The documents are first
-     * converted to vectors using the `embedDocuments` method of the
-     * `embeddings` instance.
-     * @param documents Array of Document instances to be added to the collection.
-     * @returns Promise that resolves when the documents are added.
-     */
-    async addDocuments(documents) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
-    }
-    /**
-     * Adds an array of vectors and corresponding documents to the collection.
-     * The vectors and documents are batch inserted into the database.
-     * @param vectors Array of vectors to be added to the collection.
-     * @param documents Array of Document instances corresponding to the vectors.
-     * @returns Promise that resolves when the vectors and documents are added.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        if (vectors.length !== documents.length) {
-            throw new Error(`Vectors and documents must have the same length`);
-        }
-        if (!this.embeddingDimension) {
-            this.embeddingDimension = (await this.embeddings.embedQuery("test")).length;
-        }
-        if (vectors[0].length !== this.embeddingDimension) {
-            throw new Error(`Vectors must have the same length as the number of dimensions (${this.embeddingDimension})`);
-        }
-        if (!this.isCreateCollection) {
-            await this.createCollection();
-        }
-        const client = await this.pool.connect();
-        try {
-            const chunkSize = 500;
-            const chunksTableData = [];
-            for (let i = 0; i < documents.length; i += 1) {
-                chunksTableData.push({
-                    id: uuid.v4(),
-                    embedding: vectors[i],
-                    document: documents[i].pageContent,
-                    metadata: documents[i].metadata,
-                });
-                // Execute the batch insert when the batch size is reached
-                if (chunksTableData.length === chunkSize) {
-                    const rs = new Readable();
-                    let currentIndex = 0;
-                    rs._read = function () {
-                        if (currentIndex === chunkSize) {
-                            rs.push(null);
-                        }
-                        else {
-                            const data = chunksTableData[currentIndex];
-                            rs.push(`${data.id}\t{${data.embedding.join(",")}}\t${data.document}\t${JSON.stringify(data.metadata)}\n`);
-                            currentIndex += 1;
-                        }
-                    };
-                    const ws = client.query(copyFrom(`COPY ${this.collectionName}(id, embedding, document, metadata) FROM STDIN`));
-                    await pipeline(rs, ws);
-                    // Clear the chunksTableData list for the next batch
-                    chunksTableData.length = 0;
-                }
-            }
-            // Insert any remaining records that didn't make up a full batch
-            if (chunksTableData.length > 0) {
-                const rs = new Readable();
-                let currentIndex = 0;
-                rs._read = function () {
-                    if (currentIndex === chunksTableData.length) {
-                        rs.push(null);
-                    }
-                    else {
-                        const data = chunksTableData[currentIndex];
-                        rs.push(`${data.id}\t{${data.embedding.join(",")}}\t${data.document}\t${JSON.stringify(data.metadata)}\n`);
-                        currentIndex += 1;
-                    }
-                };
-                const ws = client.query(copyFrom(`COPY ${this.collectionName}(id, embedding, document, metadata) FROM STDIN`));
-                await pipeline(rs, ws);
-            }
-        }
-        finally {
-            client.release();
-        }
-    }
-    /**
-     * Performs a similarity search on the vectors in the collection. The
-     * search is performed using the given query vector and returns the top k
-     * most similar vectors along with their corresponding documents and
-     * similarity scores.
-     * @param query Query vector for the similarity search.
-     * @param k Number of top similar vectors to return.
-     * @param filter Optional. Filter to apply on the metadata of the documents.
-     * @returns Promise that resolves to an array of tuples, each containing a Document instance and its similarity score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!this.isCreateCollection) {
-            await this.createCollection();
-        }
-        let filterCondition = "";
-        const filterEntries = filter ? Object.entries(filter) : [];
-        if (filterEntries.length > 0) {
-            const conditions = filterEntries.map((_, index) => `metadata->>$${2 * index + 3} = $${2 * index + 4}`);
-            filterCondition = `WHERE ${conditions.join(" AND ")}`;
-        }
-        const sqlQuery = `
-      SELECT *, l2_distance(embedding, $1::real[]) AS distance
-      FROM ${this.collectionName}
-      ${filterCondition}
-      ORDER BY embedding <-> $1
-      LIMIT $2;
-    `;
-        // Execute the query and fetch the results
-        const { rows } = await this.pool.query(sqlQuery, [
-            query,
-            k,
-            ...filterEntries.flatMap(([key, value]) => [key, value]),
-        ]);
-        const result = rows.map((row) => [
-            new Document({ pageContent: row.document, metadata: row.metadata }),
-            row.distance,
-        ]);
-        return result;
-    }
-    /**
-     * Creates an instance of `AnalyticDBVectorStore` from an array of texts
-     * and corresponding metadata. The texts are first converted to Document
-     * instances before being added to the collection.
-     * @param texts Array of texts to be added to the collection.
-     * @param metadatas Array or object of metadata corresponding to the texts.
-     * @param embeddings Embeddings instance used to convert the texts to vectors.
-     * @param dbConfig Configuration for the AnalyticDB.
-     * @returns Promise that resolves to an instance of `AnalyticDBVectorStore`.
-     */
-    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return AnalyticDBVectorStore.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Creates an instance of `AnalyticDBVectorStore` from an array of
-     * Document instances. The documents are added to the collection.
-     * @param docs Array of Document instances to be added to the collection.
-     * @param embeddings Embeddings instance used to convert the documents to vectors.
-     * @param dbConfig Configuration for the AnalyticDB.
-     * @returns Promise that resolves to an instance of `AnalyticDBVectorStore`.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Creates an instance of `AnalyticDBVectorStore` from an existing index
-     * in the database. A new collection is created in the database.
-     * @param embeddings Embeddings instance used to convert the documents to vectors.
-     * @param dbConfig Configuration for the AnalyticDB.
-     * @returns Promise that resolves to an instance of `AnalyticDBVectorStore`.
-     */
-    static async fromExistingIndex(embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.createCollection();
-        return instance;
-    }
-}
+export * from "@langchain/community/vectorstores/analyticdb";

package/dist/vectorstores/base.cjs

@@ -1,167 +1,17 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SaveableVectorStore = exports.VectorStore = exports.VectorStoreRetriever = void 0;
-const retriever_js_1 = require("../schema/retriever.cjs");
-const serializable_js_1 = require("../load/serializable.cjs");
-/**
- * Class for performing document retrieval from a VectorStore. Can perform
- * similarity search or maximal marginal relevance search.
- */
-class VectorStoreRetriever extends retriever_js_1.BaseRetriever {
-    static lc_name() {
-        return "VectorStoreRetriever";
-    }
-    get lc_namespace() {
-        return ["langchain", "retrievers", "base"];
-    }
-    _vectorstoreType() {
-        return this.vectorStore._vectorstoreType();
-    }
-    constructor(fields) {
-        super(fields);
-        Object.defineProperty(this, "vectorStore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "k", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 4
-        });
-        Object.defineProperty(this, "searchType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "similarity"
-        });
-        Object.defineProperty(this, "searchKwargs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "filter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.vectorStore = fields.vectorStore;
-        this.k = fields.k ?? this.k;
-        this.searchType = fields.searchType ?? this.searchType;
-        this.filter = fields.filter;
-        if (fields.searchType === "mmr") {
-            this.searchKwargs = fields.searchKwargs;
-        }
-    }
-    async _getRelevantDocuments(query, runManager) {
-        if (this.searchType === "mmr") {
-            if (typeof this.vectorStore.maxMarginalRelevanceSearch !== "function") {
-                throw new Error(`The vector store backing this retriever, ${this._vectorstoreType()} does not support max marginal relevance search.`);
-            }
-            return this.vectorStore.maxMarginalRelevanceSearch(query, {
-                k: this.k,
-                filter: this.filter,
-                ...this.searchKwargs,
-            }, runManager?.getChild("vectorstore"));
-        }
-        return this.vectorStore.similaritySearch(query, this.k, this.filter, runManager?.getChild("vectorstore"));
-    }
-    async addDocuments(documents, options) {
-        return this.vectorStore.addDocuments(documents, options);
-    }
-}
-exports.VectorStoreRetriever = VectorStoreRetriever;
-/**
- * Abstract class representing a store of vectors. Provides methods for
- * adding vectors and documents, deleting from the store, and searching
- * the store.
- */
-class VectorStore extends serializable_js_1.Serializable {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    constructor(embeddings, dbConfig) {
-        super(dbConfig);
-        Object.defineProperty(this, "lc_namespace", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: ["langchain", "vectorstores", this._vectorstoreType()]
-        });
-        Object.defineProperty(this, "embeddings", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.embeddings = embeddings;
-    }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async delete(_params) {
-        throw new Error("Not implemented.");
-    }
-    async similaritySearch(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
-    ) {
-        const results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
-        return results.map((result) => result[0]);
-    }
-    async similaritySearchWithScore(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
-    ) {
-        return this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
-    }
-    static fromTexts(_texts, _metadatas, _embeddings, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    _dbConfig) {
-        throw new Error("the Langchain vectorstore implementation you are using forgot to override this, please report a bug");
-    }
-    static fromDocuments(_docs, _embeddings, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    _dbConfig) {
-        throw new Error("the Langchain vectorstore implementation you are using forgot to override this, please report a bug");
-    }
-    asRetriever(kOrFields, filter, callbacks, tags, metadata, verbose) {
-        if (typeof kOrFields === "number") {
-            return new VectorStoreRetriever({
-                vectorStore: this,
-                k: kOrFields,
-                filter,
-                tags: [...(tags ?? []), this._vectorstoreType()],
-                metadata,
-                verbose,
-                callbacks,
-            });
-        }
-        else {
-            const params = {
-                vectorStore: this,
-                k: kOrFields?.k,
-                filter: kOrFields?.filter,
-                tags: [...(kOrFields?.tags ?? []), this._vectorstoreType()],
-                metadata: kOrFields?.metadata,
-                verbose: kOrFields?.verbose,
-                callbacks: kOrFields?.callbacks,
-                searchType: kOrFields?.searchType,
-            };
-            if (kOrFields?.searchType === "mmr") {
-                return new VectorStoreRetriever({
-                    ...params,
-                    searchKwargs: kOrFields.searchKwargs,
-                });
-            }
-            return new VectorStoreRetriever({ ...params });
-        }
-    }
-}
-exports.VectorStore = VectorStore;
-/**
- * Abstract class extending VectorStore with functionality for saving and
- * loading the vector store.
- */
-class SaveableVectorStore extends VectorStore {
-    static load(_directory, _embeddings) {
-        throw new Error("Not implemented");
-    }
-}
-exports.SaveableVectorStore = SaveableVectorStore;
+__exportStar(require("@langchain/core/vectorstores"), exports);

package/dist/vectorstores/base.d.ts

@@ -1,104 +1 @@
-import { Embeddings } from "../embeddings/base.js";
-import { Document } from "../document.js";
-import { BaseRetriever, BaseRetrieverInput } from "../schema/retriever.js";
-import { Serializable } from "../load/serializable.js";
-import { CallbackManagerForRetrieverRun, Callbacks } from "../callbacks/manager.js";
-/**
- * Type for options when adding a document to the VectorStore.
- */
-type AddDocumentOptions = Record<string, any>;
-/**
- * Type for options when performing a maximal marginal relevance search.
- */
-export type MaxMarginalRelevanceSearchOptions<FilterType> = {
-    k: number;
-    fetchK?: number;
-    lambda?: number;
-    filter?: FilterType;
-};
-/**
- * Type for options when performing a maximal marginal relevance search
- * with the VectorStoreRetriever.
- */
-export type VectorStoreRetrieverMMRSearchKwargs = {
-    fetchK?: number;
-    lambda?: number;
-};
-/**
- * Type for input when creating a VectorStoreRetriever instance.
- */
-export type VectorStoreRetrieverInput<V extends VectorStore> = BaseRetrieverInput & ({
-    vectorStore: V;
-    k?: number;
-    filter?: V["FilterType"];
-    searchType?: "similarity";
-} | {
-    vectorStore: V;
-    k?: number;
-    filter?: V["FilterType"];
-    searchType: "mmr";
-    searchKwargs?: VectorStoreRetrieverMMRSearchKwargs;
-});
-/**
- * Class for performing document retrieval from a VectorStore. Can perform
- * similarity search or maximal marginal relevance search.
- */
-export declare class VectorStoreRetriever<V extends VectorStore = VectorStore> extends BaseRetriever {
-    static lc_name(): string;
-    get lc_namespace(): string[];
-    vectorStore: V;
-    k: number;
-    searchType: string;
-    searchKwargs?: VectorStoreRetrieverMMRSearchKwargs;
-    filter?: V["FilterType"];
-    _vectorstoreType(): string;
-    constructor(fields: VectorStoreRetrieverInput<V>);
-    _getRelevantDocuments(query: string, runManager?: CallbackManagerForRetrieverRun): Promise<Document[]>;
-    addDocuments(documents: Document[], options?: AddDocumentOptions): Promise<string[] | void>;
-}
-/**
- * Abstract class representing a store of vectors. Provides methods for
- * adding vectors and documents, deleting from the store, and searching
- * the store.
- */
-export declare abstract class VectorStore extends Serializable {
-    FilterType: object | string;
-    lc_namespace: string[];
-    embeddings: Embeddings;
-    constructor(embeddings: Embeddings, dbConfig: Record<string, any>);
-    abstract _vectorstoreType(): string;
-    abstract addVectors(vectors: number[][], documents: Document[], options?: AddDocumentOptions): Promise<string[] | void>;
-    abstract addDocuments(documents: Document[], options?: AddDocumentOptions): Promise<string[] | void>;
-    delete(_params?: Record<string, any>): Promise<void>;
-    abstract similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document, number][]>;
-    similaritySearch(query: string, k?: number, filter?: this["FilterType"] | undefined, _callbacks?: Callbacks | undefined): Promise<Document[]>;
-    similaritySearchWithScore(query: string, k?: number, filter?: this["FilterType"] | undefined, _callbacks?: Callbacks | undefined): Promise<[Document, number][]>;
-    /**
-     * Return documents selected using the maximal marginal relevance.
-     * Maximal marginal relevance optimizes for similarity to the query AND diversity
-     * among selected documents.
-     *
-     * @param {string} query - Text to look up documents similar to.
-     * @param {number} options.k - Number of documents to return.
-     * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
-     * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
-     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
-     * @param {this["FilterType"]} options.filter - Optional filter
-     * @param _callbacks
-     *
-     * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
-     */
-    maxMarginalRelevanceSearch?(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>, _callbacks: Callbacks | undefined): Promise<Document[]>;
-    static fromTexts(_texts: string[], _metadatas: object[] | object, _embeddings: Embeddings, _dbConfig: Record<string, any>): Promise<VectorStore>;
-    static fromDocuments(_docs: Document[], _embeddings: Embeddings, _dbConfig: Record<string, any>): Promise<VectorStore>;
-    asRetriever(kOrFields?: number | Partial<VectorStoreRetrieverInput<this>>, filter?: this["FilterType"], callbacks?: Callbacks, tags?: string[], metadata?: Record<string, unknown>, verbose?: boolean): VectorStoreRetriever<this>;
-}
-/**
- * Abstract class extending VectorStore with functionality for saving and
- * loading the vector store.
- */
-export declare abstract class SaveableVectorStore extends VectorStore {
-    abstract save(directory: string): Promise<void>;
-    static load(_directory: string, _embeddings: Embeddings): Promise<SaveableVectorStore>;
-}
-export {};
+export * from "@langchain/core/vectorstores";