langchain 0.0.197 → 0.0.199

package/dist/vectorstores/clickhouse.js

@@ -0,0 +1,259 @@
+import * as uuid from "uuid";
+import { createClient } from "@clickhouse/client";
+import { format } from "mysql2";
+import { VectorStore } from "./base.js";
+import { Document } from "../document.js";
+/**
+ * Class for interacting with the ClickHouse database. It extends the
+ * VectorStore class and provides methods for adding vectors and
+ * documents, searching for similar vectors, and creating instances from
+ * texts or documents.
+ */
+export class ClickHouseStore extends VectorStore {
+    _vectorstoreType() {
+        return "clickhouse";
+    }
+    constructor(embeddings, args) {
+        super(embeddings, args);
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexType", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexParam", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexQueryParams", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "columnMap", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "database", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "table", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isInitialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.indexType = args.indexType || "annoy";
+        this.indexParam = args.indexParam || { L2Distance: 100 };
+        this.indexQueryParams = args.indexQueryParams || {};
+        this.columnMap = args.columnMap || {
+            id: "id",
+            document: "document",
+            embedding: "embedding",
+            metadata: "metadata",
+            uuid: "uuid",
+        };
+        this.database = args.database || "default";
+        this.table = args.table || "vector_table";
+        this.client = createClient({
+            host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
+            username: args.username,
+            password: args.password,
+            session_id: uuid.v4(),
+        });
+    }
+    /**
+     * Method to add vectors to the ClickHouse database.
+     * @param vectors The vectors to add.
+     * @param documents The documents associated with the vectors.
+     * @returns Promise that resolves when the vectors have been added.
+     */
+    async addVectors(vectors, documents) {
+        if (vectors.length === 0) {
+            return;
+        }
+        if (!this.isInitialized) {
+            await this.initialize(vectors[0].length);
+        }
+        const queryStr = this.buildInsertQuery(vectors, documents);
+        await this.client.exec({ query: queryStr });
+    }
+    /**
+     * Method to add documents to the ClickHouse database.
+     * @param documents The documents to add.
+     * @returns Promise that resolves when the documents have been added.
+     */
+    async addDocuments(documents) {
+        return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
+    }
+    /**
+     * Method to search for vectors that are similar to a given query vector.
+     * @param query The query vector.
+     * @param k The number of similar vectors to return.
+     * @param filter Optional filter for the search results.
+     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
+     */
+    async similaritySearchVectorWithScore(query, k, filter) {
+        if (!this.isInitialized) {
+            await this.initialize(query.length);
+        }
+        const queryStr = this.buildSearchQuery(query, k, filter);
+        const queryResultSet = await this.client.query({ query: queryStr });
+        const queryResult = await queryResultSet.json();
+        const result = queryResult.data.map((item) => [
+            new Document({ pageContent: item.document, metadata: item.metadata }),
+            item.dist,
+        ]);
+        return result;
+    }
+    /**
+     * Static method to create an instance of ClickHouseStore from texts.
+     * @param texts The texts to use.
+     * @param metadatas The metadata associated with the texts.
+     * @param embeddings The embeddings to use.
+     * @param args The arguments for the ClickHouseStore.
+     * @returns Promise that resolves with a new instance of ClickHouseStore.
+     */
+    static async fromTexts(texts, metadatas, embeddings, args) {
+        const docs = [];
+        for (let i = 0; i < texts.length; i += 1) {
+            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
+            const newDoc = new Document({
+                pageContent: texts[i],
+                metadata,
+            });
+            docs.push(newDoc);
+        }
+        return ClickHouseStore.fromDocuments(docs, embeddings, args);
+    }
+    /**
+     * Static method to create an instance of ClickHouseStore from documents.
+     * @param docs The documents to use.
+     * @param embeddings The embeddings to use.
+     * @param args The arguments for the ClickHouseStore.
+     * @returns Promise that resolves with a new instance of ClickHouseStore.
+     */
+    static async fromDocuments(docs, embeddings, args) {
+        const instance = new this(embeddings, args);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+    /**
+     * Static method to create an instance of ClickHouseStore from an existing
+     * index.
+     * @param embeddings The embeddings to use.
+     * @param args The arguments for the ClickHouseStore.
+     * @returns Promise that resolves with a new instance of ClickHouseStore.
+     */
+    static async fromExistingIndex(embeddings, args) {
+        const instance = new this(embeddings, args);
+        await instance.initialize();
+        return instance;
+    }
+    /**
+     * Method to initialize the ClickHouse database.
+     * @param dimension Optional dimension of the vectors.
+     * @returns Promise that resolves when the database has been initialized.
+     */
+    async initialize(dimension) {
+        const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
+        const indexParamStr = this.indexParam
+            ? Object.entries(this.indexParam)
+                .map(([key, value]) => `'${key}', ${value}`)
+                .join(", ")
+            : "";
+        const query = `
+    CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
+      ${this.columnMap.id} Nullable(String),
+      ${this.columnMap.document} Nullable(String),
+      ${this.columnMap.embedding} Array(Float32),
+      ${this.columnMap.metadata} JSON,
+      ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),
+      CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},
+      INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000
+    ) ENGINE =  MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;
+        await this.client.exec({
+            query,
+            clickhouse_settings: {
+                allow_experimental_object_type: 1,
+                allow_experimental_annoy_index: 1,
+            },
+        });
+        this.isInitialized = true;
+    }
+    /**
+     * Method to build an SQL query for inserting vectors and documents into
+     * the ClickHouse database.
+     * @param vectors The vectors to insert.
+     * @param documents The documents to insert.
+     * @returns The SQL query string.
+     */
+    buildInsertQuery(vectors, documents) {
+        const columnsStr = Object.values(Object.fromEntries(Object.entries(this.columnMap).filter(([key]) => key !== this.columnMap.uuid))).join(", ");
+        const placeholders = vectors.map(() => "(?, ?, ?, ?)").join(", ");
+        const values = [];
+        for (let i = 0; i < vectors.length; i += 1) {
+            const vector = vectors[i];
+            const document = documents[i];
+            values.push(uuid.v4(), this.escapeString(document.pageContent), JSON.stringify(vector), JSON.stringify(document.metadata));
+        }
+        const insertQueryStr = `
+      INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr}) 
+      VALUES ${placeholders}
+    `;
+        const insertQuery = format(insertQueryStr, values);
+        return insertQuery;
+    }
+    escapeString(str) {
+        return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+    }
+    /**
+     * Method to build an SQL query for searching for similar vectors in the
+     * ClickHouse database.
+     * @param query The query vector.
+     * @param k The number of similar vectors to return.
+     * @param filter Optional filter for the search results.
+     * @returns The SQL query string.
+     */
+    buildSearchQuery(query, k, filter) {
+        const order = "ASC";
+        const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
+        const placeholders = query.map(() => "?").join(", ");
+        const settingStrings = [];
+        if (this.indexQueryParams) {
+            for (const [key, value] of Object.entries(this.indexQueryParams)) {
+                settingStrings.push(`SETTING ${key}=${value}`);
+            }
+        }
+        const searchQueryStr = `
+      SELECT ${this.columnMap.document} AS document, ${this.columnMap.metadata} AS metadata, dist
+      FROM ${this.database}.${this.table}
+      ${whereStr}
+      ORDER BY L2Distance(${this.columnMap.embedding}, [${placeholders}]) AS dist ${order}
+      LIMIT ${k} ${settingStrings.join(" ")}
+    `;
+        // Format the query with actual values
+        const searchQuery = format(searchQueryStr, query);
+        return searchQuery;
+    }
+}

package/dist/vectorstores/elasticsearch.cjs

@@ -258,12 +258,25 @@ class ElasticVectorSearch extends base_js_1.VectorStore {
             return;
         await this.client.indices.create(request);
     }
-    buildMetadataTerms(filter) {
+    buildMetadataTerms(filter
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ) {
         if (filter == null)
             return [];
         const result = [];
-        for (const [key, value] of Object.entries(filter)) {
-            result.push({ term: { [`metadata.${key}`]: value } });
+        const filters = Array.isArray(filter)
+            ? filter
+            : Object.entries(filter).map(([key, value]) => ({
+                operator: "term",
+                field: key,
+                value,
+            }));
+        for (const condition of filters) {
+            result.push({
+                [condition.operator]: {
+                    [`metadata.${condition.field}`]: condition.value,
+                },
+            });
         }
         return result;
     }

package/dist/vectorstores/elasticsearch.d.ts

@@ -33,7 +33,11 @@ export interface ElasticClientArgs {
 /**
  * Type representing a filter object in Elasticsearch.
  */
-type ElasticFilter = object;
+type ElasticFilter = object | {
+    field: string;
+    operator: string;
+    value: any;
+}[];
 /**
  * Class for interacting with an Elasticsearch database. It extends the
  * VectorStore base class and provides methods for adding documents and
@@ -83,7 +87,7 @@ export declare class ElasticVectorSearch extends VectorStore {
      * @param filter Optional filter to apply to the search.
      * @returns A promise that resolves with an array of tuples, where each tuple contains a Document and its similarity score.
      */
-    similaritySearchVectorWithScore(query: number[], k: number, filter?: ElasticFilter | undefined): Promise<[Document, number][]>;
+    similaritySearchVectorWithScore(query: number[], k: number, filter?: ElasticFilter): Promise<[Document, number][]>;
     /**
      * Method to delete documents from the Elasticsearch database.
      * @param params Object containing the IDs of the documents to delete.

package/dist/vectorstores/elasticsearch.js

@@ -232,12 +232,25 @@ export class ElasticVectorSearch extends VectorStore {
             return;
         await this.client.indices.create(request);
     }
-    buildMetadataTerms(filter) {
+    buildMetadataTerms(filter
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ) {
         if (filter == null)
             return [];
         const result = [];
-        for (const [key, value] of Object.entries(filter)) {
-            result.push({ term: { [`metadata.${key}`]: value } });
+        const filters = Array.isArray(filter)
+            ? filter
+            : Object.entries(filter).map(([key, value]) => ({
+                operator: "term",
+                field: key,
+                value,
+            }));
+        for (const condition of filters) {
+            result.push({
+                [condition.operator]: {
+                    [`metadata.${condition.field}`]: condition.value,
+                },
+            });
         }
         return result;
     }

package/dist/vectorstores/pgvector.cjs

@@ -26,6 +26,24 @@ class PGVectorStore extends base_js_1.VectorStore {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "collectionTableName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "collectionName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "langchain"
+        });
+        Object.defineProperty(this, "collectionMetadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "idColumnName", {
             enumerable: true,
             configurable: true,
@@ -81,6 +99,9 @@ class PGVectorStore extends base_js_1.VectorStore {
             value: 500
         });
         this.tableName = config.tableName;
+        this.collectionTableName = config.collectionTableName;
+        this.collectionName = config.collectionName ?? "langchain";
+        this.collectionMetadata = config.collectionMetadata ?? null;
         this.filter = config.filter;
         this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
         this.contentColumnName = config.columns?.contentColumnName ?? "text";
@@ -106,6 +127,9 @@ class PGVectorStore extends base_js_1.VectorStore {
         const postgresqlVectorStore = new PGVectorStore(embeddings, config);
         await postgresqlVectorStore._initializeClient();
         await postgresqlVectorStore.ensureTableInDatabase();
+        if (postgresqlVectorStore.collectionTableName) {
+            await postgresqlVectorStore.ensureCollectionTableInDatabase();
+        }
         return postgresqlVectorStore;
     }
     async _initializeClient() {
@@ -122,15 +146,56 @@ class PGVectorStore extends base_js_1.VectorStore {
         const texts = documents.map(({ pageContent }) => pageContent);
         return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
     }
+    /**
+     * Inserts a row for the collectionName provided at initialization if it does not
+     * exist and returns the collectionId.
+     *
+     * @returns The collectionId for the given collectionName.
+     */
+    async getOrCreateCollection() {
+        const queryString = `
+      SELECT uuid from ${this.collectionTableName}
+      WHERE name = $1;
+    `;
+        const queryResult = await this.pool.query(queryString, [
+            this.collectionName,
+        ]);
+        let collectionId = queryResult.rows[0]?.uuid;
+        if (!collectionId) {
+            const insertString = `
+        INSERT INTO ${this.collectionTableName}(
+          uuid,
+          name,
+          cmetadata
+        )
+        VALUES (
+          uuid_generate_v4(),
+          $1,
+          $2
+        )
+        RETURNING uuid;
+      `;
+            const insertResult = await this.pool.query(insertString, [
+                this.collectionName,
+                this.collectionMetadata,
+            ]);
+            collectionId = insertResult.rows[0]?.uuid;
+        }
+        return collectionId;
+    }
     /**
      * Generates the SQL placeholders for a specific row at the provided index.
      *
      * @param index - The index of the row for which placeholders need to be generated.
+     * @param numOfColumns - The number of columns we are inserting data into.
      * @returns The SQL placeholders for the row values.
      */
-    generatePlaceholderForRowAt(index) {
-        const base = index * 3;
-        return `($${base + 1}, $${base + 2}, $${base + 3})`;
+    generatePlaceholderForRowAt(index, numOfColumns) {
+        const placeholders = [];
+        for (let i = 0; i < numOfColumns; i += 1) {
+            placeholders.push(`$${index * numOfColumns + i + 1}`);
+        }
+        return `(${placeholders.join(", ")})`;
     }
     /**
      * Constructs the SQL query for inserting rows into the specified table.
@@ -139,15 +204,25 @@ class PGVectorStore extends base_js_1.VectorStore {
      * @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
      * @returns The complete SQL INSERT INTO query string.
      */
-    buildInsertQuery(rows) {
+    async buildInsertQuery(rows) {
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        const columns = [
+            this.contentColumnName,
+            this.vectorColumnName,
+            this.metadataColumnName,
+        ];
+        if (collectionId) {
+            columns.push("collection_id");
+        }
         const valuesPlaceholders = rows
-            .map((_, j) => this.generatePlaceholderForRowAt(j))
+            .map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
             .join(", ");
         const text = `
       INSERT INTO ${this.tableName}(
-        ${this.contentColumnName},
-        ${this.vectorColumnName},
-        ${this.metadataColumnName}
+        ${columns}
       )
       VALUES ${valuesPlaceholders}
     `;
@@ -162,17 +237,24 @@ class PGVectorStore extends base_js_1.VectorStore {
      * @returns Promise that resolves when the vectors have been added.
      */
     async addVectors(vectors, documents) {
-        const rows = vectors.map((embedding, idx) => {
+        const rows = [];
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        for (let i = 0; i < vectors.length; i += 1) {
+            const values = [];
+            const embedding = vectors[i];
             const embeddingString = `[${embedding.join(",")}]`;
-            return [
-                documents[idx].pageContent,
-                embeddingString,
-                documents[idx].metadata,
-            ];
-        });
+            values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
+            if (collectionId) {
+                values.push(collectionId);
+            }
+            rows.push(values);
+        }
         for (let i = 0; i < rows.length; i += this.chunkSize) {
             const chunk = rows.slice(i, i + this.chunkSize);
-            const insertQuery = this.buildInsertQuery(chunk);
+            const insertQuery = await this.buildInsertQuery(chunk);
             const flatValues = chunk.flat();
             try {
                 await this.pool.query(insertQuery, flatValues);
@@ -196,13 +278,23 @@ class PGVectorStore extends base_js_1.VectorStore {
     async similaritySearchVectorWithScore(query, k, filter) {
         const embeddingString = `[${query.join(",")}]`;
         const _filter = filter ?? "{}";
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        const parameters = [embeddingString, _filter, k];
+        if (collectionId) {
+            parameters.push(collectionId);
+        }
         const queryString = `
       SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
       FROM ${this.tableName}
       WHERE ${this.metadataColumnName}::jsonb @> $2
+      ${collectionId ? "AND collection_id = $4" : ""}
       ORDER BY "_distance" ASC
-      LIMIT $3;`;
-        const documents = (await this.pool.query(queryString, [embeddingString, _filter, k])).rows;
+      LIMIT $3;
+    `;
+        const documents = (await this.pool.query(queryString, parameters)).rows;
         const results = [];
         for (const doc of documents) {
             if (doc._distance != null && doc[this.contentColumnName] != null) {
@@ -233,6 +325,38 @@ class PGVectorStore extends base_js_1.VectorStore {
       );
     `);
     }
+    /**
+     * Method to ensure the existence of the collection table in the database.
+     * It creates the table if it does not already exist.
+     *
+     * @returns Promise that resolves when the collection table has been ensured.
+     */
+    async ensureCollectionTableInDatabase() {
+        try {
+            await this.pool.query(`
+        CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
+          uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
+          name character varying,
+          cmetadata jsonb
+        );
+
+        ALTER TABLE ${this.tableName}
+          ADD COLUMN collection_id uuid;
+
+        ALTER TABLE ${this.tableName}
+          ADD CONSTRAINT ${this.tableName}_collection_id_fkey
+          FOREIGN KEY (collection_id)
+          REFERENCES ${this.collectionTableName}(uuid)
+          ON DELETE CASCADE;
+      `);
+        }
+        catch (e) {
+            if (!e.message.includes("already exists")) {
+                console.error(e);
+                throw new Error(`Error adding column: ${e.message}`);
+            }
+        }
+    }
     /**
      * Static method to create a new `PGVectorStore` instance from an
      * array of texts and their metadata. It converts the texts into

package/dist/vectorstores/pgvector.d.ts

@@ -11,6 +11,9 @@ type Metadata = Record<string, unknown>;
 export interface PGVectorStoreArgs {
     postgresConnectionOptions: PoolConfig;
     tableName: string;
+    collectionTableName?: string;
+    collectionName?: string;
+    collectionMetadata?: Metadata | null;
     columns?: {
         idColumnName?: string;
         vectorColumnName?: string;
@@ -35,6 +38,9 @@ export interface PGVectorStoreArgs {
 export declare class PGVectorStore extends VectorStore {
     FilterType: Metadata;
     tableName: string;
+    collectionTableName?: string;
+    collectionName: string;
+    collectionMetadata: Metadata | null;
     idColumnName: string;
     vectorColumnName: string;
     contentColumnName: string;
@@ -65,10 +71,18 @@ export declare class PGVectorStore extends VectorStore {
      * @returns Promise that resolves when the documents have been added.
      */
     addDocuments(documents: Document[]): Promise<void>;
+    /**
+     * Inserts a row for the collectionName provided at initialization if it does not
+     * exist and returns the collectionId.
+     *
+     * @returns The collectionId for the given collectionName.
+     */
+    getOrCreateCollection(): Promise<string>;
     /**
      * Generates the SQL placeholders for a specific row at the provided index.
      *
      * @param index - The index of the row for which placeholders need to be generated.
+     * @param numOfColumns - The number of columns we are inserting data into.
      * @returns The SQL placeholders for the row values.
      */
     private generatePlaceholderForRowAt;
@@ -107,6 +121,13 @@ export declare class PGVectorStore extends VectorStore {
      * @returns Promise that resolves when the table has been ensured.
      */
     ensureTableInDatabase(): Promise<void>;
+    /**
+     * Method to ensure the existence of the collection table in the database.
+     * It creates the table if it does not already exist.
+     *
+     * @returns Promise that resolves when the collection table has been ensured.
+     */
+    ensureCollectionTableInDatabase(): Promise<void>;
     /**
      * Static method to create a new `PGVectorStore` instance from an
      * array of texts and their metadata. It converts the texts into