langchain 0.0.203 → 0.0.204-rc.0

package/dist/vectorstores/clickhouse.d.ts

@@ -1,126 +1 @@
-import { Embeddings } from "../embeddings/base.js";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-/**
- * Arguments for the ClickHouseStore class, which include the host, port,
- * protocol, username, password, index type, index parameters,
- * index query params, column map, database, table.
- */
-export interface ClickHouseLibArgs {
-    host: string;
-    port: string | number;
-    protocol?: string;
-    username: string;
-    password: string;
-    indexType?: string;
-    indexParam?: Record<string, number>;
-    indexQueryParams?: Record<string, string>;
-    columnMap?: ColumnMap;
-    database?: string;
-    table?: string;
-}
-/**
- * Mapping of columns in the ClickHouse database.
- */
-export interface ColumnMap {
-    id: string;
-    uuid: string;
-    document: string;
-    embedding: string;
-    metadata: string;
-}
-/**
- * Type for filtering search results in the ClickHouse database.
- */
-export interface ClickHouseFilter {
-    whereStr: string;
-}
-/**
- * Class for interacting with the ClickHouse database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-export declare class ClickHouseStore extends VectorStore {
-    FilterType: ClickHouseFilter;
-    private client;
-    private indexType;
-    private indexParam;
-    private indexQueryParams;
-    private columnMap;
-    private database;
-    private table;
-    private isInitialized;
-    _vectorstoreType(): string;
-    constructor(embeddings: Embeddings, args: ClickHouseLibArgs);
-    /**
-     * Method to add vectors to the ClickHouse database.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    addVectors(vectors: number[][], documents: Document[]): Promise<void>;
-    /**
-     * Method to add documents to the ClickHouse database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    addDocuments(documents: Document[]): Promise<void>;
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document, number][]>;
-    /**
-     * Static method to create an instance of ClickHouseStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static fromTexts(texts: string[], metadatas: object | object[], embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
-    /**
-     * Static method to create an instance of ClickHouseStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static fromDocuments(docs: Document[], embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
-    /**
-     * Static method to create an instance of ClickHouseStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static fromExistingIndex(embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
-    /**
-     * Method to initialize the ClickHouse database.
-     * @param dimension Optional dimension of the vectors.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    private initialize;
-    /**
-     * Method to build an SQL query for inserting vectors and documents into
-     * the ClickHouse database.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns The SQL query string.
-     */
-    private buildInsertQuery;
-    private escapeString;
-    /**
-     * Method to build an SQL query for searching for similar vectors in the
-     * ClickHouse database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns The SQL query string.
-     */
-    private buildSearchQuery;
-}
+export * from "@langchain/community/vectorstores/clickhouse";

package/dist/vectorstores/clickhouse.js

@@ -1,259 +1 @@
-import * as uuid from "uuid";
-import { createClient } from "@clickhouse/client";
-import { format } from "mysql2";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-/**
- * Class for interacting with the ClickHouse database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-export class ClickHouseStore extends VectorStore {
-    _vectorstoreType() {
-        return "clickhouse";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexParam", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexQueryParams", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "columnMap", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "database", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "table", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isInitialized", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        this.indexType = args.indexType || "annoy";
-        this.indexParam = args.indexParam || { L2Distance: 100 };
-        this.indexQueryParams = args.indexQueryParams || {};
-        this.columnMap = args.columnMap || {
-            id: "id",
-            document: "document",
-            embedding: "embedding",
-            metadata: "metadata",
-            uuid: "uuid",
-        };
-        this.database = args.database || "default";
-        this.table = args.table || "vector_table";
-        this.client = createClient({
-            host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
-            username: args.username,
-            password: args.password,
-            session_id: uuid.v4(),
-        });
-    }
-    /**
-     * Method to add vectors to the ClickHouse database.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        if (!this.isInitialized) {
-            await this.initialize(vectors[0].length);
-        }
-        const queryStr = this.buildInsertQuery(vectors, documents);
-        await this.client.exec({ query: queryStr });
-    }
-    /**
-     * Method to add documents to the ClickHouse database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents) {
-        return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
-    }
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!this.isInitialized) {
-            await this.initialize(query.length);
-        }
-        const queryStr = this.buildSearchQuery(query, k, filter);
-        const queryResultSet = await this.client.query({ query: queryStr });
-        const queryResult = await queryResultSet.json();
-        const result = queryResult.data.map((item) => [
-            new Document({ pageContent: item.document, metadata: item.metadata }),
-            item.dist,
-        ]);
-        return result;
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromTexts(texts, metadatas, embeddings, args) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return ClickHouseStore.fromDocuments(docs, embeddings, args);
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromDocuments(docs, embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromExistingIndex(embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.initialize();
-        return instance;
-    }
-    /**
-     * Method to initialize the ClickHouse database.
-     * @param dimension Optional dimension of the vectors.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    async initialize(dimension) {
-        const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
-        const indexParamStr = this.indexParam
-            ? Object.entries(this.indexParam)
-                .map(([key, value]) => `'${key}', ${value}`)
-                .join(", ")
-            : "";
-        const query = `
-    CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
-      ${this.columnMap.id} Nullable(String),
-      ${this.columnMap.document} Nullable(String),
-      ${this.columnMap.embedding} Array(Float32),
-      ${this.columnMap.metadata} JSON,
-      ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),
-      CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},
-      INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000
-    ) ENGINE =  MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;
-        await this.client.exec({
-            query,
-            clickhouse_settings: {
-                allow_experimental_object_type: 1,
-                allow_experimental_annoy_index: 1,
-            },
-        });
-        this.isInitialized = true;
-    }
-    /**
-     * Method to build an SQL query for inserting vectors and documents into
-     * the ClickHouse database.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns The SQL query string.
-     */
-    buildInsertQuery(vectors, documents) {
-        const columnsStr = Object.values(Object.fromEntries(Object.entries(this.columnMap).filter(([key]) => key !== this.columnMap.uuid))).join(", ");
-        const placeholders = vectors.map(() => "(?, ?, ?, ?)").join(", ");
-        const values = [];
-        for (let i = 0; i < vectors.length; i += 1) {
-            const vector = vectors[i];
-            const document = documents[i];
-            values.push(uuid.v4(), this.escapeString(document.pageContent), JSON.stringify(vector), JSON.stringify(document.metadata));
-        }
-        const insertQueryStr = `
-      INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr}) 
-      VALUES ${placeholders}
-    `;
-        const insertQuery = format(insertQueryStr, values);
-        return insertQuery;
-    }
-    escapeString(str) {
-        return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
-    }
-    /**
-     * Method to build an SQL query for searching for similar vectors in the
-     * ClickHouse database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns The SQL query string.
-     */
-    buildSearchQuery(query, k, filter) {
-        const order = "ASC";
-        const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
-        const placeholders = query.map(() => "?").join(", ");
-        const settingStrings = [];
-        if (this.indexQueryParams) {
-            for (const [key, value] of Object.entries(this.indexQueryParams)) {
-                settingStrings.push(`SETTING ${key}=${value}`);
-            }
-        }
-        const searchQueryStr = `
-      SELECT ${this.columnMap.document} AS document, ${this.columnMap.metadata} AS metadata, dist
-      FROM ${this.database}.${this.table}
-      ${whereStr}
-      ORDER BY L2Distance(${this.columnMap.embedding}, [${placeholders}]) AS dist ${order}
-      LIMIT ${k} ${settingStrings.join(" ")}
-    `;
-        // Format the query with actual values
-        const searchQuery = format(searchQueryStr, query);
-        return searchQuery;
-    }
-}
+export * from "@langchain/community/vectorstores/clickhouse";

package/dist/vectorstores/closevector/node.cjs

@@ -1,109 +1,17 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloseVectorNode = void 0;
-const closevector_node_1 = require("closevector-node");
-const common_js_1 = require("./common.cjs");
-/**
- * Class that implements a vector store using Hierarchical Navigable Small
- * World (HNSW) graphs. It extends the SaveableVectorStore class and
- * provides methods for adding documents and vectors, performing
- * similarity searches, and saving and loading the vector store.
- */
-class CloseVectorNode extends common_js_1.CloseVector {
-    constructor(embeddings, args, credentials) {
-        super(embeddings, args, credentials);
-        if (args.instance) {
-            this.instance = args.instance;
-        }
-        else {
-            this.instance = new closevector_node_1.CloseVectorHNSWNode(embeddings, args);
-        }
-        if (this.credentials?.key) {
-            this.instance.accessKey = this.credentials.key;
-        }
-        if (this.credentials?.secret) {
-            this.instance.secret = this.credentials.secret;
-        }
-    }
-    /**
-     * Method to save the index to the CloseVector CDN.
-     * @param options
-     * @param options.description A description of the index.
-     * @param options.public Whether the index should be public or private. Defaults to false.
-     * @param options.uuid A UUID for the index. If not provided, a new index will be created.
-     * @param options.onProgress A callback function that will be called with the progress of the upload.
-     */
-    async saveToCloud(options) {
-        await this.instance.saveToCloud(options);
-    }
-    /**
-     * Method to load the index from the CloseVector CDN.
-     * @param options
-     * @param options.uuid The UUID of the index to be downloaded.
-     * @param options.credentials The credentials to be used by the CloseVectorNode instance.
-     * @param options.embeddings The embeddings to be used by the CloseVectorNode instance.
-     * @param options.onProgress A callback function that will be called with the progress of the download.
-     */
-    static async loadFromCloud(options) {
-        if (!options.credentials.key || !options.credentials.secret) {
-            throw new Error("key and secret must be provided");
-        }
-        const instance = await closevector_node_1.CloseVectorHNSWNode.loadFromCloud({
-            ...options,
-            accessKey: options.credentials.key,
-            secret: options.credentials.secret,
-        });
-        const vectorstore = new this(options.embeddings, instance.args, options.credentials);
-        return vectorstore;
-    }
-    /**
-     * Static method to load a vector store from a directory. It reads the
-     * HNSW index, the arguments, and the document store from the directory,
-     * then creates a new HNSWLib instance with these values.
-     * @param directory The directory from which to load the vector store.
-     * @param embeddings The embeddings to be used by the CloseVectorNode instance.
-     * @returns A Promise that resolves to a new CloseVectorNode instance.
-     */
-    static async load(directory, embeddings, credentials) {
-        const instance = await closevector_node_1.CloseVectorHNSWNode.load(directory, embeddings);
-        const vectorstore = new this(embeddings, instance.args, credentials);
-        return vectorstore;
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
     }
-    /**
-     * Static method to create a new CloseVectorWeb instance from texts and metadata.
-     * It creates a new Document instance for each text and metadata, then
-     * calls the fromDocuments method to create the CloseVectorWeb instance.
-     * @param texts The texts to be used to create the documents.
-     * @param metadatas The metadata to be used to create the documents.
-     * @param embeddings The embeddings to be used by the CloseVectorWeb instance.
-     * @param args An optional configuration object for the CloseVectorWeb instance.
-     * @param credential An optional credential object for the CloseVector API.
-     * @returns A Promise that resolves to a new CloseVectorWeb instance.
-     */
-    static async fromTexts(texts, metadatas, embeddings, args, credential) {
-        const docs = common_js_1.CloseVector.textsToDocuments(texts, metadatas);
-        return await CloseVectorNode.fromDocuments(docs, embeddings, args, credential);
-    }
-    /**
-     * Static method to create a new CloseVectorNode instance from documents. It
-     * creates a new CloseVectorNode instance, adds the documents to it, then returns
-     * the instance.
-     * @param docs The documents to be added to the HNSWLib instance.
-     * @param embeddings The embeddings to be used by the HNSWLib instance.
-     * @param args An optional configuration object for the HNSWLib instance.
-     * @param credentials An optional credential object for the CloseVector API.
-     * @returns A Promise that resolves to a new CloseVectorNode instance.
-     */
-    static async fromDocuments(docs, embeddings, args, credentials) {
-        const _args = args || {
-            space: "cosine",
-        };
-        const instance = new this(embeddings, _args, credentials);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    static async imports() {
-        return closevector_node_1.CloseVectorHNSWNode.imports();
-    }
-}
-exports.CloseVectorNode = CloseVectorNode;
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("@langchain/community/vectorstores/closevector/node"), exports);

package/dist/vectorstores/closevector/node.d.ts

@@ -1,83 +1 @@
-import { CloseVectorHNSWNode, HierarchicalNSWT, CloseVectorHNSWLibArgs, CloseVectorCredentials } from "closevector-node";
-import { CloseVector } from "./common.js";
-import { Embeddings } from "../../embeddings/base.js";
-import { Document } from "../../document.js";
-/**
- * package closevector-node is largely based on hnswlib.ts in the current folder with the following exceptions:
- * 1. It uses a modified version of hnswlib-node to ensure the generated index can be loaded by closevector_web.ts.
- * 2. It adds features to upload and download the index to/from the CDN provided by CloseVector.
- *
- * For more information, check out https://closevector-docs.getmegaportal.com/
- */
-/**
- * Arguments for creating a CloseVectorNode instance, extending CloseVectorHNSWLibArgs.
- */
-export interface CloseVectorNodeArgs extends CloseVectorHNSWLibArgs<HierarchicalNSWT> {
-    instance?: CloseVectorHNSWNode;
-}
-/**
- * Class that implements a vector store using Hierarchical Navigable Small
- * World (HNSW) graphs. It extends the SaveableVectorStore class and
- * provides methods for adding documents and vectors, performing
- * similarity searches, and saving and loading the vector store.
- */
-export declare class CloseVectorNode extends CloseVector<CloseVectorHNSWNode> {
-    FilterType: (doc: Document) => boolean;
-    constructor(embeddings: Embeddings, args: CloseVectorNodeArgs, credentials?: CloseVectorCredentials);
-    /**
-     * Method to save the index to the CloseVector CDN.
-     * @param options
-     * @param options.description A description of the index.
-     * @param options.public Whether the index should be public or private. Defaults to false.
-     * @param options.uuid A UUID for the index. If not provided, a new index will be created.
-     * @param options.onProgress A callback function that will be called with the progress of the upload.
-     */
-    saveToCloud(options: Parameters<CloseVectorHNSWNode["saveToCloud"]>[0]): Promise<void>;
-    /**
-     * Method to load the index from the CloseVector CDN.
-     * @param options
-     * @param options.uuid The UUID of the index to be downloaded.
-     * @param options.credentials The credentials to be used by the CloseVectorNode instance.
-     * @param options.embeddings The embeddings to be used by the CloseVectorNode instance.
-     * @param options.onProgress A callback function that will be called with the progress of the download.
-     */
-    static loadFromCloud(options: Omit<Parameters<(typeof CloseVectorHNSWNode)["loadFromCloud"]>[0] & {
-        embeddings: Embeddings;
-        credentials: CloseVectorCredentials;
-    }, "accessKey" | "secret">): Promise<CloseVectorNode>;
-    /**
-     * Static method to load a vector store from a directory. It reads the
-     * HNSW index, the arguments, and the document store from the directory,
-     * then creates a new HNSWLib instance with these values.
-     * @param directory The directory from which to load the vector store.
-     * @param embeddings The embeddings to be used by the CloseVectorNode instance.
-     * @returns A Promise that resolves to a new CloseVectorNode instance.
-     */
-    static load(directory: string, embeddings: Embeddings, credentials?: CloseVectorCredentials): Promise<CloseVectorNode>;
-    /**
-     * Static method to create a new CloseVectorWeb instance from texts and metadata.
-     * It creates a new Document instance for each text and metadata, then
-     * calls the fromDocuments method to create the CloseVectorWeb instance.
-     * @param texts The texts to be used to create the documents.
-     * @param metadatas The metadata to be used to create the documents.
-     * @param embeddings The embeddings to be used by the CloseVectorWeb instance.
-     * @param args An optional configuration object for the CloseVectorWeb instance.
-     * @param credential An optional credential object for the CloseVector API.
-     * @returns A Promise that resolves to a new CloseVectorWeb instance.
-     */
-    static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, args?: Record<string, unknown>, credential?: CloseVectorCredentials): Promise<CloseVectorNode>;
-    /**
-     * Static method to create a new CloseVectorNode instance from documents. It
-     * creates a new CloseVectorNode instance, adds the documents to it, then returns
-     * the instance.
-     * @param docs The documents to be added to the HNSWLib instance.
-     * @param embeddings The embeddings to be used by the HNSWLib instance.
-     * @param args An optional configuration object for the HNSWLib instance.
-     * @param credentials An optional credential object for the CloseVector API.
-     * @returns A Promise that resolves to a new CloseVectorNode instance.
-     */
-    static fromDocuments(docs: Document[], embeddings: Embeddings, args?: Record<string, unknown>, credentials?: CloseVectorCredentials): Promise<CloseVectorNode>;
-    static imports(): Promise<{
-        HierarchicalNSW: typeof HierarchicalNSWT;
-    }>;
-}
+export * from "@langchain/community/vectorstores/closevector/node";