langchain 0.0.202 → 0.0.204-rc.0

package/dist/vectorstores/chroma.js

@@ -1,299 +1 @@
-import * as uuid from "uuid";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-/**
- * The main class that extends the `VectorStore` class. It provides
- * methods for interacting with the Chroma database, such as adding
- * documents, deleting documents, and searching for similar vectors.
- */
-export class Chroma extends VectorStore {
-    _vectorstoreType() {
-        return "chroma";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "index", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collectionName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "collectionMetadata", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "numDimensions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "url", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "filter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.numDimensions = args.numDimensions;
-        this.embeddings = embeddings;
-        this.collectionName = ensureCollectionName(args.collectionName);
-        this.collectionMetadata = args.collectionMetadata;
-        if ("index" in args) {
-            this.index = args.index;
-        }
-        else if ("url" in args) {
-            this.url = args.url || "http://localhost:8000";
-        }
-        this.filter = args.filter;
-    }
-    /**
-     * Adds documents to the Chroma database. The documents are first
-     * converted to vectors using the `embeddings` instance, and then added to
-     * the database.
-     * @param documents An array of `Document` instances to be added to the database.
-     * @param options Optional. An object containing an array of `ids` for the documents.
-     * @returns A promise that resolves when the documents have been added to the database.
-     */
-    async addDocuments(documents, options) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
-    }
-    /**
-     * Ensures that a collection exists in the Chroma database. If the
-     * collection does not exist, it is created.
-     * @returns A promise that resolves with the `Collection` instance.
-     */
-    async ensureCollection() {
-        if (!this.collection) {
-            if (!this.index) {
-                const { ChromaClient } = await Chroma.imports();
-                this.index = new ChromaClient({ path: this.url });
-            }
-            try {
-                this.collection = await this.index.getOrCreateCollection({
-                    name: this.collectionName,
-                    ...(this.collectionMetadata && { metadata: this.collectionMetadata }),
-                });
-            }
-            catch (err) {
-                throw new Error(`Chroma getOrCreateCollection error: ${err}`);
-            }
-        }
-        return this.collection;
-    }
-    /**
-     * Adds vectors to the Chroma database. The vectors are associated with
-     * the provided documents.
-     * @param vectors An array of vectors to be added to the database.
-     * @param documents An array of `Document` instances associated with the vectors.
-     * @param options Optional. An object containing an array of `ids` for the vectors.
-     * @returns A promise that resolves with an array of document IDs when the vectors have been added to the database.
-     */
-    async addVectors(vectors, documents, options) {
-        if (vectors.length === 0) {
-            return [];
-        }
-        if (this.numDimensions === undefined) {
-            this.numDimensions = vectors[0].length;
-        }
-        if (vectors.length !== documents.length) {
-            throw new Error(`Vectors and metadatas must have the same length`);
-        }
-        if (vectors[0].length !== this.numDimensions) {
-            throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
-        }
-        const documentIds = options?.ids ?? Array.from({ length: vectors.length }, () => uuid.v1());
-        const collection = await this.ensureCollection();
-        const mappedMetadatas = documents.map(({ metadata }) => {
-            let locFrom;
-            let locTo;
-            if (metadata?.loc) {
-                if (metadata.loc.lines?.from !== undefined)
-                    locFrom = metadata.loc.lines.from;
-                if (metadata.loc.lines?.to !== undefined)
-                    locTo = metadata.loc.lines.to;
-            }
-            const newMetadata = {
-                ...metadata,
-                ...(locFrom !== undefined && { locFrom }),
-                ...(locTo !== undefined && { locTo }),
-            };
-            if (newMetadata.loc)
-                delete newMetadata.loc;
-            return newMetadata;
-        });
-        await collection.upsert({
-            ids: documentIds,
-            embeddings: vectors,
-            metadatas: mappedMetadatas,
-            documents: documents.map(({ pageContent }) => pageContent),
-        });
-        return documentIds;
-    }
-    /**
-     * Deletes documents from the Chroma database. The documents to be deleted
-     * can be specified by providing an array of `ids` or a `filter` object.
-     * @param params An object containing either an array of `ids` of the documents to be deleted or a `filter` object to specify the documents to be deleted.
-     * @returns A promise that resolves when the specified documents have been deleted from the database.
-     */
-    async delete(params) {
-        const collection = await this.ensureCollection();
-        if (Array.isArray(params.ids)) {
-            await collection.delete({ ids: params.ids });
-        }
-        else if (params.filter) {
-            await collection.delete({
-                where: { ...params.filter },
-            });
-        }
-        else {
-            throw new Error(`You must provide one of "ids or "filter".`);
-        }
-    }
-    /**
-     * Searches for vectors in the Chroma database that are similar to the
-     * provided query vector. The search can be filtered using the provided
-     * `filter` object or the `filter` property of the `Chroma` instance.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional. A `filter` object to filter the search results.
-     * @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (filter && this.filter) {
-            throw new Error("cannot provide both `filter` and `this.filter`");
-        }
-        const _filter = filter ?? this.filter;
-        const collection = await this.ensureCollection();
-        // similaritySearchVectorWithScore supports one query vector at a time
-        // chroma supports multiple query vectors at a time
-        const result = await collection.query({
-            queryEmbeddings: query,
-            nResults: k,
-            where: { ..._filter },
-        });
-        const { ids, distances, documents, metadatas } = result;
-        if (!ids || !distances || !documents || !metadatas) {
-            return [];
-        }
-        // get the result data from the first and only query vector
-        const [firstIds] = ids;
-        const [firstDistances] = distances;
-        const [firstDocuments] = documents;
-        const [firstMetadatas] = metadatas;
-        const results = [];
-        for (let i = 0; i < firstIds.length; i += 1) {
-            let metadata = firstMetadatas?.[i] ?? {};
-            if (metadata.locFrom && metadata.locTo) {
-                metadata = {
-                    ...metadata,
-                    loc: {
-                        lines: {
-                            from: metadata.locFrom,
-                            to: metadata.locTo,
-                        },
-                    },
-                };
-                delete metadata.locFrom;
-                delete metadata.locTo;
-            }
-            results.push([
-                new Document({
-                    pageContent: firstDocuments?.[i] ?? "",
-                    metadata,
-                }),
-                firstDistances[i],
-            ]);
-        }
-        return results;
-    }
-    /**
-     * Creates a new `Chroma` instance from an array of text strings. The text
-     * strings are converted to `Document` instances and added to the Chroma
-     * database.
-     * @param texts An array of text strings.
-     * @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
-     * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
-     * @param dbConfig A `ChromaLibArgs` object containing the configuration for the Chroma database.
-     * @returns A promise that resolves with a new `Chroma` instance.
-     */
-    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return this.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Creates a new `Chroma` instance from an array of `Document` instances.
-     * The documents are added to the Chroma database.
-     * @param docs An array of `Document` instances.
-     * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
-     * @param dbConfig A `ChromaLibArgs` object containing the configuration for the Chroma database.
-     * @returns A promise that resolves with a new `Chroma` instance.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Creates a new `Chroma` instance from an existing collection in the
-     * Chroma database.
-     * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
-     * @param dbConfig A `ChromaLibArgs` object containing the configuration for the Chroma database.
-     * @returns A promise that resolves with a new `Chroma` instance.
-     */
-    static async fromExistingCollection(embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.ensureCollection();
-        return instance;
-    }
-    /**
-     * Imports the `ChromaClient` from the `chromadb` module.
-     * @returns A promise that resolves with an object containing the `ChromaClient` constructor.
-     */
-    static async imports() {
-        try {
-            const { ChromaClient } = await import("chromadb");
-            return { ChromaClient };
-        }
-        catch (e) {
-            throw new Error("Please install chromadb as a dependency with, e.g. `npm install -S chromadb`");
-        }
-    }
-}
-/**
- * Generates a unique collection name if none is provided.
- */
-function ensureCollectionName(collectionName) {
-    if (!collectionName) {
-        return `langchain-${uuid.v4()}`;
-    }
-    return collectionName;
-}
+export * from "@langchain/community/vectorstores/chroma";

package/dist/vectorstores/clickhouse.cjs

@@ -10,277 +10,8 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ClickHouseStore = void 0;
-const uuid = __importStar(require("uuid"));
-const client_1 = require("@clickhouse/client");
-const mysql2_1 = require("mysql2");
-const base_js_1 = require("./base.cjs");
-const document_js_1 = require("../document.cjs");
-/**
- * Class for interacting with the ClickHouse database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-class ClickHouseStore extends base_js_1.VectorStore {
-    _vectorstoreType() {
-        return "clickhouse";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexParam", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexQueryParams", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "columnMap", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "database", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "table", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isInitialized", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        this.indexType = args.indexType || "annoy";
-        this.indexParam = args.indexParam || { L2Distance: 100 };
-        this.indexQueryParams = args.indexQueryParams || {};
-        this.columnMap = args.columnMap || {
-            id: "id",
-            document: "document",
-            embedding: "embedding",
-            metadata: "metadata",
-            uuid: "uuid",
-        };
-        this.database = args.database || "default";
-        this.table = args.table || "vector_table";
-        this.client = (0, client_1.createClient)({
-            host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
-            username: args.username,
-            password: args.password,
-            session_id: uuid.v4(),
-        });
-    }
-    /**
-     * Method to add vectors to the ClickHouse database.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        if (!this.isInitialized) {
-            await this.initialize(vectors[0].length);
-        }
-        const queryStr = this.buildInsertQuery(vectors, documents);
-        await this.client.exec({ query: queryStr });
-    }
-    /**
-     * Method to add documents to the ClickHouse database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents) {
-        return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
-    }
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!this.isInitialized) {
-            await this.initialize(query.length);
-        }
-        const queryStr = this.buildSearchQuery(query, k, filter);
-        const queryResultSet = await this.client.query({ query: queryStr });
-        const queryResult = await queryResultSet.json();
-        const result = queryResult.data.map((item) => [
-            new document_js_1.Document({ pageContent: item.document, metadata: item.metadata }),
-            item.dist,
-        ]);
-        return result;
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromTexts(texts, metadatas, embeddings, args) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new document_js_1.Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return ClickHouseStore.fromDocuments(docs, embeddings, args);
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromDocuments(docs, embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to create an instance of ClickHouseStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the ClickHouseStore.
-     * @returns Promise that resolves with a new instance of ClickHouseStore.
-     */
-    static async fromExistingIndex(embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.initialize();
-        return instance;
-    }
-    /**
-     * Method to initialize the ClickHouse database.
-     * @param dimension Optional dimension of the vectors.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    async initialize(dimension) {
-        const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
-        const indexParamStr = this.indexParam
-            ? Object.entries(this.indexParam)
-                .map(([key, value]) => `'${key}', ${value}`)
-                .join(", ")
-            : "";
-        const query = `
-    CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
-      ${this.columnMap.id} Nullable(String),
-      ${this.columnMap.document} Nullable(String),
-      ${this.columnMap.embedding} Array(Float32),
-      ${this.columnMap.metadata} JSON,
-      ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),
-      CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},
-      INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000
-    ) ENGINE =  MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;
-        await this.client.exec({
-            query,
-            clickhouse_settings: {
-                allow_experimental_object_type: 1,
-                allow_experimental_annoy_index: 1,
-            },
-        });
-        this.isInitialized = true;
-    }
-    /**
-     * Method to build an SQL query for inserting vectors and documents into
-     * the ClickHouse database.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns The SQL query string.
-     */
-    buildInsertQuery(vectors, documents) {
-        const columnsStr = Object.values(Object.fromEntries(Object.entries(this.columnMap).filter(([key]) => key !== this.columnMap.uuid))).join(", ");
-        const placeholders = vectors.map(() => "(?, ?, ?, ?)").join(", ");
-        const values = [];
-        for (let i = 0; i < vectors.length; i += 1) {
-            const vector = vectors[i];
-            const document = documents[i];
-            values.push(uuid.v4(), this.escapeString(document.pageContent), JSON.stringify(vector), JSON.stringify(document.metadata));
-        }
-        const insertQueryStr = `
-      INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr}) 
-      VALUES ${placeholders}
-    `;
-        const insertQuery = (0, mysql2_1.format)(insertQueryStr, values);
-        return insertQuery;
-    }
-    escapeString(str) {
-        return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
-    }
-    /**
-     * Method to build an SQL query for searching for similar vectors in the
-     * ClickHouse database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns The SQL query string.
-     */
-    buildSearchQuery(query, k, filter) {
-        const order = "ASC";
-        const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
-        const placeholders = query.map(() => "?").join(", ");
-        const settingStrings = [];
-        if (this.indexQueryParams) {
-            for (const [key, value] of Object.entries(this.indexQueryParams)) {
-                settingStrings.push(`SETTING ${key}=${value}`);
-            }
-        }
-        const searchQueryStr = `
-      SELECT ${this.columnMap.document} AS document, ${this.columnMap.metadata} AS metadata, dist
-      FROM ${this.database}.${this.table}
-      ${whereStr}
-      ORDER BY L2Distance(${this.columnMap.embedding}, [${placeholders}]) AS dist ${order}
-      LIMIT ${k} ${settingStrings.join(" ")}
-    `;
-        // Format the query with actual values
-        const searchQuery = (0, mysql2_1.format)(searchQueryStr, query);
-        return searchQuery;
-    }
-}
-exports.ClickHouseStore = ClickHouseStore;
+__exportStar(require("@langchain/community/vectorstores/clickhouse"), exports);