langchain 0.0.203 → 0.0.204-rc.1

package/dist/vectorstores/mongodb_atlas.js

@@ -1,216 +1 @@
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-import { maximalMarginalRelevance } from "../util/math.js";
-/**
- * Class that is a wrapper around MongoDB Atlas Vector Search. It is used
- * to store embeddings in MongoDB documents, create a vector search index,
- * and perform K-Nearest Neighbors (KNN) search with an approximate
- * nearest neighbor algorithm.
- */
-export class MongoDBAtlasVectorSearch extends VectorStore {
-    _vectorstoreType() {
-        return "mongodb_atlas";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "collection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "textKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "embeddingKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.collection = args.collection;
-        this.indexName = args.indexName ?? "default";
-        this.textKey = args.textKey ?? "text";
-        this.embeddingKey = args.embeddingKey ?? "embedding";
-    }
-    /**
-     * Method to add vectors and their corresponding documents to the MongoDB
-     * collection.
-     * @param vectors Vectors to be added.
-     * @param documents Corresponding documents to be added.
-     * @returns Promise that resolves when the vectors and documents have been added.
-     */
-    async addVectors(vectors, documents) {
-        const docs = vectors.map((embedding, idx) => ({
-            [this.textKey]: documents[idx].pageContent,
-            [this.embeddingKey]: embedding,
-            ...documents[idx].metadata,
-        }));
-        await this.collection.insertMany(docs);
-    }
-    /**
-     * Method to add documents to the MongoDB collection. It first converts
-     * the documents to vectors using the embeddings and then calls the
-     * addVectors method.
-     * @param documents Documents to be added.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
-    }
-    /**
-     * Method that performs a similarity search on the vectors stored in the
-     * MongoDB collection. It returns a list of documents and their
-     * corresponding similarity scores.
-     * @param query Query vector for the similarity search.
-     * @param k Number of nearest neighbors to return.
-     * @param filter Optional filter to be applied.
-     * @returns Promise that resolves to a list of documents and their corresponding similarity scores.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        const postFilterPipeline = filter?.postFilterPipeline ?? [];
-        const preFilter = filter?.preFilter ||
-            filter?.postFilterPipeline ||
-            filter?.includeEmbeddings
-            ? filter.preFilter
-            : filter;
-        const removeEmbeddingsPipeline = !filter?.includeEmbeddings
-            ? [
-                {
-                    $project: {
-                        [this.embeddingKey]: 0,
-                    },
-                },
-            ]
-            : [];
-        const pipeline = [
-            {
-                $vectorSearch: {
-                    queryVector: MongoDBAtlasVectorSearch.fixArrayPrecision(query),
-                    index: this.indexName,
-                    path: this.embeddingKey,
-                    limit: k,
-                    numCandidates: 10 * k,
-                    ...(preFilter && { filter: preFilter }),
-                },
-            },
-            {
-                $set: {
-                    score: { $meta: "vectorSearchScore" },
-                },
-            },
-            ...removeEmbeddingsPipeline,
-            ...postFilterPipeline,
-        ];
-        const results = this.collection
-            .aggregate(pipeline)
-            .map((result) => {
-            const { score, [this.textKey]: text, ...metadata } = result;
-            return [new Document({ pageContent: text, metadata }), score];
-        });
-        return results.toArray();
-    }
-    /**
-     * Return documents selected using the maximal marginal relevance.
-     * Maximal marginal relevance optimizes for similarity to the query AND diversity
-     * among selected documents.
-     *
-     * @param {string} query - Text to look up documents similar to.
-     * @param {number} options.k - Number of documents to return.
-     * @param {number} options.fetchK=20- Number of documents to fetch before passing to the MMR algorithm.
-     * @param {number} options.lambda=0.5 - Number between 0 and 1 that determines the degree of diversity among the results,
-     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
-     * @param {MongoDBAtlasFilter} options.filter - Optional Atlas Search operator to pre-filter on document fields
-     *                                      or post-filter following the knnBeta search.
-     *
-     * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
-     */
-    async maxMarginalRelevanceSearch(query, options) {
-        const { k, fetchK = 20, lambda = 0.5, filter } = options;
-        const queryEmbedding = await this.embeddings.embedQuery(query);
-        // preserve the original value of includeEmbeddings
-        const includeEmbeddingsFlag = options.filter?.includeEmbeddings || false;
-        // update filter to include embeddings, as they will be used in MMR
-        const includeEmbeddingsFilter = {
-            ...filter,
-            includeEmbeddings: true,
-        };
-        const resultDocs = await this.similaritySearchVectorWithScore(MongoDBAtlasVectorSearch.fixArrayPrecision(queryEmbedding), fetchK, includeEmbeddingsFilter);
-        const embeddingList = resultDocs.map((doc) => doc[0].metadata[this.embeddingKey]);
-        const mmrIndexes = maximalMarginalRelevance(queryEmbedding, embeddingList, lambda, k);
-        return mmrIndexes.map((idx) => {
-            const doc = resultDocs[idx][0];
-            // remove embeddings if they were not requested originally
-            if (!includeEmbeddingsFlag) {
-                delete doc.metadata[this.embeddingKey];
-            }
-            return doc;
-        });
-    }
-    /**
-     * Static method to create an instance of MongoDBAtlasVectorSearch from a
-     * list of texts. It first converts the texts to vectors and then adds
-     * them to the MongoDB collection.
-     * @param texts List of texts to be converted to vectors.
-     * @param metadatas Metadata for the texts.
-     * @param embeddings Embeddings to be used for conversion.
-     * @param dbConfig Database configuration for MongoDB Atlas.
-     * @returns Promise that resolves to a new instance of MongoDBAtlasVectorSearch.
-     */
-    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return MongoDBAtlasVectorSearch.fromDocuments(docs, embeddings, dbConfig);
-    }
-    /**
-     * Static method to create an instance of MongoDBAtlasVectorSearch from a
-     * list of documents. It first converts the documents to vectors and then
-     * adds them to the MongoDB collection.
-     * @param docs List of documents to be converted to vectors.
-     * @param embeddings Embeddings to be used for conversion.
-     * @param dbConfig Database configuration for MongoDB Atlas.
-     * @returns Promise that resolves to a new instance of MongoDBAtlasVectorSearch.
-     */
-    static async fromDocuments(docs, embeddings, dbConfig) {
-        const instance = new this(embeddings, dbConfig);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to fix the precision of the array that ensures that
-     * every number in this array is always float when casted to other types.
-     * This is needed since MongoDB Atlas Vector Search does not cast integer
-     * inside vector search to float automatically.
-     * This method shall introduce a hint of error but should be safe to use
-     * since introduced error is very small, only applies to integer numbers
-     * returned by embeddings, and most embeddings shall not have precision
-     * as high as 15 decimal places.
-     * @param array Array of number to be fixed.
-     * @returns
-     */
-    static fixArrayPrecision(array) {
-        return array.map((value) => {
-            if (Number.isInteger(value)) {
-                return value + 0.000000000000001;
-            }
-            return value;
-        });
-    }
-}
+export * from "@langchain/community/vectorstores/mongodb_atlas";

package/dist/vectorstores/myscale.cjs

@@ -10,268 +10,8 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MyScaleStore = void 0;
-const uuid = __importStar(require("uuid"));
-const client_1 = require("@clickhouse/client");
-const base_js_1 = require("./base.cjs");
-const document_js_1 = require("../document.cjs");
-/**
- * Class for interacting with the MyScale database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-class MyScaleStore extends base_js_1.VectorStore {
-    _vectorstoreType() {
-        return "myscale";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indexParam", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "columnMap", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "database", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "table", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "metric", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isInitialized", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        this.indexType = args.indexType || "MSTG";
-        this.indexParam = args.indexParam || {};
-        this.columnMap = args.columnMap || {
-            id: "id",
-            text: "text",
-            vector: "vector",
-            metadata: "metadata",
-        };
-        this.database = args.database || "default";
-        this.table = args.table || "vector_table";
-        this.metric = args.metric || "Cosine";
-        this.client = (0, client_1.createClient)({
-            host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
-            username: args.username,
-            password: args.password,
-            session_id: uuid.v4(),
-        });
-    }
-    /**
-     * Method to add vectors to the MyScale database.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        if (!this.isInitialized) {
-            await this.initialize(vectors[0].length);
-        }
-        const queryStr = this.buildInsertQuery(vectors, documents);
-        await this.client.exec({ query: queryStr });
-    }
-    /**
-     * Method to add documents to the MyScale database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents) {
-        return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
-    }
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!this.isInitialized) {
-            await this.initialize(query.length);
-        }
-        const queryStr = this.buildSearchQuery(query, k, filter);
-        const queryResultSet = await this.client.query({ query: queryStr });
-        const queryResult = await queryResultSet.json();
-        const result = queryResult.data.map((item) => [
-            new document_js_1.Document({ pageContent: item.text, metadata: item.metadata }),
-            item.dist,
-        ]);
-        return result;
-    }
-    /**
-     * Static method to create an instance of MyScaleStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static async fromTexts(texts, metadatas, embeddings, args) {
-        const docs = [];
-        for (let i = 0; i < texts.length; i += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
-            const newDoc = new document_js_1.Document({
-                pageContent: texts[i],
-                metadata,
-            });
-            docs.push(newDoc);
-        }
-        return MyScaleStore.fromDocuments(docs, embeddings, args);
-    }
-    /**
-     * Static method to create an instance of MyScaleStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static async fromDocuments(docs, embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to create an instance of MyScaleStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static async fromExistingIndex(embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.initialize();
-        return instance;
-    }
-    /**
-     * Method to initialize the MyScale database.
-     * @param dimension Optional dimension of the vectors.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    async initialize(dimension) {
-        const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
-        let indexParamStr = "";
-        for (const [key, value] of Object.entries(this.indexParam)) {
-            indexParamStr += `, '${key}=${value}'`;
-        }
-        const query = `
-      CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
-        ${this.columnMap.id} String,
-        ${this.columnMap.text} String,
-        ${this.columnMap.vector} Array(Float32),
-        ${this.columnMap.metadata} JSON,
-        CONSTRAINT cons_vec_len CHECK length(${this.columnMap.vector}) = ${dim},
-        VECTOR INDEX vidx ${this.columnMap.vector} TYPE ${this.indexType}('metric_type=${this.metric}'${indexParamStr})
-      ) ENGINE = MergeTree ORDER BY ${this.columnMap.id}
-    `;
-        await this.client.exec({ query: "SET allow_experimental_object_type=1" });
-        await this.client.exec({
-            query: "SET output_format_json_named_tuples_as_objects = 1",
-        });
-        await this.client.exec({ query });
-        this.isInitialized = true;
-    }
-    /**
-     * Method to build an SQL query for inserting vectors and documents into
-     * the MyScale database.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns The SQL query string.
-     */
-    buildInsertQuery(vectors, documents) {
-        const columnsStr = Object.values(this.columnMap).join(", ");
-        const data = [];
-        for (let i = 0; i < vectors.length; i += 1) {
-            const vector = vectors[i];
-            const document = documents[i];
-            const item = [
-                `'${uuid.v4()}'`,
-                `'${this.escapeString(document.pageContent)}'`,
-                `[${vector}]`,
-                `'${JSON.stringify(document.metadata)}'`,
-            ].join(", ");
-            data.push(`(${item})`);
-        }
-        const dataStr = data.join(", ");
-        return `
-      INSERT INTO TABLE
-        ${this.database}.${this.table}(${columnsStr})
-      VALUES
-        ${dataStr}
-    `;
-    }
-    escapeString(str) {
-        return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
-    }
-    /**
-     * Method to build an SQL query for searching for similar vectors in the
-     * MyScale database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns The SQL query string.
-     */
-    buildSearchQuery(query, k, filter) {
-        const order = this.metric === "IP" ? "DESC" : "ASC";
-        const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
-        return `
-      SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist
-      FROM ${this.database}.${this.table}
-      ${whereStr}
-      ORDER BY distance(${this.columnMap.vector}, [${query}]) AS dist ${order}
-      LIMIT ${k}
-    `;
-    }
-}
-exports.MyScaleStore = MyScaleStore;
+__exportStar(require("@langchain/community/vectorstores/myscale"), exports);

package/dist/vectorstores/myscale.d.ts

@@ -1,129 +1 @@
-import { Embeddings } from "../embeddings/base.js";
-import { VectorStore } from "./base.js";
-import { Document } from "../document.js";
-/**
- * Arguments for the MyScaleStore class, which include the host, port,
- * protocol, username, password, index type, index parameters, column map,
- * database, table, and metric.
- */
-export interface MyScaleLibArgs {
-    host: string;
-    port: string | number;
-    protocol?: string;
-    username: string;
-    password: string;
-    indexType?: string;
-    indexParam?: Record<string, string>;
-    columnMap?: ColumnMap;
-    database?: string;
-    table?: string;
-    metric?: metric;
-}
-/**
- * Mapping of columns in the MyScale database.
- */
-export interface ColumnMap {
-    id: string;
-    text: string;
-    vector: string;
-    metadata: string;
-}
-/**
- * Type of metric used in the MyScale database.
- */
-export type metric = "L2" | "Cosine" | "IP";
-/**
- * Type for filtering search results in the MyScale database.
- */
-export interface MyScaleFilter {
-    whereStr: string;
-}
-/**
- * Class for interacting with the MyScale database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-export declare class MyScaleStore extends VectorStore {
-    FilterType: MyScaleFilter;
-    private client;
-    private indexType;
-    private indexParam;
-    private columnMap;
-    private database;
-    private table;
-    private metric;
-    private isInitialized;
-    _vectorstoreType(): string;
-    constructor(embeddings: Embeddings, args: MyScaleLibArgs);
-    /**
-     * Method to add vectors to the MyScale database.
-     * @param vectors The vectors to add.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    addVectors(vectors: number[][], documents: Document[]): Promise<void>;
-    /**
-     * Method to add documents to the MyScale database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    addDocuments(documents: Document[]): Promise<void>;
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document, number][]>;
-    /**
-     * Static method to create an instance of MyScaleStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static fromTexts(texts: string[], metadatas: object | object[], embeddings: Embeddings, args: MyScaleLibArgs): Promise<MyScaleStore>;
-    /**
-     * Static method to create an instance of MyScaleStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static fromDocuments(docs: Document[], embeddings: Embeddings, args: MyScaleLibArgs): Promise<MyScaleStore>;
-    /**
-     * Static method to create an instance of MyScaleStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the MyScaleStore.
-     * @returns Promise that resolves with a new instance of MyScaleStore.
-     */
-    static fromExistingIndex(embeddings: Embeddings, args: MyScaleLibArgs): Promise<MyScaleStore>;
-    /**
-     * Method to initialize the MyScale database.
-     * @param dimension Optional dimension of the vectors.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    private initialize;
-    /**
-     * Method to build an SQL query for inserting vectors and documents into
-     * the MyScale database.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns The SQL query string.
-     */
-    private buildInsertQuery;
-    private escapeString;
-    /**
-     * Method to build an SQL query for searching for similar vectors in the
-     * MyScale database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter Optional filter for the search results.
-     * @returns The SQL query string.
-     */
-    private buildSearchQuery;
-}
+export * from "@langchain/community/vectorstores/myscale";