langchain 0.0.203 → 0.0.204-rc.1

package/dist/vectorstores/base.js

@@ -1,161 +1 @@
-import { BaseRetriever } from "../schema/retriever.js";
-import { Serializable } from "../load/serializable.js";
-/**
- * Class for performing document retrieval from a VectorStore. Can perform
- * similarity search or maximal marginal relevance search.
- */
-export class VectorStoreRetriever extends BaseRetriever {
-    static lc_name() {
-        return "VectorStoreRetriever";
-    }
-    get lc_namespace() {
-        return ["langchain", "retrievers", "base"];
-    }
-    _vectorstoreType() {
-        return this.vectorStore._vectorstoreType();
-    }
-    constructor(fields) {
-        super(fields);
-        Object.defineProperty(this, "vectorStore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "k", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: 4
-        });
-        Object.defineProperty(this, "searchType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "similarity"
-        });
-        Object.defineProperty(this, "searchKwargs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "filter", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.vectorStore = fields.vectorStore;
-        this.k = fields.k ?? this.k;
-        this.searchType = fields.searchType ?? this.searchType;
-        this.filter = fields.filter;
-        if (fields.searchType === "mmr") {
-            this.searchKwargs = fields.searchKwargs;
-        }
-    }
-    async _getRelevantDocuments(query, runManager) {
-        if (this.searchType === "mmr") {
-            if (typeof this.vectorStore.maxMarginalRelevanceSearch !== "function") {
-                throw new Error(`The vector store backing this retriever, ${this._vectorstoreType()} does not support max marginal relevance search.`);
-            }
-            return this.vectorStore.maxMarginalRelevanceSearch(query, {
-                k: this.k,
-                filter: this.filter,
-                ...this.searchKwargs,
-            }, runManager?.getChild("vectorstore"));
-        }
-        return this.vectorStore.similaritySearch(query, this.k, this.filter, runManager?.getChild("vectorstore"));
-    }
-    async addDocuments(documents, options) {
-        return this.vectorStore.addDocuments(documents, options);
-    }
-}
-/**
- * Abstract class representing a store of vectors. Provides methods for
- * adding vectors and documents, deleting from the store, and searching
- * the store.
- */
-export class VectorStore extends Serializable {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    constructor(embeddings, dbConfig) {
-        super(dbConfig);
-        Object.defineProperty(this, "lc_namespace", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: ["langchain", "vectorstores", this._vectorstoreType()]
-        });
-        Object.defineProperty(this, "embeddings", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.embeddings = embeddings;
-    }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    async delete(_params) {
-        throw new Error("Not implemented.");
-    }
-    async similaritySearch(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
-    ) {
-        const results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
-        return results.map((result) => result[0]);
-    }
-    async similaritySearchWithScore(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
-    ) {
-        return this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
-    }
-    static fromTexts(_texts, _metadatas, _embeddings, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    _dbConfig) {
-        throw new Error("the Langchain vectorstore implementation you are using forgot to override this, please report a bug");
-    }
-    static fromDocuments(_docs, _embeddings, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    _dbConfig) {
-        throw new Error("the Langchain vectorstore implementation you are using forgot to override this, please report a bug");
-    }
-    asRetriever(kOrFields, filter, callbacks, tags, metadata, verbose) {
-        if (typeof kOrFields === "number") {
-            return new VectorStoreRetriever({
-                vectorStore: this,
-                k: kOrFields,
-                filter,
-                tags: [...(tags ?? []), this._vectorstoreType()],
-                metadata,
-                verbose,
-                callbacks,
-            });
-        }
-        else {
-            const params = {
-                vectorStore: this,
-                k: kOrFields?.k,
-                filter: kOrFields?.filter,
-                tags: [...(kOrFields?.tags ?? []), this._vectorstoreType()],
-                metadata: kOrFields?.metadata,
-                verbose: kOrFields?.verbose,
-                callbacks: kOrFields?.callbacks,
-                searchType: kOrFields?.searchType,
-            };
-            if (kOrFields?.searchType === "mmr") {
-                return new VectorStoreRetriever({
-                    ...params,
-                    searchKwargs: kOrFields.searchKwargs,
-                });
-            }
-            return new VectorStoreRetriever({ ...params });
-        }
-    }
-}
-/**
- * Abstract class extending VectorStore with functionality for saving and
- * loading the vector store.
- */
-export class SaveableVectorStore extends VectorStore {
-    static load(_directory, _embeddings) {
-        throw new Error("Not implemented");
-    }
-}
+export * from "@langchain/core/vectorstores";

package/dist/vectorstores/cassandra.cjs

@@ -1,480 +1,17 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CassandraStore = void 0;
-/* eslint-disable prefer-template */
-const cassandra_driver_1 = require("cassandra-driver");
-const async_caller_js_1 = require("../util/async_caller.cjs");
-const base_js_1 = require("./base.cjs");
-const document_js_1 = require("../document.cjs");
-/**
- * Class for interacting with the Cassandra database. It extends the
- * VectorStore class and provides methods for adding vectors and
- * documents, searching for similar vectors, and creating instances from
- * texts or documents.
- */
-class CassandraStore extends base_js_1.VectorStore {
-    _vectorstoreType() {
-        return "cassandra";
-    }
-    constructor(embeddings, args) {
-        super(embeddings, args);
-        Object.defineProperty(this, "client", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "vectorType", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "dimensions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "keyspace", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "primaryKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "metadataColumns", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "withClause", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "selectColumns", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "table", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "indices", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "isInitialized", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "asyncCaller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "batchSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { indices = [], maxConcurrency = 25, withClause = "", batchSize = 1, vectorType = "cosine", dimensions, keyspace, table, primaryKey, metadataColumns, } = args;
-        const argsWithDefaults = {
-            ...args,
-            indices,
-            maxConcurrency,
-            withClause,
-            batchSize,
-            vectorType,
-        };
-        this.asyncCaller = new async_caller_js_1.AsyncCaller(argsWithDefaults);
-        this.client = new cassandra_driver_1.Client(argsWithDefaults);
-        // Assign properties
-        this.vectorType = vectorType;
-        this.dimensions = dimensions;
-        this.keyspace = keyspace;
-        this.table = table;
-        this.primaryKey = Array.isArray(primaryKey) ? primaryKey : [primaryKey];
-        this.metadataColumns = metadataColumns;
-        this.withClause = withClause.trim().replace(/^with\s*/i, "");
-        this.indices = indices;
-        this.batchSize = batchSize >= 1 ? batchSize : 1;
-    }
-    /**
-     * Method to save vectors to the Cassandra database.
-     * @param vectors Vectors to save.
-     * @param documents The documents associated with the vectors.
-     * @returns Promise that resolves when the vectors have been added.
-     */
-    async addVectors(vectors, documents) {
-        if (vectors.length === 0) {
-            return;
-        }
-        if (!this.isInitialized) {
-            await this.initialize();
-        }
-        await this.insertAll(vectors, documents);
-    }
-    /**
-     * Method to add documents to the Cassandra database.
-     * @param documents The documents to add.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async addDocuments(documents) {
-        return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
-    }
-    /**
-     * Method to search for vectors that are similar to a given query vector.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filter
-     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
-     */
-    async similaritySearchVectorWithScore(query, k, filter) {
-        if (!this.isInitialized) {
-            await this.initialize();
-        }
-        // Ensure we have an array of Filter from the public interface
-        const filters = this.asFilters(filter);
-        const queryStr = this.buildSearchQuery(filters);
-        // Search query will be of format:
-        //   SELECT ..., text, similarity_x(?) AS similarity_score
-        //     FROM ...
-        //   <WHERE ...>
-        //    ORDER BY vector ANN OF ?
-        //    LIMIT ?
-        // If any filter values are specified, they will be in the WHERE clause as
-        //   filter.name filter.operator ?
-        // queryParams is a list of bind variables sent with the prepared statement
-        const queryParams = [];
-        const vectorAsFloat32Array = new Float32Array(query);
-        queryParams.push(vectorAsFloat32Array);
-        if (filters) {
-            const values = filters.map(({ value }) => value);
-            queryParams.push(...values);
-        }
-        queryParams.push(vectorAsFloat32Array);
-        queryParams.push(k);
-        const queryResultSet = await this.client.execute(queryStr, queryParams, {
-            prepare: true,
-        });
-        return queryResultSet?.rows.map((row) => {
-            const textContent = row.text;
-            const sanitizedRow = { ...row };
-            delete sanitizedRow.text;
-            delete sanitizedRow.similarity_score;
-            // A null value in Cassandra evaluates to a deleted column
-            // as this is treated as a tombstone record for the cell.
-            Object.keys(sanitizedRow).forEach((key) => {
-                if (sanitizedRow[key] === null) {
-                    delete sanitizedRow[key];
-                }
-            });
-            return [
-                new document_js_1.Document({ pageContent: textContent, metadata: sanitizedRow }),
-                row.similarity_score,
-            ];
-        });
-    }
-    /**
-     * Static method to create an instance of CassandraStore from texts.
-     * @param texts The texts to use.
-     * @param metadatas The metadata associated with the texts.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the CassandraStore.
-     * @returns Promise that resolves with a new instance of CassandraStore.
-     */
-    static async fromTexts(texts, metadatas, embeddings, args) {
-        const docs = [];
-        for (let index = 0; index < texts.length; index += 1) {
-            const metadata = Array.isArray(metadatas) ? metadatas[index] : metadatas;
-            const doc = new document_js_1.Document({
-                pageContent: texts[index],
-                metadata,
-            });
-            docs.push(doc);
-        }
-        return CassandraStore.fromDocuments(docs, embeddings, args);
-    }
-    /**
-     * Static method to create an instance of CassandraStore from documents.
-     * @param docs The documents to use.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the CassandraStore.
-     * @returns Promise that resolves with a new instance of CassandraStore.
-     */
-    static async fromDocuments(docs, embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.addDocuments(docs);
-        return instance;
-    }
-    /**
-     * Static method to create an instance of CassandraStore from an existing
-     * index.
-     * @param embeddings The embeddings to use.
-     * @param args The arguments for the CassandraStore.
-     * @returns Promise that resolves with a new instance of CassandraStore.
-     */
-    static async fromExistingIndex(embeddings, args) {
-        const instance = new this(embeddings, args);
-        await instance.initialize();
-        return instance;
-    }
-    /**
-     * Method to initialize the Cassandra database.
-     * @returns Promise that resolves when the database has been initialized.
-     */
-    async initialize() {
-        let cql = "";
-        cql = `CREATE TABLE IF NOT EXISTS ${this.keyspace}.${this.table} (
-      ${this.primaryKey.map((col) => `${col.name} ${col.type}`).join(", ")}
-      , text TEXT
-      ${this.metadataColumns.length > 0
-            ? ", " +
-                this.metadataColumns
-                    .map((col) => `${col.name} ${col.type}`)
-                    .join(", ")
-            : ""}
-      , vector VECTOR<FLOAT, ${this.dimensions}>
-      , ${this.buildPrimaryKey(this.primaryKey)}
-    ) ${this.withClause ? `WITH ${this.withClause}` : ""};`;
-        await this.client.execute(cql);
-        this.selectColumns = `${this.primaryKey
-            .map((col) => `${col.name}`)
-            .join(", ")}
-                          ${this.metadataColumns.length > 0
-            ? ", " +
-                this.metadataColumns
-                    .map((col) => `${col.name}`)
-                    .join(", ")
-            : ""}`;
-        cql = `CREATE CUSTOM INDEX IF NOT EXISTS idx_vector_${this.table}
-           ON ${this.keyspace}.${this.table}(vector) USING 'StorageAttachedIndex' WITH OPTIONS = {'similarity_function': '${this.vectorType.toUpperCase()}'};`;
-        await this.client.execute(cql);
-        for await (const { name, value } of this.indices) {
-            cql = `CREATE CUSTOM INDEX IF NOT EXISTS idx_${this.table}_${name}
-             ON ${this.keyspace}.${this.table} ${value} USING 'StorageAttachedIndex';`;
-            await this.client.execute(cql);
-        }
-        this.isInitialized = true;
-    }
-    /**
-     * Method to build the PRIMARY KEY clause for CREATE TABLE.
-     * @param columns: list of Column to include in the key
-     * @returns The clause, including PRIMARY KEY
-     */
-    buildPrimaryKey(columns) {
-        // Partition columns may be specified with optional attribute col.partition
-        const partitionColumns = columns
-            .filter((col) => col.partition)
-            .map((col) => col.name)
-            .join(", ");
-        // All columns not part of the partition key are clustering columns
-        const clusteringColumns = columns
-            .filter((col) => !col.partition)
-            .map((col) => col.name)
-            .join(", ");
-        let primaryKey = "";
-        // If partition columns are specified, they are included in a () wrapper
-        // If not, the clustering columns are used, and the first clustering column
-        // is the partition key per normal Cassandra behaviour.
-        if (partitionColumns) {
-            primaryKey = `PRIMARY KEY ((${partitionColumns}), ${clusteringColumns})`;
-        }
-        else {
-            primaryKey = `PRIMARY KEY (${clusteringColumns})`;
-        }
-        return primaryKey;
-    }
-    /**
-     * Type guard to check if an object is a Filter.
-     * @param obj: the object to check
-     * @returns boolean indicating if the object is a Filter
-     */
-    isFilter(obj) {
-        return (typeof obj === "object" && obj !== null && "name" in obj && "value" in obj);
-    }
-    /**
-     * Helper to convert Record<string,unknown> to a Filter[]
-     * @param record: a key-value Record collection
-     * @returns Record as a Filter[]
-     */
-    convertToFilters(record) {
-        return Object.entries(record).map(([name, value]) => ({
-            name,
-            value,
-            operator: "=",
-        }));
-    }
-    /**
-     * Input santisation method for filters, as FilterType is not required to be
-     * Filter[], but we want to use Filter[] internally.
-     * @param record: the proposed filter
-     * @returns A Filter[], which may be empty
-     */
-    asFilters(record) {
-        if (!record) {
-            return [];
-        }
-        // If record is already an array
-        if (Array.isArray(record)) {
-            return record.flatMap((item) => {
-                // Check if item is a Filter before passing it to convertToFilters
-                if (this.isFilter(item)) {
-                    return [item];
-                }
-                else {
-                    // Here item is treated as Record<string, unknown>
-                    return this.convertToFilters(item);
-                }
-            });
-        }
-        // If record is a single Filter object, return it in an array
-        if (this.isFilter(record)) {
-            return [record];
-        }
-        // If record is a Record<string, unknown>, convert it to an array of Filter
-        return this.convertToFilters(record);
-    }
-    /**
-     * Method to build the WHERE clause of a CQL query, using bind variable ?
-     * @param filters list of filters to include in the WHERE clause
-     * @returns The WHERE clause
-     */
-    buildWhereClause(filters) {
-        if (!filters || filters.length === 0) {
-            return "";
-        }
-        const whereConditions = filters.map(({ name, operator = "=" }) => `${name} ${operator} ?`);
-        return `WHERE ${whereConditions.join(" AND ")}`;
-    }
-    /**
-     * Method to build an CQL query for searching for similar vectors in the
-     * Cassandra database.
-     * @param query The query vector.
-     * @param k The number of similar vectors to return.
-     * @param filters
-     * @returns The CQL query string.
-     */
-    buildSearchQuery(filters) {
-        const whereClause = filters ? this.buildWhereClause(filters) : "";
-        const cqlQuery = `SELECT ${this.selectColumns}, text, similarity_${this.vectorType}(vector, ?) AS similarity_score
-                        FROM ${this.keyspace}.${this.table} ${whereClause} ORDER BY vector ANN OF ? LIMIT ?`;
-        return cqlQuery;
-    }
-    /**
-     * Method for inserting vectors and documents into the Cassandra database in a batch.
-     * @param batchVectors The list of vectors to insert.
-     * @param batchDocuments The list of documents to insert.
-     * @returns Promise that resolves when the batch has been inserted.
-     */
-    async executeInsert(batchVectors, batchDocuments) {
-        // Input validation: Check if the lengths of batchVectors and batchDocuments are the same
-        if (batchVectors.length !== batchDocuments.length) {
-            throw new Error(`The lengths of vectors (${batchVectors.length}) and documents (${batchDocuments.length}) must be the same.`);
-        }
-        // Initialize an array to hold query objects
-        const queries = [];
-        // Loop through each vector and document in the batch
-        for (let i = 0; i < batchVectors.length; i += 1) {
-            // Convert the list of numbers to a Float32Array, the driver's expected format of a vector
-            const preparedVector = new Float32Array(batchVectors[i]);
-            // Retrieve the corresponding document
-            const document = batchDocuments[i];
-            // Extract metadata column names and values from the document
-            const metadataColNames = Object.keys(document.metadata);
-            const metadataVals = Object.values(document.metadata);
-            // Prepare the metadata columns string for the query, if metadata exists
-            const metadataInsert = metadataColNames.length > 0 ? ", " + metadataColNames.join(", ") : "";
-            // Construct the query string and parameters
-            const query = {
-                query: `INSERT INTO ${this.keyspace}.${this.table} (vector, text${metadataInsert})
-                VALUES (?, ?${", ?".repeat(metadataColNames.length)})`,
-                params: [preparedVector, document.pageContent, ...metadataVals],
-            };
-            // Add the query to the list
-            queries.push(query);
-        }
-        // Execute the queries: use a batch if multiple, otherwise execute a single query
-        if (queries.length === 1) {
-            await this.client.execute(queries[0].query, queries[0].params, {
-                prepare: true,
-            });
-        }
-        else {
-            await this.client.batch(queries, { prepare: true, logged: false });
-        }
-    }
-    /**
-     * Method for inserting vectors and documents into the Cassandra database in
-     * parallel, keeping within maxConcurrency number of active insert statements.
-     * @param vectors The vectors to insert.
-     * @param documents The documents to insert.
-     * @returns Promise that resolves when the documents have been added.
-     */
-    async insertAll(vectors, documents) {
-        // Input validation: Check if the lengths of vectors and documents are the same
-        if (vectors.length !== documents.length) {
-            throw new Error(`The lengths of vectors (${vectors.length}) and documents (${documents.length}) must be the same.`);
-        }
-        // Early exit: If there are no vectors or documents to insert, return immediately
-        if (vectors.length === 0) {
-            return;
-        }
-        // Ensure the store is initialized before proceeding
-        if (!this.isInitialized) {
-            await this.initialize();
-        }
-        // Initialize an array to hold promises for each batch insert
-        const insertPromises = [];
-        // Buffers to hold the current batch of vectors and documents
-        let currentBatchVectors = [];
-        let currentBatchDocuments = [];
-        // Loop through each vector/document pair to insert; we use
-        // <= vectors.length to ensure the last batch is inserted
-        for (let i = 0; i <= vectors.length; i += 1) {
-            // Check if we're still within the array boundaries
-            if (i < vectors.length) {
-                // Add the current vector and document to the batch
-                currentBatchVectors.push(vectors[i]);
-                currentBatchDocuments.push(documents[i]);
-            }
-            // Check if we've reached the batch size or end of the array
-            if (currentBatchVectors.length >= this.batchSize ||
-                i === vectors.length) {
-                // Only proceed if there are items in the current batch
-                if (currentBatchVectors.length > 0) {
-                    // Create copies of the current batch arrays to use in the async insert operation
-                    const batchVectors = [...currentBatchVectors];
-                    const batchDocuments = [...currentBatchDocuments];
-                    // Execute the insert using the AsyncCaller - it will handle concurrency and queueing.
-                    insertPromises.push(this.asyncCaller.call(() => this.executeInsert(batchVectors, batchDocuments)));
-                    // Clear the current buffers for the next iteration
-                    currentBatchVectors = [];
-                    currentBatchDocuments = [];
-                }
-            }
-        }
-        // Wait for all insert operations to complete.
-        await Promise.all(insertPromises);
-    }
-}
-exports.CassandraStore = CassandraStore;
+__exportStar(require("@langchain/community/vectorstores/cassandra"), exports);