@oracle/langchain-oracledb 0.1.1

package/dist/vectorstores.cjs

@@ -0,0 +1,658 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OracleVS = exports.dropTablePurge = exports.createIndex = exports.createTable = exports.DistanceStrategy = exports.generateWhereClause = void 0;
+const oracledb_1 = __importDefault(require("oracledb"));
+const crypto_1 = require("crypto");
+const vectorstores_1 = require("@langchain/core/vectorstores");
+const documents_1 = require("@langchain/core/documents");
+const math_1 = require("@langchain/core/utils/math");
+function generateWhereClause(dbFilter, bindValues) {
+    // Handle $and
+    if ("$and" in dbFilter && Array.isArray(dbFilter.$and)) {
+        const andConditions = dbFilter.$and.map((cond) => generateWhereClause(cond, bindValues));
+        return `(${andConditions.join(" AND ")})`;
+    }
+    // Handle $or
+    if ("$or" in dbFilter && Array.isArray(dbFilter.$or)) {
+        const orConditions = dbFilter.$or.map((cond) => generateWhereClause(cond, bindValues));
+        return `(${orConditions.join(" OR ")})`;
+    }
+    // Otherwise, normal object with key: condition(s)
+    const conditions = [];
+    for (const [col, val] of Object.entries(dbFilter)) {
+        if (typeof val === "object" && val !== null && !Array.isArray(val)) {
+            // Operator-based filters ($eq, $lte, $in,...)
+            for (const [op, operand] of Object.entries(val)) {
+                conditions.push(generateOperatorCondition(col, op, operand, bindValues));
+            }
+        }
+        else if (Array.isArray(val)) {
+            // shorthand { tags: ["a","b"] } → IN
+            conditions.push(generateOperatorCondition(col, "$in", val, bindValues));
+        }
+        else {
+            // shorthand { name: "John" } → equality
+            conditions.push(generateOperatorCondition(col, "$eq", val, bindValues));
+        }
+    }
+    return conditions.length > 1
+        ? `(${conditions.join(" AND ")})`
+        : conditions[0];
+}
+exports.generateWhereClause = generateWhereClause;
+function generateOperatorCondition(column, operator, value, bindValues) {
+    switch (operator) {
+        case "$eq":
+            return jsonCompare(column, "=", value, bindValues);
+        case "$ne":
+            return jsonCompare(column, "!=", value, bindValues);
+        case "$lt":
+            return jsonCompare(column, "<", value, bindValues);
+        case "$lte":
+            return jsonCompare(column, "<=", value, bindValues);
+        case "$gt":
+            return jsonCompare(column, ">", value, bindValues);
+        case "$gte":
+            return jsonCompare(column, ">=", value, bindValues);
+        case "$in": {
+            if (!Array.isArray(value))
+                throw new Error("$in requires array");
+            const inClauses = value.map((v) => jsonCompare(column, "=", v, bindValues));
+            return `(${inClauses.join(" OR ")})`;
+        }
+        case "$nin": {
+            if (!Array.isArray(value))
+                throw new Error("$nin requires array");
+            const ninClauses = value.map((v) => jsonCompare(column, "!=", v, bindValues));
+            return `(${ninClauses.join(" AND ")})`;
+        }
+        case "$between": {
+            if (!Array.isArray(value) || value.length !== 2)
+                throw new Error("$between requires [low, high]");
+            const [low, high] = value;
+            bindValues.push(low, high);
+            const posLow = bindValues.length - 1;
+            const posHigh = bindValues.length;
+            return `JSON_EXISTS(metadata, '$.${column}?(@ >= $low && @ <= $high)' PASSING :${posLow} AS "low", :${posHigh} AS "high")`;
+        }
+        case "$exists":
+            if (value) {
+                return `JSON_EXISTS(metadata, '$.${column}')`;
+            }
+            else {
+                return `NOT JSON_EXISTS(metadata, '$.${column}')`;
+            }
+        default:
+            throw new Error(`Unsupported operator: ${operator}`);
+    }
+}
+// Helper: build JSON_EXISTS clause for comparison
+function jsonCompare(column, op, value, bindValues) {
+    bindValues.push(value);
+    const pos = bindValues.length;
+    const alias = `val${pos}`; // unique bind alias
+    const operator = op === "=" ? "==" : op; // Oracle requires '==' for equality
+    return `JSON_EXISTS(metadata, '$.${column}?(@ ${operator} $${alias})' PASSING :${pos} AS "${alias}")`;
+}
+exports.DistanceStrategy = {
+    COSINE: "COSINE",
+    DOT_PRODUCT: "DOT",
+    EUCLIDEAN: "EUCLIDEAN",
+    MANHATTAN: "MANHATTAN",
+    HAMMING: "HAMMING",
+    EUCLIDEAN_SQUARED: "EUCLIDEAN_SQUARED",
+};
+function handleError(error) {
+    // Type guard to check if the error is an object and has 'name' and 'message' properties
+    if (typeof error === "object" &&
+        error !== null &&
+        "name" in error &&
+        "message" in error) {
+        const err = error; // Type assertion based on guarded checks
+        // Handle specific error types based on the name property
+        switch (err.name) {
+            case "RuntimeError":
+                throw new Error("Database operation failed due to a runtime error.");
+            case "ValidationError":
+                throw new Error("Operation failed due to a validation error.");
+            default:
+                throw new Error(`An unexpected error occurred during the operation. ${error}`);
+        }
+    }
+    throw new Error(`An unknown and unexpected error occurred. ${error}`);
+}
+function isPool(client) {
+    return "getConnection" in client;
+}
+function quoteIdentifier(identifier) {
+    const name = identifier.trim();
+    const validateRegex = /^(?:"[^"]+"|[^".]+)(?:\.(?:"[^"]+"|[^".]+))*$/;
+    if (!validateRegex.test(name)) {
+        throw new Error(`Identifier name ${identifier} is not valid.`);
+    }
+    // extracts parts of the identifier with quoted and unquoted.
+    const matchRegex = /"([^"]+)"|([^".]+)/g;
+    const groups = [];
+    for (const match of name.matchAll(matchRegex)) {
+        groups.push(match[1] || match[2]);
+    }
+    const quotedParts = groups.map((g) => `"${g}"`);
+    return quotedParts.join(".");
+}
+async function createTable(connection, tableName, embeddingDim) {
+    const colsDict = {
+        id: "RAW(16) DEFAULT SYS_GUID() PRIMARY KEY",
+        embedding: `vector(${embeddingDim}, FLOAT32)`,
+        text: "CLOB",
+        metadata: "JSON",
+    };
+    try {
+        const ddlBody = Object.entries(colsDict)
+            .map(([colName, colType]) => `${colName} ${colType}`)
+            .join(", ");
+        const ddl = `CREATE TABLE IF NOT EXISTS ${quoteIdentifier(tableName)}
+                   (
+                       ${ddlBody}
+                   )`;
+        await connection.execute(ddl);
+    }
+    catch (error) {
+        handleError(error);
+    }
+}
+exports.createTable = createTable;
+function _getIndexName(baseName) {
+    const uniqueId = crypto.randomUUID().replace(/-/g, "");
+    return `${baseName}_${uniqueId}`;
+}
+async function createIndex(client, vectorStore, params) {
+    const idxType = params?.idxType || "HNSW";
+    if (idxType === "IVF") {
+        await createIVFIndex(client, vectorStore, params);
+    }
+    else {
+        await createHNSWIndex(client, vectorStore, params);
+    }
+}
+exports.createIndex = createIndex;
+async function createHNSWIndex(connection, oraclevs, params) {
+    try {
+        const defaults = {
+            idxName: "HNSW",
+            idxType: "HNSW",
+            neighbors: 32,
+            efConstruction: 200,
+            accuracy: 90,
+            parallel: 8,
+        };
+        // if params then copy params to config
+        const config = params
+            ? { ...params }
+            : { ...defaults };
+        // Ensure compulsory parts are included
+        const compulsoryKeys = ["idxName", "parallel"];
+        for (const key of compulsoryKeys) {
+            if (!(key in config)) {
+                if (key === "idxName") {
+                    config[key] = _getIndexName(defaults[key]);
+                }
+                else {
+                    config[key] = defaults[key];
+                }
+            }
+        }
+        // Validate keys in config against defaults
+        for (const key of Object.keys(config)) {
+            if (!(key in defaults)) {
+                throw new Error(`Invalid parameter: ${key}`);
+            }
+        }
+        const { idxName } = config;
+        const baseSql = `CREATE VECTOR INDEX IF NOT EXISTS ${quoteIdentifier(idxName)}
+                              ON ${oraclevs.tableName}(embedding) 
+                              ORGANIZATION INMEMORY NEIGHBOR GRAPH`;
+        const accuracyPart = config.accuracy
+            ? ` WITH TARGET ACCURACY ${config.accuracy}`
+            : "";
+        const distancePart = ` DISTANCE ${oraclevs.distanceStrategy}`;
+        let parametersPart = "";
+        if ("neighbors" in config && "efConstruction" in config) {
+            parametersPart = ` parameters (type ${config.idxType}, 
+                                     neighbors ${config.neighbors}, 
+                                     efConstruction ${config.efConstruction})`;
+        }
+        else if ("neighbors" in config && !("efConstruction" in config)) {
+            config.efConstruction = defaults.efConstruction;
+            parametersPart = ` parameters (type ${config.idxType}, 
+                                     neighbors ${config.neighbors}, 
+                                     efConstruction ${config.efConstruction})`;
+        }
+        else if (!("neighbors" in config) && "efConstruction" in config) {
+            config.neighbors = defaults.neighbors;
+            parametersPart = ` parameters (type ${config.idxType}, 
+                                     neighbors ${config.neighbors}, 
+                                     efConstruction ${config.efConstruction})`;
+        }
+        const parallelPart = ` PARALLEL ${config.parallel}`;
+        const ddl = baseSql + accuracyPart + distancePart + parametersPart + parallelPart;
+        await connection.execute(ddl);
+    }
+    catch (error) {
+        handleError(error);
+    }
+}
+async function createIVFIndex(connection, oraclevs, params) {
+    try {
+        const defaults = {
+            idxName: "IVF",
+            idxType: "IVF",
+            neighborPart: 32,
+            accuracy: 90,
+            parallel: 8,
+        };
+        // Combine defaults with any provided params. Note: params could contain keys not explicitly declared in IndexConfig
+        const config = params
+            ? { ...params }
+            : { ...defaults };
+        // Ensure compulsory parts are included
+        const compulsoryKeys = ["idxName", "parallel"];
+        for (const key of compulsoryKeys) {
+            if (!(key in config)) {
+                if (key === "idxName") {
+                    config[key] = _getIndexName(defaults[key]);
+                }
+                else {
+                    config[key] = defaults[key];
+                }
+            }
+        }
+        // Validate keys in config against defaults
+        for (const key of Object.keys(config)) {
+            if (!(key in defaults)) {
+                throw new Error(`Invalid parameter: ${key}`);
+            }
+        }
+        // Base SQL statement
+        const { idxName } = config;
+        const baseSql = `CREATE VECTOR INDEX IF NOT EXISTS ${quoteIdentifier(idxName)}
+                              ON ${oraclevs.tableName}(embedding) 
+                              ORGANIZATION NEIGHBOR PARTITIONS`;
+        // Optional parts depending on parameters
+        const accuracyPart = config.accuracy
+            ? ` WITH TARGET ACCURACY ${config.accuracy}`
+            : "";
+        const distancePart = ` DISTANCE ${oraclevs.distanceStrategy}`;
+        let parametersPart = "";
+        if ("idxType" in config && "neighborPart" in config) {
+            parametersPart = ` PARAMETERS (type ${config.idxType}, 
+                         neighbor partitions ${config.neighborPart})`;
+        }
+        // Always included part for parallel - assuming parallel is compulsory and always included
+        const parallelPart = ` PARALLEL ${config.parallel}`;
+        // Combine all parts
+        const ddl = baseSql + accuracyPart + distancePart + parametersPart + parallelPart;
+        await connection.execute(ddl);
+    }
+    catch (error) {
+        handleError(error);
+    }
+}
+async function dropTablePurge(connection, tableName) {
+    try {
+        const ddl = `DROP TABLE IF EXISTS ${quoteIdentifier(tableName)} PURGE`;
+        await connection.execute(ddl);
+    }
+    catch (error) {
+        handleError(error);
+    }
+}
+exports.dropTablePurge = dropTablePurge;
+class OracleVS extends vectorstores_1.VectorStore {
+    _vectorstoreType() {
+        return "oraclevs";
+    }
+    constructor(embeddings, dbConfig) {
+        super(embeddings, dbConfig);
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "embeddingDimension", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tableName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "distanceStrategy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: exports.DistanceStrategy.COSINE
+        });
+        Object.defineProperty(this, "filter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "query", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        try {
+            this.client = dbConfig.client;
+            this.tableName = quoteIdentifier(dbConfig.tableName);
+            this.distanceStrategy =
+                dbConfig.distanceStrategy ?? this.distanceStrategy;
+            this.query = dbConfig.query;
+            this.filter = dbConfig.filter;
+        }
+        catch (error) {
+            handleError(error);
+        }
+    }
+    async getEmbeddingDimension(query) {
+        const embeddingVector = await this.embeddings.embedQuery(query);
+        return embeddingVector.length;
+    }
+    async initialize() {
+        let connection = null;
+        try {
+            this.embeddingDimension = await this.getEmbeddingDimension(this.query);
+            connection = await this.getConnection();
+            await createTable(connection, this.tableName, this.embeddingDimension);
+        }
+        catch (error) {
+            handleError(error);
+        }
+        finally {
+            if (connection)
+                await this.retConnection(connection);
+        }
+    }
+    async getConnection() {
+        try {
+            if (isPool(this.client)) {
+                return await this.client.getConnection();
+            }
+            return this.client;
+        }
+        catch (error) {
+            handleError(error);
+        }
+    }
+    // Close connection or return it to the pool
+    async retConnection(connection) {
+        try {
+            // If the client is a pool, close the connection (return it to the pool)
+            if (isPool(this.client)) {
+                await connection.close();
+            }
+        }
+        catch (error) {
+            console.error("Error in retConnection:", error);
+            throw error;
+        }
+    }
+    /**
+     * Method to add vectors to the Oracle database.
+     * @param vectors The vectors to add.
+     * @param documents The documents associated with the vectors.
+     * @param options
+     * @returns Promise that resolves when the vectors have been added.
+     */
+    async addVectors(vectors, documents, options) {
+        if (vectors.length === 0) {
+            throw new Error("Vectors input null. Nothing to add...");
+        }
+        const ids = options?.ids;
+        let connection = null;
+        try {
+            // Ensure there are IDs for all documents
+            if (ids !== undefined && ids.length !== vectors.length) {
+                throw new Error("The number of ids must match the number of vectors provided.");
+            }
+            connection = await this.getConnection();
+            const binds = [];
+            for (let index = 0; index < documents.length; index += 1) {
+                const doc = documents[index];
+                let processedId;
+                if (ids) {
+                    processedId = (0, crypto_1.createHash)("sha256")
+                        .update(ids[index])
+                        .digest("hex")
+                        .substring(0, 16)
+                        .toUpperCase();
+                }
+                else {
+                    const sourceId = doc.metadata.id ?? crypto.randomUUID();
+                    processedId = (0, crypto_1.createHash)("sha256")
+                        .update(sourceId)
+                        .digest("hex")
+                        .substring(0, 16)
+                        .toUpperCase();
+                }
+                const idBuffer = Buffer.from(processedId, "hex");
+                const bind = {
+                    id: idBuffer,
+                    text: doc.pageContent,
+                    metadata: doc.metadata,
+                    embedding: new Float32Array(vectors[index]),
+                };
+                binds.push(bind);
+            }
+            const sql = `INSERT INTO ${this.tableName} (id, embedding, text, metadata )
+               VALUES (:id, :embedding, :text, :metadata)`;
+            const options = {
+                bindDefs: {
+                    id: { type: oracledb_1.default.BUFFER, maxSize: 24 },
+                    text: { type: oracledb_1.default.STRING, maxSize: 10000000 },
+                    metadata: { type: oracledb_1.default.DB_TYPE_JSON },
+                    embedding: { type: oracledb_1.default.DB_TYPE_VECTOR },
+                },
+                autoCommit: false,
+            };
+            await connection.executeMany(sql, binds, options);
+            // Commit once all inserts are queued up
+            await connection.commit();
+            console.log("All documents have been inserted and committed.");
+        }
+        catch (error) {
+            handleError(error);
+        }
+        finally {
+            if (connection) {
+                await this.retConnection(connection);
+            }
+        }
+        return ids;
+    }
+    async addDocuments(documents, options) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
+    }
+    /**
+     * Method to search for vectors that are similar to a given query vector.
+     * @param query The query vector.
+     * @param k The number of similar vectors to return.
+     * @param filter Optional filter for the search results.
+     * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
+     */
+    async similaritySearchByVectorReturningEmbeddings(query, k = 4, filter) {
+        const docsScoresAndEmbeddings = [];
+        let connection = null;
+        try {
+            const convertedEmbedding = new Float32Array(query);
+            const bindValues = [convertedEmbedding];
+            let sqlQuery = `
+      SELECT id, 
+        text,
+        metadata,
+        vector_distance(embedding, :1, ${this.distanceStrategy}) as distance,
+        embedding
+      FROM ${this.tableName} `;
+            if (filter && Object.keys(filter).length > 0) {
+                sqlQuery += ` WHERE ${generateWhereClause(filter, bindValues)}`;
+            }
+            bindValues.push(k);
+            sqlQuery += ` ORDER BY distance FETCH APPROX FIRST :${bindValues.length} ROWS ONLY `;
+            // Execute the query
+            connection = await this.getConnection();
+            const resultSet = await connection.execute(sqlQuery, bindValues, {
+                fetchInfo: {
+                    TEXT: { type: oracledb_1.default.STRING },
+                },
+            });
+            if (Array.isArray(resultSet.rows) && resultSet.rows.length > 0) {
+                const rows = resultSet.rows;
+                for (let idx = 0; idx < resultSet.rows.length; idx += 1) {
+                    const row = rows[idx];
+                    const text = row[1];
+                    const metadata = row[2];
+                    const distance = row[3];
+                    const embedding = row[4];
+                    const document = new documents_1.Document({
+                        pageContent: text || "",
+                        metadata: metadata || {},
+                        id: row[0].toString("hex"),
+                    });
+                    docsScoresAndEmbeddings.push([document, distance, embedding]);
+                }
+            }
+            else {
+                // Throw an exception if no rows are found
+                throw new Error("No rows found.");
+            }
+        }
+        finally {
+            if (connection) {
+                await connection.close();
+            }
+        }
+        return docsScoresAndEmbeddings;
+    }
+    async similaritySearchVectorWithScore(query, k, filter) {
+        const docsScoresAndEmbeddings = await this.similaritySearchByVectorReturningEmbeddings(query, k, filter);
+        return docsScoresAndEmbeddings.map(([document, score]) => [
+            document,
+            score,
+        ]);
+    }
+    /**
+     * Return documents selected using the maximal marginal relevance.
+     * Maximal marginal relevance optimizes for similarity to the query AND diversity
+     * among selected documents.
+     *
+     * @param {string} query - Text to look up documents similar to.
+     * @param options
+     * @param {number} options.k - Number of documents to return.
+     * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
+     * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
+     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
+     * @param {this["FilterType"]} options.filter - Optional filter
+     * @param _callbacks
+     *
+     * @returns {Promise<DocumentInterface[]>} - List of documents selected by maximal marginal relevance.
+     */
+    async maxMarginalRelevanceSearch(query, options) {
+        const embedding = await this.embeddings.embedQuery(query);
+        return await this.maxMarginalRelevanceSearchByVector(embedding, options);
+    }
+    async maxMarginalRelevanceSearchByVector(embedding, options) {
+        // Fetch documents and their scores. This calls the previously adapted function.
+        const docsAndScores = await this.maxMarginalRelevanceSearchWithScoreByVector(embedding, options);
+        // Extract and return only the documents from the results
+        return docsAndScores.map((ds) => ds.document);
+    }
+    async maxMarginalRelevanceSearchWithScoreByVector(embedding, options) {
+        // Fetch documents and their scores.
+        const docsScoresEmbeddings = await this.similaritySearchByVectorReturningEmbeddings(embedding, options.fetchK, options.filter);
+        if (!docsScoresEmbeddings.length) {
+            return [];
+        }
+        // Split documents, scores, and embeddings
+        const documents = docsScoresEmbeddings.map(([document]) => document);
+        const scores = docsScoresEmbeddings.map(([, score]) => score);
+        const embeddings = docsScoresEmbeddings.map(([, , embedding]) => new Float32Array(embedding));
+        // Convert all embeddings to Float32Array for consistency
+        const consistentEmbeddings = embeddings.map((embedding) => Array.from(embedding));
+        const queryEmbedding = Array.from(embedding);
+        // Ensure lambdaMult has a default value if not provided
+        const lambdaMult = options.lambda ?? 0.5;
+        const mmrSelectedIndices = (0, math_1.maximalMarginalRelevance)(queryEmbedding, consistentEmbeddings, lambdaMult, options.k);
+        // Filter documents based on MMR-selected indices and map scores
+        return mmrSelectedIndices.map((index) => ({
+            document: documents[index],
+            score: scores[index],
+        }));
+    }
+    async delete(params) {
+        let connection = null;
+        try {
+            connection = await this.getConnection();
+            const options = { autoCommit: true };
+            if (params.ids && params.ids.length > 0) {
+                // Dynamically create placeholders
+                const placeholders = params.ids
+                    .map((_, index) => `:${index + 1}`)
+                    .join(",");
+                // Prepare the query
+                const query = `DELETE FROM ${this.tableName} WHERE id IN (${placeholders})`;
+                // Execute the query with the IDs as bind parameters
+                await connection.execute(query, [...params.ids], options);
+            }
+            else if (params.deleteAll) {
+                await connection.execute(`TRUNCATE TABLE ${this.tableName}`, [], options);
+            }
+        }
+        catch (error) {
+            handleError(error);
+        }
+        finally {
+            if (connection)
+                await connection.close();
+        }
+    }
+    static async fromDocuments(documents, embeddings, dbConfig) {
+        const { client } = dbConfig;
+        if (!client)
+            throw new Error("client parameter is required...");
+        try {
+            const vss = new OracleVS(embeddings, dbConfig);
+            await vss.initialize();
+            const texts = documents.map(({ pageContent }) => pageContent);
+            const vectors = await embeddings.embedDocuments(texts);
+            // Assuming a method exists to handle adding texts and metadata appropriately
+            await vss.addVectors(vectors, documents);
+            return vss;
+        }
+        catch (error) {
+            handleError(error);
+        }
+    }
+    /**
+     *
+     * @returns Promise that resolves when all connections
+     * inside the pool are terminated.
+     */
+    async end() {
+        if (isPool(this.client)) {
+            await this.client?.close();
+        }
+    }
+}
+exports.OracleVS = OracleVS;
+//# sourceMappingURL=vectorstores.cjs.map