@pleaseai/context-please-core 0.6.0 → 0.7.0

package/dist/vectordb/libsql-vectordb.js

@@ -0,0 +1,803 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LibSQLVectorDatabase = void 0;
+const os = __importStar(require("node:os"));
+const path = __importStar(require("node:path"));
+const client_1 = require("@libsql/client");
+const fs = __importStar(require("fs-extra"));
+const base_vector_database_1 = require("./base/base-vector-database");
+const simple_bm25_1 = require("./sparse/simple-bm25");
+/**
+ * LibSQL Vector Database implementation for local-only deployments
+ *
+ * Features:
+ * - Pure JavaScript SDK (no native bindings required)
+ * - Full document deletion support via SQL DELETE
+ * - Query filtering support via SQL WHERE clauses
+ * - Single SQLite file per collection
+ * - Hybrid search with BM25 sparse vectors
+ * - RRF (Reciprocal Rank Fusion) reranking
+ *
+ * Architecture:
+ * - Dense vectors: Stored in F32_BLOB columns with DiskANN indexing
+ * - Sparse vectors: Stored as JSON (indices/values) for BM25
+ * - Hybrid search: Combines both using RRF fusion
+ *
+ * Storage structure:
+ * ~/.context/libsql-indexes/
+ *   └── {collection_name}.db    # SQLite database file
+ *
+ * Key advantages over FAISS:
+ * - Document deletion IS supported (SQL DELETE)
+ * - Query filters ARE supported (SQL WHERE)
+ * - No native bindings required
+ */
+class LibSQLVectorDatabase extends base_vector_database_1.BaseVectorDatabase {
+    constructor(config) {
+        const configWithDefaults = {
+            ...config,
+            storageDir: config.storageDir || path.join(os.homedir(), '.context', 'libsql-indexes'),
+            walMode: config.walMode !== false,
+            cacheSize: config.cacheSize || 2000,
+        };
+        super(configWithDefaults);
+        this.clients = new Map();
+        this.bm25Generators = new Map();
+        this.metadataCache = new Map();
+    }
+    get storageDir() {
+        return this.config.storageDir;
+    }
+    /**
+     * Initialize LibSQL storage directory
+     */
+    async initialize() {
+        try {
+            console.log('[LibSQLDB] Initializing LibSQL storage at:', this.storageDir);
+            await fs.ensureDir(this.storageDir);
+            console.log('[LibSQLDB] LibSQL storage initialized');
+        }
+        catch (error) {
+            const errorMsg = `Failed to initialize LibSQL storage at ${this.storageDir}: ${error.message}`;
+            console.error(`[LibSQLDB] ${errorMsg}`);
+            if (error.code === 'EACCES') {
+                throw new Error(`${errorMsg}\nPermission denied. Check directory permissions.`);
+            }
+            else if (error.code === 'ENOSPC') {
+                throw new Error(`${errorMsg}\nDisk space exhausted. Free up disk space and try again.`);
+            }
+            throw new Error(errorMsg);
+        }
+    }
+    /**
+     * LibSQL collections are loaded on-demand when accessed
+     */
+    async ensureLoaded(collectionName) {
+        if (this.clients.has(collectionName)) {
+            return;
+        }
+        const dbPath = this.getDbPath(collectionName);
+        if (!(await fs.pathExists(dbPath))) {
+            throw new Error(`Collection ${collectionName} does not exist`);
+        }
+        await this.loadCollection(collectionName);
+    }
+    /**
+     * Get database file path for a collection
+     */
+    getDbPath(collectionName) {
+        return path.join(this.storageDir, `${collectionName}.db`);
+    }
+    /**
+     * Load collection from disk
+     */
+    async loadCollection(collectionName) {
+        const dbPath = this.getDbPath(collectionName);
+        console.log('[LibSQLDB] Loading collection:', collectionName);
+        try {
+            const client = (0, client_1.createClient)({ url: `file:${dbPath}` });
+            this.clients.set(collectionName, client);
+            // Load metadata
+            const result = await client.execute('SELECT key, value FROM _metadata');
+            const metadata = {};
+            for (const row of result.rows) {
+                metadata[row.key] = row.value;
+            }
+            const collectionMetadata = {
+                dimension: Number.parseInt(metadata.dimension, 10),
+                isHybrid: metadata.isHybrid === 'true',
+                documentCount: Number.parseInt(metadata.documentCount || '0', 10),
+                createdAt: metadata.createdAt,
+            };
+            this.metadataCache.set(collectionName, collectionMetadata);
+            // Load BM25 if hybrid collection
+            if (collectionMetadata.isHybrid) {
+                const bm25Path = path.join(this.storageDir, `${collectionName}_bm25.json`);
+                if (await fs.pathExists(bm25Path)) {
+                    const bm25Json = await fs.readFile(bm25Path, 'utf-8');
+                    const bm25 = simple_bm25_1.SimpleBM25.fromJSON(bm25Json);
+                    this.bm25Generators.set(collectionName, bm25);
+                }
+                else {
+                    console.warn(`[LibSQLDB] BM25 model file missing for hybrid collection ${collectionName}. Sparse search will be unavailable until re-indexing.`);
+                    this.bm25Generators.set(collectionName, new simple_bm25_1.SimpleBM25(this.config.bm25Config));
+                }
+            }
+            console.log('[LibSQLDB] Loaded collection:', collectionName);
+        }
+        catch (error) {
+            console.error(`[LibSQLDB] Failed to load collection ${collectionName}:`, error.message);
+            throw error;
+        }
+    }
+    /**
+     * Get or create client for a collection
+     */
+    async getClient(collectionName) {
+        await this.ensureLoaded(collectionName);
+        return this.clients.get(collectionName);
+    }
+    /**
+     * Save BM25 model for a collection
+     */
+    async saveBM25(collectionName) {
+        const bm25 = this.bm25Generators.get(collectionName);
+        if (!bm25) {
+            return;
+        }
+        const bm25Path = path.join(this.storageDir, `${collectionName}_bm25.json`);
+        try {
+            await fs.writeFile(bm25Path, bm25.toJSON(), 'utf-8');
+        }
+        catch (error) {
+            console.error(`[LibSQLDB] Failed to save BM25 model for ${collectionName}:`, error.message);
+            throw new Error(`Failed to save BM25 model for ${collectionName}: ${error.message}`);
+        }
+    }
+    /**
+     * Update document count in metadata
+     */
+    async updateDocumentCount(collectionName) {
+        const client = this.clients.get(collectionName);
+        if (!client) {
+            console.warn(`[LibSQLDB] Cannot update document count: client not found for ${collectionName}`);
+            return;
+        }
+        try {
+            const result = await client.execute('SELECT COUNT(*) as count FROM documents');
+            const count = Number(result.rows[0].count);
+            if (Number.isNaN(count)) {
+                console.error(`[LibSQLDB] Invalid document count result for ${collectionName}`);
+                return;
+            }
+            await client.execute({
+                sql: 'INSERT OR REPLACE INTO _metadata (key, value) VALUES (?, ?)',
+                args: ['documentCount', String(count)],
+            });
+            const metadata = this.metadataCache.get(collectionName);
+            if (metadata) {
+                metadata.documentCount = count;
+            }
+        }
+        catch (error) {
+            console.error(`[LibSQLDB] Failed to update document count for ${collectionName}:`, error.message);
+            // Don't throw - this is a non-critical metadata update
+        }
+    }
+    /**
+     * Create collection with dense vectors only
+     */
+    async createCollection(collectionName, dimension, _description) {
+        await this.ensureInitialized();
+        const dbPath = this.getDbPath(collectionName);
+        if (await fs.pathExists(dbPath)) {
+            throw new Error(`Collection ${collectionName} already exists`);
+        }
+        console.log('[LibSQLDB] Creating collection:', collectionName);
+        console.log('[LibSQLDB] Vector dimension:', dimension);
+        const client = (0, client_1.createClient)({ url: `file:${dbPath}` });
+        // Configure SQLite settings
+        if (this.config.walMode) {
+            await client.execute('PRAGMA journal_mode=WAL');
+        }
+        await client.execute(`PRAGMA cache_size=${this.config.cacheSize}`);
+        // Create metadata table
+        await client.execute(`
+      CREATE TABLE _metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      )
+    `);
+        // Create documents table with vector column
+        await client.execute(`
+      CREATE TABLE documents (
+        id TEXT PRIMARY KEY,
+        content TEXT NOT NULL,
+        relative_path TEXT NOT NULL,
+        start_line INTEGER NOT NULL,
+        end_line INTEGER NOT NULL,
+        file_extension TEXT NOT NULL,
+        metadata TEXT NOT NULL,
+        dense_vector F32_BLOB(${dimension})
+      )
+    `);
+        // Create vector index
+        await client.execute('CREATE INDEX idx_dense ON documents(libsql_vector_idx(dense_vector))');
+        // Insert metadata
+        await client.batch([
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['dimension', String(dimension)] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['isHybrid', 'false'] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['createdAt', new Date().toISOString()] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['documentCount', '0'] },
+        ]);
+        this.clients.set(collectionName, client);
+        this.metadataCache.set(collectionName, {
+            dimension,
+            isHybrid: false,
+            documentCount: 0,
+            createdAt: new Date().toISOString(),
+        });
+        console.log('[LibSQLDB] Collection created:', collectionName);
+    }
+    /**
+     * Create collection with hybrid search support (dense + sparse vectors)
+     */
+    async createHybridCollection(collectionName, dimension, _description) {
+        await this.ensureInitialized();
+        const dbPath = this.getDbPath(collectionName);
+        if (await fs.pathExists(dbPath)) {
+            throw new Error(`Collection ${collectionName} already exists`);
+        }
+        console.log('[LibSQLDB] Creating hybrid collection:', collectionName);
+        console.log('[LibSQLDB] Vector dimension:', dimension);
+        const client = (0, client_1.createClient)({ url: `file:${dbPath}` });
+        // Configure SQLite settings
+        if (this.config.walMode) {
+            await client.execute('PRAGMA journal_mode=WAL');
+        }
+        await client.execute(`PRAGMA cache_size=${this.config.cacheSize}`);
+        // Create metadata table
+        await client.execute(`
+      CREATE TABLE _metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      )
+    `);
+        // Create documents table with vector and sparse columns
+        await client.execute(`
+      CREATE TABLE documents (
+        id TEXT PRIMARY KEY,
+        content TEXT NOT NULL,
+        relative_path TEXT NOT NULL,
+        start_line INTEGER NOT NULL,
+        end_line INTEGER NOT NULL,
+        file_extension TEXT NOT NULL,
+        metadata TEXT NOT NULL,
+        dense_vector F32_BLOB(${dimension}),
+        sparse_indices TEXT,
+        sparse_values TEXT
+      )
+    `);
+        // Create vector index
+        await client.execute('CREATE INDEX idx_dense ON documents(libsql_vector_idx(dense_vector))');
+        // Insert metadata
+        await client.batch([
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['dimension', String(dimension)] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['isHybrid', 'true'] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['createdAt', new Date().toISOString()] },
+            { sql: 'INSERT INTO _metadata VALUES (?, ?)', args: ['documentCount', '0'] },
+        ]);
+        this.clients.set(collectionName, client);
+        this.metadataCache.set(collectionName, {
+            dimension,
+            isHybrid: true,
+            documentCount: 0,
+            createdAt: new Date().toISOString(),
+        });
+        // Initialize BM25 generator
+        this.bm25Generators.set(collectionName, new simple_bm25_1.SimpleBM25(this.config.bm25Config));
+        console.log('[LibSQLDB] Hybrid collection created:', collectionName);
+    }
+    /**
+     * Drop collection
+     */
+    async dropCollection(collectionName) {
+        await this.ensureInitialized();
+        console.log('[LibSQLDB] Dropping collection:', collectionName);
+        // Close client if exists
+        const client = this.clients.get(collectionName);
+        if (client) {
+            client.close();
+            this.clients.delete(collectionName);
+        }
+        // Remove from caches
+        this.metadataCache.delete(collectionName);
+        this.bm25Generators.delete(collectionName);
+        // Remove database file
+        const dbPath = this.getDbPath(collectionName);
+        if (await fs.pathExists(dbPath)) {
+            await fs.remove(dbPath);
+        }
+        // Remove BM25 file if exists
+        const bm25Path = path.join(this.storageDir, `${collectionName}_bm25.json`);
+        if (await fs.pathExists(bm25Path)) {
+            await fs.remove(bm25Path);
+        }
+        // Remove WAL files if they exist
+        const walPath = `${dbPath}-wal`;
+        const shmPath = `${dbPath}-shm`;
+        await fs.remove(walPath);
+        await fs.remove(shmPath);
+        console.log('[LibSQLDB] Collection dropped:', collectionName);
+    }
+    /**
+     * Check if collection exists
+     */
+    async hasCollection(collectionName) {
+        await this.ensureInitialized();
+        if (this.clients.has(collectionName)) {
+            return true;
+        }
+        const dbPath = this.getDbPath(collectionName);
+        return await fs.pathExists(dbPath);
+    }
+    /**
+     * List all collections
+     */
+    async listCollections() {
+        await this.ensureInitialized();
+        const collections = [];
+        if (await fs.pathExists(this.storageDir)) {
+            const entries = await fs.readdir(this.storageDir, { withFileTypes: true });
+            for (const entry of entries) {
+                if (entry.isFile() && entry.name.endsWith('.db')) {
+                    collections.push(entry.name.replace('.db', ''));
+                }
+            }
+        }
+        return collections;
+    }
+    /**
+     * Insert vector documents (dense only)
+     */
+    async insert(collectionName, documents) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        const metadata = this.metadataCache.get(collectionName);
+        if (!metadata) {
+            throw new Error(`Collection ${collectionName} metadata not found`);
+        }
+        console.log('[LibSQLDB] Inserting documents:', documents.length);
+        // Validate vector dimensions
+        for (const doc of documents) {
+            if (doc.vector.length !== metadata.dimension) {
+                throw new Error(`Vector dimension mismatch for document '${doc.id}': `
+                    + `expected ${metadata.dimension}, got ${doc.vector.length}`);
+            }
+        }
+        // Batch insert
+        const statements = documents.map((doc) => ({
+            sql: `INSERT OR REPLACE INTO documents
+            (id, content, relative_path, start_line, end_line, file_extension, metadata, dense_vector)
+            VALUES (?, ?, ?, ?, ?, ?, ?, vector32(?))`,
+            args: [
+                doc.id,
+                doc.content,
+                doc.relativePath,
+                doc.startLine,
+                doc.endLine,
+                doc.fileExtension,
+                JSON.stringify(doc.metadata),
+                `[${doc.vector.join(',')}]`,
+            ],
+        }));
+        await client.batch(statements);
+        await this.updateDocumentCount(collectionName);
+        console.log('[LibSQLDB] Inserted documents:', documents.length);
+    }
+    /**
+     * Insert hybrid vector documents (dense + sparse)
+     */
+    async insertHybrid(collectionName, documents) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        const metadata = this.metadataCache.get(collectionName);
+        if (!metadata) {
+            throw new Error(`Collection ${collectionName} metadata not found`);
+        }
+        if (!metadata.isHybrid) {
+            throw new Error(`Collection ${collectionName} is not a hybrid collection`);
+        }
+        console.log('[LibSQLDB] Inserting hybrid documents:', documents.length);
+        // Validate vector dimensions
+        for (const doc of documents) {
+            if (doc.vector.length !== metadata.dimension) {
+                throw new Error(`Vector dimension mismatch for document '${doc.id}': `
+                    + `expected ${metadata.dimension}, got ${doc.vector.length}`);
+            }
+        }
+        // Get or create BM25 generator
+        let bm25 = this.bm25Generators.get(collectionName);
+        if (!bm25) {
+            bm25 = new simple_bm25_1.SimpleBM25(this.config.bm25Config);
+            this.bm25Generators.set(collectionName, bm25);
+        }
+        // Train BM25 on all documents (existing + new)
+        const existingResult = await client.execute('SELECT content FROM documents');
+        const existingContents = existingResult.rows.map((r) => r.content);
+        const allContents = [...existingContents, ...documents.map((d) => d.content)];
+        bm25.learn(allContents);
+        // Generate sparse vectors
+        const sparseVectors = documents.map((doc) => bm25.generate(doc.content));
+        // Batch insert
+        const statements = documents.map((doc, i) => ({
+            sql: `INSERT OR REPLACE INTO documents
+            (id, content, relative_path, start_line, end_line, file_extension, metadata,
+             dense_vector, sparse_indices, sparse_values)
+            VALUES (?, ?, ?, ?, ?, ?, ?, vector32(?), ?, ?)`,
+            args: [
+                doc.id,
+                doc.content,
+                doc.relativePath,
+                doc.startLine,
+                doc.endLine,
+                doc.fileExtension,
+                JSON.stringify(doc.metadata),
+                `[${doc.vector.join(',')}]`,
+                JSON.stringify(sparseVectors[i].indices),
+                JSON.stringify(sparseVectors[i].values),
+            ],
+        }));
+        await client.batch(statements);
+        await this.updateDocumentCount(collectionName);
+        await this.saveBM25(collectionName);
+        console.log('[LibSQLDB] Inserted hybrid documents:', documents.length);
+    }
+    /**
+     * Search similar vectors (dense search only)
+     */
+    async search(collectionName, queryVector, options) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        const topK = options?.topK || 10;
+        const queryVectorStr = `[${queryVector.join(',')}]`;
+        console.log('[LibSQLDB] Searching vectors, topK:', topK);
+        // Build query with vector_top_k
+        // Note: vector_top_k returns 'id' (rowid), we calculate distance with vector_distance_cos
+        let sql = `
+      SELECT d.*, vector_distance_cos(d.dense_vector, vector32(?)) AS distance
+      FROM vector_top_k('idx_dense', vector32(?), ?) AS vt
+      JOIN documents d ON d.rowid = vt.id
+    `;
+        const args = [queryVectorStr, queryVectorStr, topK * 2];
+        // Apply filter if provided
+        if (options?.filterExpr) {
+            const whereClause = this.parseFilterExpression(options.filterExpr);
+            sql += ` WHERE ${whereClause}`;
+        }
+        sql += ' ORDER BY distance ASC LIMIT ?';
+        args.push(topK);
+        const result = await client.execute({ sql, args });
+        const searchResults = [];
+        for (const row of result.rows) {
+            const score = 1 / (1 + row.distance);
+            // Apply threshold filter
+            if (options?.threshold !== undefined && score < options.threshold) {
+                continue;
+            }
+            searchResults.push({
+                document: this.rowToDocument(row),
+                score,
+            });
+        }
+        console.log('[LibSQLDB] Found results:', searchResults.length);
+        return searchResults;
+    }
+    /**
+     * Hybrid search with multiple vector fields (dense + sparse)
+     */
+    async hybridSearch(collectionName, searchRequests, options) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        const metadata = this.metadataCache.get(collectionName);
+        if (!metadata?.isHybrid) {
+            throw new Error(`Collection ${collectionName} is not a hybrid collection`);
+        }
+        const limit = options?.limit || 10;
+        console.log('[LibSQLDB] Hybrid search, requests:', searchRequests.length);
+        // Process search requests
+        const denseResults = new Map();
+        const sparseResults = new Map();
+        for (const request of searchRequests) {
+            if (request.anns_field === 'vector' || request.anns_field === 'dense') {
+                await this.performDenseSearch(client, request.data, limit, denseResults);
+            }
+            else if (request.anns_field === 'sparse' || request.anns_field === 'sparse_vector') {
+                await this.performSparseSearch(client, collectionName, request.data, sparseResults);
+            }
+        }
+        // Apply RRF reranking
+        const k = options?.rerank?.params?.k || 60;
+        const rrfScores = this.applyRRF(denseResults, sparseResults, k);
+        // Fetch full documents for top results
+        const topIds = Array.from(rrfScores.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, limit)
+            .map(([id]) => id);
+        if (topIds.length === 0) {
+            return [];
+        }
+        const placeholders = topIds.map(() => '?').join(',');
+        const docsResult = await client.execute({
+            sql: `SELECT * FROM documents WHERE id IN (${placeholders})`,
+            args: topIds,
+        });
+        const results = [];
+        const docMap = new Map(docsResult.rows.map((row) => [row.id, row]));
+        for (const id of topIds) {
+            const row = docMap.get(id);
+            if (row) {
+                results.push({
+                    document: this.rowToDocument(row),
+                    score: rrfScores.get(id) || 0,
+                });
+            }
+        }
+        console.log('[LibSQLDB] Hybrid search results:', results.length);
+        return results;
+    }
+    /**
+     * Perform dense vector search
+     */
+    async performDenseSearch(client, queryVector, limit, results) {
+        const queryVectorStr = `[${queryVector.join(',')}]`;
+        const sql = `
+      SELECT d.id, vector_distance_cos(d.dense_vector, vector32(?)) AS distance
+      FROM vector_top_k('idx_dense', vector32(?), ?) AS vt
+      JOIN documents d ON d.rowid = vt.id
+    `;
+        const result = await client.execute({
+            sql,
+            args: [queryVectorStr, queryVectorStr, limit * 2],
+        });
+        for (const row of result.rows) {
+            const score = 1 / (1 + row.distance);
+            results.set(row.id, score);
+        }
+    }
+    /**
+     * Perform sparse search using BM25
+     */
+    async performSparseSearch(client, collectionName, queryText, results) {
+        const bm25 = this.bm25Generators.get(collectionName);
+        if (!bm25) {
+            console.warn(`[LibSQLDB] BM25 generator not found for collection ${collectionName}. Sparse search skipped.`);
+            return;
+        }
+        if (!bm25.isTrained()) {
+            console.warn(`[LibSQLDB] BM25 model not trained for collection ${collectionName}. Sparse search skipped.`);
+            return;
+        }
+        const queryVector = bm25.generate(queryText);
+        const queryMap = new Map();
+        for (let i = 0; i < queryVector.indices.length; i++) {
+            queryMap.set(queryVector.indices[i], queryVector.values[i]);
+        }
+        // Fetch all documents with sparse vectors and compute scores
+        const result = await client.execute(`
+      SELECT id, sparse_indices, sparse_values FROM documents
+      WHERE sparse_indices IS NOT NULL
+    `);
+        for (const row of result.rows) {
+            const indices = JSON.parse(row.sparse_indices);
+            const values = JSON.parse(row.sparse_values);
+            let score = 0;
+            for (let i = 0; i < indices.length; i++) {
+                const queryVal = queryMap.get(indices[i]);
+                if (queryVal !== undefined) {
+                    score += values[i] * queryVal;
+                }
+            }
+            if (score > 0) {
+                results.set(row.id, score);
+            }
+        }
+    }
+    /**
+     * Apply Reciprocal Rank Fusion (RRF) reranking
+     */
+    applyRRF(denseResults, sparseResults, k) {
+        const denseRanks = this.computeRanks(denseResults);
+        const sparseRanks = this.computeRanks(sparseResults);
+        const allIds = new Set([...denseResults.keys(), ...sparseResults.keys()]);
+        const rrfScores = new Map();
+        for (const id of allIds) {
+            let score = 0;
+            const denseRank = denseRanks.get(id);
+            const sparseRank = sparseRanks.get(id);
+            if (denseRank !== undefined) {
+                score += 1 / (k + denseRank);
+            }
+            if (sparseRank !== undefined) {
+                score += 1 / (k + sparseRank);
+            }
+            rrfScores.set(id, score);
+        }
+        return rrfScores;
+    }
+    /**
+     * Compute ranks from scores
+     */
+    computeRanks(scores) {
+        const ranks = new Map();
+        const sorted = Array.from(scores.entries()).sort((a, b) => b[1] - a[1]);
+        sorted.forEach(([id], index) => ranks.set(id, index + 1));
+        return ranks;
+    }
+    /**
+     * Convert database row to VectorDocument
+     */
+    rowToDocument(row) {
+        return {
+            id: row.id,
+            vector: [],
+            content: row.content,
+            relativePath: row.relative_path,
+            startLine: row.start_line,
+            endLine: row.end_line,
+            fileExtension: row.file_extension,
+            metadata: JSON.parse(row.metadata),
+        };
+    }
+    /**
+     * Parse filter expression to SQL WHERE clause
+     */
+    parseFilterExpression(expr) {
+        // Convert Milvus-style filters to SQL WHERE clause
+        // "fileExtension == '.ts'" -> "file_extension = '.ts'"
+        // "fileExtension in ['.ts', '.js']" -> "file_extension IN ('.ts', '.js')"
+        if (expr.includes(' in ')) {
+            const match = expr.match(/(\w+)\s+in\s+\[(.*)\]/);
+            if (match) {
+                const field = this.mapFieldName(match[1]);
+                const values = match[2].split(',').map((v) => v.trim());
+                return `${field} IN (${values.join(',')})`;
+            }
+        }
+        if (expr.includes('==')) {
+            const match = expr.match(/(\w+)\s*==\s*(.+)/);
+            if (match) {
+                const field = this.mapFieldName(match[1]);
+                return `${field} = ${match[2].trim()}`;
+            }
+        }
+        // Return as-is if not recognized
+        console.warn(`[LibSQLDB] Unrecognized filter expression: ${expr}`);
+        return expr;
+    }
+    /**
+     * Map field names to database column names
+     */
+    mapFieldName(field) {
+        const mapping = {
+            relativePath: 'relative_path',
+            startLine: 'start_line',
+            endLine: 'end_line',
+            fileExtension: 'file_extension',
+        };
+        return mapping[field] || field;
+    }
+    /**
+     * Delete documents by IDs
+     *
+     * Key advantage over FAISS: LibSQL supports document deletion via SQL DELETE
+     */
+    async delete(collectionName, ids) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        console.log(`[LibSQLDB] Deleting ${ids.length} documents from ${collectionName}`);
+        const placeholders = ids.map(() => '?').join(',');
+        await client.execute({
+            sql: `DELETE FROM documents WHERE id IN (${placeholders})`,
+            args: ids,
+        });
+        await this.updateDocumentCount(collectionName);
+        // Re-train BM25 if hybrid collection
+        const metadata = this.metadataCache.get(collectionName);
+        if (metadata?.isHybrid) {
+            const bm25 = this.bm25Generators.get(collectionName);
+            if (bm25) {
+                const result = await client.execute('SELECT content FROM documents');
+                const contents = result.rows.map((r) => r.content);
+                if (contents.length > 0) {
+                    bm25.learn(contents);
+                    await this.saveBM25(collectionName);
+                }
+            }
+        }
+        console.log(`[LibSQLDB] Deleted ${ids.length} documents`);
+    }
+    /**
+     * Query documents with filter conditions
+     *
+     * Key advantage over FAISS: LibSQL supports SQL WHERE clauses
+     */
+    async query(collectionName, filter, outputFields, limit) {
+        await this.ensureInitialized();
+        const client = await this.getClient(collectionName);
+        console.log('[LibSQLDB] Querying documents');
+        // Build SELECT clause
+        const fields = outputFields.length > 0
+            ? outputFields.map((f) => this.mapFieldName(f)).join(', ')
+            : '*';
+        let sql = `SELECT ${fields} FROM documents`;
+        const args = [];
+        // Apply filter
+        if (filter && filter.trim()) {
+            const whereClause = this.parseFilterExpression(filter);
+            sql += ` WHERE ${whereClause}`;
+        }
+        sql += ' LIMIT ?';
+        args.push(limit || 100);
+        const result = await client.execute({ sql, args });
+        return result.rows.map((row) => this.rowToResult(row, outputFields));
+    }
+    /**
+     * Convert row to result object
+     */
+    rowToResult(row, outputFields) {
+        const result = {};
+        for (const field of outputFields) {
+            const dbField = this.mapFieldName(field);
+            if (row[dbField] !== undefined) {
+                result[field] = row[dbField];
+            }
+            else if (row[field] !== undefined) {
+                result[field] = row[field];
+            }
+        }
+        return result;
+    }
+    /**
+     * Check collection limit
+     * LibSQL has no inherent collection limit (only limited by disk space)
+     */
+    async checkCollectionLimit() {
+        return true;
+    }
+}
+exports.LibSQLVectorDatabase = LibSQLVectorDatabase;
+//# sourceMappingURL=libsql-vectordb.js.map