@matperez/coderag 0.1.24

package/dist/storage-persistent.js

@@ -0,0 +1,897 @@
+/**
+ * Persistent storage implementation using SQLite + Drizzle ORM (LibSQL WASM-compatible)
+ * Now supports chunk-level indexing for better search granularity
+ */
+import { eq, sql } from 'drizzle-orm';
+import { createDb } from './db/client.js';
+import { runMigrations } from './db/migrations.js';
+import * as schema from './db/schema.js';
+export class PersistentStorage {
+    dbInstance;
+    initPromise;
+    useBulkInsertChunks;
+    constructor(config = {}) {
+        this.useBulkInsertChunks = config.useBulkInsertChunks ?? false;
+        this.initPromise = this.initialize(config);
+    }
+    async initialize(config) {
+        this.dbInstance = await createDb(config);
+        await runMigrations(this.dbInstance.client);
+    }
+    /**
+     * Ensure database is initialized before operations
+     */
+    async ensureInit() {
+        await this.initPromise;
+    }
+    /**
+     * Get the LibSQL client for raw SQL operations
+     */
+    get client() {
+        return this.dbInstance.client;
+    }
+    /**
+     * Store a file
+     */
+    async storeFile(file) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const mtime = typeof file.mtime === 'number' ? file.mtime : file.mtime.getTime();
+        const values = {
+            path: file.path,
+            content: file.content,
+            hash: file.hash,
+            size: file.size,
+            mtime,
+            ...(file.language ? { language: file.language } : {}),
+            indexedAt: Date.now(),
+        };
+        await db
+            .insert(schema.files)
+            .values(values)
+            .onConflictDoUpdate({
+            target: schema.files.path,
+            set: {
+                content: values.content,
+                hash: values.hash,
+                size: values.size,
+                mtime: values.mtime,
+                ...(values.language ? { language: values.language } : {}),
+                indexedAt: values.indexedAt,
+            },
+        });
+    }
+    /**
+     * Store multiple files in a single transaction (batch operation)
+     * Much faster than storing one by one for large datasets
+     */
+    async storeFiles(files) {
+        if (files.length === 0) {
+            return;
+        }
+        await this.ensureInit();
+        // LibSQL supports batch transactions
+        await this.client.batch(files.map((file) => {
+            const mtime = typeof file.mtime === 'number' ? file.mtime : file.mtime.getTime();
+            return {
+                sql: `INSERT INTO files (path, content, hash, size, mtime, language, indexed_at)
+					      VALUES (?, ?, ?, ?, ?, ?, ?)
+					      ON CONFLICT(path) DO UPDATE SET
+					        content = excluded.content,
+					        hash = excluded.hash,
+					        size = excluded.size,
+					        mtime = excluded.mtime,
+					        language = excluded.language,
+					        indexed_at = excluded.indexed_at`,
+                args: [
+                    file.path,
+                    file.content,
+                    file.hash,
+                    file.size,
+                    mtime,
+                    file.language || null,
+                    Date.now(),
+                ],
+            };
+        }), 'write');
+    }
+    /**
+     * Get a file by path
+     */
+    async getFile(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db.select().from(schema.files).where(eq(schema.files.path, path)).get();
+        if (!result) {
+            return null;
+        }
+        return {
+            path: result.path,
+            content: result.content,
+            hash: result.hash,
+            size: result.size,
+            mtime: result.mtime,
+            language: result.language || undefined,
+        };
+    }
+    /**
+     * Get all files
+     */
+    async getAllFiles() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db.select().from(schema.files).all();
+        return results.map((file) => ({
+            path: file.path,
+            content: file.content,
+            hash: file.hash,
+            size: file.size,
+            mtime: file.mtime,
+            language: file.language || undefined,
+        }));
+    }
+    /**
+     * Get path -> hash for all files (for skip-unchanged during full index)
+     */
+    async getFileHashes() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({ path: schema.files.path, hash: schema.files.hash })
+            .from(schema.files)
+            .all();
+        const map = new Map();
+        for (const row of results)
+            map.set(row.path, row.hash);
+        return map;
+    }
+    /**
+     * Delete a file
+     */
+    async deleteFile(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.files).where(eq(schema.files.path, path));
+    }
+    /**
+     * Clear all files
+     */
+    async clear() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.chunks);
+        await db.delete(schema.files);
+        await db.delete(schema.documentVectors);
+        await db.delete(schema.idfScores);
+        await db.delete(schema.indexMetadata);
+    }
+    // ============ CHUNK METHODS ============
+    /**
+     * Store chunks for a file (replaces existing chunks)
+     */
+    async storeChunks(filePath, chunks) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Get file ID
+        const file = await db.select().from(schema.files).where(eq(schema.files.path, filePath)).get();
+        if (!file) {
+            throw new Error(`File not found: ${filePath}`);
+        }
+        // Delete existing chunks for this file
+        await db.delete(schema.chunks).where(eq(schema.chunks.fileId, file.id));
+        // Insert new chunks
+        const chunkIds = [];
+        for (const chunk of chunks) {
+            const insertValues = {
+                fileId: file.id,
+                content: chunk.content,
+                type: chunk.type,
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+            };
+            if (chunk.metadata) {
+                insertValues.metadata = JSON.stringify(chunk.metadata);
+            }
+            const result = await db
+                .insert(schema.chunks)
+                .values(insertValues)
+                .returning({ id: schema.chunks.id });
+            if (result[0]) {
+                chunkIds.push(result[0].id);
+            }
+        }
+        return chunkIds;
+    }
+    /**
+     * Store chunks for multiple files in batch
+     */
+    async storeManyChunks(fileChunks) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = new Map();
+        if (fileChunks.length === 0) {
+            return result;
+        }
+        // Get file IDs
+        const filePaths = fileChunks.map((fc) => fc.filePath);
+        const files = await db
+            .select({ id: schema.files.id, path: schema.files.path })
+            .from(schema.files)
+            .where(sql `${schema.files.path} IN (${sql.join(filePaths.map((p) => sql `${p}`), sql `, `)})`)
+            .all();
+        const fileIdMap = new Map();
+        for (const file of files) {
+            fileIdMap.set(file.path, file.id);
+        }
+        // Delete existing chunks for these files
+        const fileIds = Array.from(fileIdMap.values());
+        if (fileIds.length > 0) {
+            await db.delete(schema.chunks).where(sql `${schema.chunks.fileId} IN (${sql.join(fileIds.map((id) => sql `${id}`), sql `, `)})`);
+        }
+        if (!this.useBulkInsertChunks) {
+            for (const fc of fileChunks) {
+                const fileId = fileIdMap.get(fc.filePath);
+                if (!fileId)
+                    continue;
+                const ids = [];
+                for (const chunk of fc.chunks) {
+                    const insertResult = await db
+                        .insert(schema.chunks)
+                        .values({
+                        fileId,
+                        content: chunk.content,
+                        type: chunk.type,
+                        startLine: chunk.startLine,
+                        endLine: chunk.endLine,
+                        metadata: chunk.metadata ? JSON.stringify(chunk.metadata) : null,
+                    })
+                        .returning({ id: schema.chunks.id });
+                    const row = Array.isArray(insertResult) ? insertResult[0] : insertResult;
+                    if (row?.id != null)
+                        ids.push(row.id);
+                }
+                result.set(fc.filePath, ids);
+            }
+            return result;
+        }
+        // Bulk insert (SQLite ~999 bind limit, 6 fields/row → batch 150)
+        const CHUNK_INSERT_BATCH_SIZE = 150;
+        const flatRows = [];
+        const countsPerFile = [];
+        for (const fc of fileChunks) {
+            const fileId = fileIdMap.get(fc.filePath);
+            if (!fileId) {
+                countsPerFile.push(0);
+                continue;
+            }
+            countsPerFile.push(fc.chunks.length);
+            for (const chunk of fc.chunks) {
+                flatRows.push({
+                    fileId,
+                    content: chunk.content,
+                    type: chunk.type,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    metadata: chunk.metadata ? JSON.stringify(chunk.metadata) : null,
+                });
+            }
+        }
+        const allIds = [];
+        for (let i = 0; i < flatRows.length; i += CHUNK_INSERT_BATCH_SIZE) {
+            const batch = flatRows.slice(i, i + CHUNK_INSERT_BATCH_SIZE);
+            const insertResult = await db
+                .insert(schema.chunks)
+                .values(batch)
+                .returning({ id: schema.chunks.id });
+            const rows = Array.isArray(insertResult) ? insertResult : [insertResult];
+            for (const row of rows)
+                if (row?.id != null)
+                    allIds.push(row.id);
+        }
+        let offset = 0;
+        for (let i = 0; i < fileChunks.length; i++) {
+            const count = countsPerFile[i];
+            if (count === 0)
+                continue;
+            result.set(fileChunks[i].filePath, allIds.slice(offset, offset + count));
+            offset += count;
+        }
+        return result;
+    }
+    /**
+     * Get chunks for a file
+     */
+    async getChunksForFile(filePath) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({
+            id: schema.chunks.id,
+            fileId: schema.chunks.fileId,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            metadata: schema.chunks.metadata,
+            filePath: schema.files.path,
+        })
+            .from(schema.chunks)
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id))
+            .where(eq(schema.files.path, filePath))
+            .all();
+        return results.map((r) => ({
+            id: r.id,
+            fileId: r.fileId,
+            filePath: r.filePath,
+            content: r.content,
+            type: r.type,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            metadata: r.metadata ? JSON.parse(r.metadata) : undefined,
+        }));
+    }
+    /**
+     * Get total chunk count
+     */
+    async getChunkCount() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db.select({ count: sql `count(*)` }).from(schema.chunks).get();
+        return result?.count || 0;
+    }
+    /**
+     * Get file count
+     */
+    async count() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db.select({ count: sql `count(*)` }).from(schema.files).get();
+        return result?.count || 0;
+    }
+    /**
+     * Check if file exists
+     */
+    async exists(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db.select().from(schema.files).where(eq(schema.files.path, path)).get();
+        return result !== undefined;
+    }
+    /**
+     * Store document vectors (TF-IDF) for a CHUNK
+     */
+    async storeChunkVectors(chunkId, terms, tokenCount) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Delete existing vectors for this chunk
+        await db.delete(schema.documentVectors).where(eq(schema.documentVectors.chunkId, chunkId));
+        // Update token count if provided
+        if (tokenCount !== undefined) {
+            await this.client.execute({
+                sql: 'UPDATE chunks SET token_count = ? WHERE id = ?',
+                args: [tokenCount, chunkId],
+            });
+        }
+        // Insert new vectors in batches (SQLite has ~999 bind variable limit, 5 fields per row = 199 rows)
+        const BATCH_SIZE = 199;
+        const vectors = Array.from(terms.entries()).map(([term, scores]) => ({
+            chunkId,
+            term,
+            tf: scores.tf,
+            tfidf: scores.tfidf,
+            rawFreq: scores.rawFreq,
+        }));
+        for (let i = 0; i < vectors.length; i += BATCH_SIZE) {
+            const batch = vectors.slice(i, i + BATCH_SIZE);
+            await db.insert(schema.documentVectors).values(batch);
+        }
+    }
+    /**
+     * Store document vectors for multiple CHUNKS in a single transaction (batch operation)
+     * Much faster than storing one by one for large datasets
+     */
+    async storeManyChunkVectors(chunkVectors) {
+        if (chunkVectors.length === 0) {
+            return;
+        }
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Delete all existing vectors for these chunks
+        const chunkIds = chunkVectors.map((cv) => cv.chunkId);
+        if (chunkIds.length > 0) {
+            // Delete in batches to avoid SQLite variable limits
+            const deleteBatchSize = 500;
+            for (let i = 0; i < chunkIds.length; i += deleteBatchSize) {
+                const batch = chunkIds.slice(i, i + deleteBatchSize);
+                await db.delete(schema.documentVectors).where(sql `${schema.documentVectors.chunkId} IN (${sql.join(batch.map((id) => sql `${id}`), sql `, `)})`);
+            }
+        }
+        // Prepare all vectors for batch insert
+        const allVectors = [];
+        // Track token counts for BM25
+        const tokenCountUpdates = [];
+        for (const cv of chunkVectors) {
+            // Track token count for BM25 document length normalization
+            if (cv.tokenCount !== undefined) {
+                tokenCountUpdates.push({ chunkId: cv.chunkId, tokenCount: cv.tokenCount });
+            }
+            for (const [term, scores] of cv.terms.entries()) {
+                allVectors.push({
+                    chunkId: cv.chunkId,
+                    term,
+                    tf: scores.tf,
+                    tfidf: scores.tfidf,
+                    rawFreq: scores.rawFreq,
+                });
+            }
+        }
+        // Update token counts for BM25 using batch
+        if (tokenCountUpdates.length > 0) {
+            await this.client.batch(tokenCountUpdates.map(({ chunkId, tokenCount }) => ({
+                sql: 'UPDATE chunks SET token_count = ? WHERE id = ?',
+                args: [tokenCount, chunkId],
+            })), 'write');
+        }
+        // Insert in batches to avoid SQLite variable limits (5 fields per row = 199 rows max)
+        const batchSize = 199;
+        for (let i = 0; i < allVectors.length; i += batchSize) {
+            const batch = allVectors.slice(i, i + batchSize);
+            if (batch.length > 0) {
+                await db.insert(schema.documentVectors).values(batch);
+            }
+        }
+    }
+    /**
+     * Store IDF scores
+     */
+    async storeIdfScores(idf, docFreq) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Clear existing IDF scores
+        await db.delete(schema.idfScores);
+        // Insert new scores in batches (SQLite has ~999 bind variable limit, 3 fields per row = 300 rows)
+        const BATCH_SIZE = 300;
+        const scores = Array.from(idf.entries()).map(([term, idfScore]) => ({
+            term,
+            idf: idfScore,
+            documentFrequency: docFreq.get(term) || 0,
+        }));
+        for (let i = 0; i < scores.length; i += BATCH_SIZE) {
+            const batch = scores.slice(i, i + BATCH_SIZE);
+            await db.insert(schema.idfScores).values(batch);
+        }
+    }
+    /**
+     * Get IDF scores
+     */
+    async getIdfScores() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const scores = await db.select().from(schema.idfScores).all();
+        const idf = new Map();
+        for (const score of scores) {
+            idf.set(score.term, score.idf);
+        }
+        return idf;
+    }
+    /**
+     * Get document vectors for a chunk
+     */
+    async getChunkVectors(chunkId) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const vectors = await db
+            .select()
+            .from(schema.documentVectors)
+            .where(eq(schema.documentVectors.chunkId, chunkId))
+            .all();
+        if (vectors.length === 0) {
+            return null;
+        }
+        const terms = new Map();
+        for (const vector of vectors) {
+            terms.set(vector.term, {
+                tf: vector.tf,
+                tfidf: vector.tfidf,
+                rawFreq: vector.rawFreq,
+            });
+        }
+        return terms;
+    }
+    /**
+     * Get all chunk vectors in a single batch query (CPU + Memory optimization)
+     * Avoids N+1 query pattern when loading index from storage
+     * Returns Map<chunkId, Map<term, {tf, tfidf, rawFreq}>>
+     */
+    async getAllChunkVectors() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Single query to get all vectors
+        const results = await db
+            .select({
+            chunkId: schema.documentVectors.chunkId,
+            term: schema.documentVectors.term,
+            tf: schema.documentVectors.tf,
+            tfidf: schema.documentVectors.tfidf,
+            rawFreq: schema.documentVectors.rawFreq,
+        })
+            .from(schema.documentVectors)
+            .all();
+        // Group by chunk ID
+        const allVectors = new Map();
+        for (const row of results) {
+            let chunkVectors = allVectors.get(row.chunkId);
+            if (!chunkVectors) {
+                chunkVectors = new Map();
+                allVectors.set(row.chunkId, chunkVectors);
+            }
+            chunkVectors.set(row.term, {
+                tf: row.tf,
+                tfidf: row.tfidf,
+                rawFreq: row.rawFreq,
+            });
+        }
+        return allVectors;
+    }
+    /**
+     * Search chunks by terms using SQL (Memory optimization)
+     * Returns matching chunks with their content for direct display
+     * Uses pre-computed magnitude from chunks table
+     */
+    async searchByTerms(queryTerms, options = {}) {
+        if (queryTerms.length === 0) {
+            return [];
+        }
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const { limit = 100 } = options;
+        // Step 1: Find chunk IDs that contain any query term, with pre-computed magnitude and token count
+        const matchingChunks = await db
+            .select({
+            chunkId: schema.documentVectors.chunkId,
+            filePath: schema.files.path,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            magnitude: schema.chunks.magnitude,
+            tokenCount: schema.chunks.tokenCount,
+            matchCount: sql `COUNT(DISTINCT ${schema.documentVectors.term})`,
+        })
+            .from(schema.documentVectors)
+            .innerJoin(schema.chunks, eq(schema.documentVectors.chunkId, schema.chunks.id))
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id))
+            .where(sql `${schema.documentVectors.term} IN (${sql.join(queryTerms.map((t) => sql `${t}`), sql `, `)})`)
+            .groupBy(schema.documentVectors.chunkId)
+            .orderBy(sql `COUNT(DISTINCT ${schema.documentVectors.term}) DESC`)
+            .limit(limit * 2) // Get more candidates for scoring
+            .all();
+        if (matchingChunks.length === 0) {
+            return [];
+        }
+        // Step 2: Get matched term vectors for these chunks (only query terms)
+        const chunkIds = matchingChunks.map((c) => c.chunkId);
+        const matchedVectors = await db
+            .select({
+            chunkId: schema.documentVectors.chunkId,
+            term: schema.documentVectors.term,
+            tfidf: schema.documentVectors.tfidf,
+            rawFreq: schema.documentVectors.rawFreq,
+        })
+            .from(schema.documentVectors)
+            .where(sql `${schema.documentVectors.chunkId} IN (${sql.join(chunkIds.map((id) => sql `${id}`), sql `, `)}) AND ${schema.documentVectors.term} IN (${sql.join(queryTerms.map((t) => sql `${t}`), sql `, `)})`)
+            .all();
+        // Build result map with pre-computed magnitude and token count
+        const resultMap = new Map();
+        // Initialize result entries with chunk data
+        for (const c of matchingChunks) {
+            resultMap.set(c.chunkId, {
+                chunkId: c.chunkId,
+                filePath: c.filePath,
+                content: c.content,
+                type: c.type,
+                startLine: c.startLine,
+                endLine: c.endLine,
+                matchedTerms: new Map(),
+                magnitude: c.magnitude ?? 0,
+                tokenCount: c.tokenCount ?? 0,
+            });
+        }
+        // Populate matched terms
+        for (const v of matchedVectors) {
+            const entry = resultMap.get(v.chunkId);
+            if (entry) {
+                entry.matchedTerms.set(v.term, { tfidf: v.tfidf, rawFreq: v.rawFreq });
+            }
+        }
+        return Array.from(resultMap.values());
+    }
+    /**
+     * Get IDF scores for specific terms only (Memory optimization)
+     */
+    async getIdfScoresForTerms(terms) {
+        if (terms.length === 0) {
+            return new Map();
+        }
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const scores = await db
+            .select()
+            .from(schema.idfScores)
+            .where(sql `${schema.idfScores.term} IN (${sql.join(terms.map((t) => sql `${t}`), sql `, `)})`)
+            .all();
+        const idf = new Map();
+        for (const score of scores) {
+            idf.set(score.term, score.idf);
+        }
+        return idf;
+    }
+    /**
+     * Get total chunk count (for IDF calculation)
+     * BM25/TF-IDF now operates at chunk level, not file level
+     */
+    async getTotalDocuments() {
+        return this.getChunkCount();
+    }
+    /**
+     * Get all file metadata (path, mtime, hash) without content
+     * Used for incremental diff detection
+     */
+    async getAllFileMetadata() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({
+            path: schema.files.path,
+            mtime: schema.files.mtime,
+            hash: schema.files.hash,
+        })
+            .from(schema.files)
+            .all();
+        const metadata = new Map();
+        for (const row of results) {
+            metadata.set(row.path, { mtime: row.mtime, hash: row.hash });
+        }
+        return metadata;
+    }
+    /**
+     * Delete multiple files in a single transaction (batch operation)
+     */
+    async deleteFiles(paths) {
+        if (paths.length === 0) {
+            return;
+        }
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Delete in chunks to avoid SQLite variable limits
+        const chunkSize = 500;
+        for (let i = 0; i < paths.length; i += chunkSize) {
+            const chunk = paths.slice(i, i + chunkSize);
+            await db.delete(schema.files).where(sql `${schema.files.path} IN (${sql.join(chunk.map((p) => sql `${p}`), sql `, `)})`);
+        }
+    }
+    /**
+     * Store metadata
+     */
+    async setMetadata(key, value) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db
+            .insert(schema.indexMetadata)
+            .values({
+            key,
+            value,
+            updatedAt: Date.now(),
+        })
+            .onConflictDoUpdate({
+            target: schema.indexMetadata.key,
+            set: {
+                value,
+                updatedAt: Date.now(),
+            },
+        });
+    }
+    /**
+     * Get metadata
+     */
+    async getMetadata(key) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db
+            .select()
+            .from(schema.indexMetadata)
+            .where(eq(schema.indexMetadata.key, key))
+            .get();
+        return result?.value || null;
+    }
+    /**
+     * Get average chunk length (token count) for BM25 scoring
+     * Returns cached value from metadata if available, otherwise calculates from chunks table
+     */
+    async getAverageDocLength() {
+        await this.ensureInit();
+        // Try to get cached value first
+        const cached = await this.getMetadata('avgDocLength');
+        if (cached) {
+            return parseFloat(cached);
+        }
+        // Calculate from chunks table
+        const { db } = this.dbInstance;
+        const result = await db
+            .select({
+            avgLen: sql `AVG(COALESCE(${schema.chunks.tokenCount}, 0))`,
+        })
+            .from(schema.chunks)
+            .get();
+        const avgLen = result?.avgLen || 0;
+        // Cache the result
+        await this.setMetadata('avgDocLength', avgLen.toString());
+        return avgLen;
+    }
+    /**
+     * Update average chunk length in metadata (call after indexing)
+     */
+    async updateAverageDocLength() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await db
+            .select({
+            avgLen: sql `AVG(COALESCE(${schema.chunks.tokenCount}, 0))`,
+        })
+            .from(schema.chunks)
+            .get();
+        const avgLen = result?.avgLen || 0;
+        await this.setMetadata('avgDocLength', avgLen.toString());
+        return avgLen;
+    }
+    /**
+     * Rebuild IDF scores from document vectors using SQL (Memory optimization)
+     * Calculates document frequency for each term across CHUNKS and computes IDF
+     */
+    async rebuildIdfScoresFromVectors() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        // Get total chunk count (IDF is calculated per chunk, not per file)
+        const totalChunks = await this.getChunkCount();
+        if (totalChunks === 0) {
+            await db.delete(schema.idfScores);
+            return;
+        }
+        // Calculate document frequency for each term using SQL (counting chunks, not files)
+        const dfResults = await db
+            .select({
+            term: schema.documentVectors.term,
+            df: sql `COUNT(DISTINCT ${schema.documentVectors.chunkId})`,
+        })
+            .from(schema.documentVectors)
+            .groupBy(schema.documentVectors.term)
+            .all();
+        // Clear existing IDF scores
+        await db.delete(schema.idfScores);
+        // Insert in batches using smoothed IDF formula
+        // Smoothed IDF: log((N+1)/(df+1)) + 1 ensures no term gets IDF=0
+        const BATCH_SIZE = 300;
+        const scores = dfResults.map((row) => ({
+            term: row.term,
+            idf: Math.log((totalChunks + 1) / (row.df + 1)) + 1,
+            documentFrequency: row.df,
+        }));
+        for (let i = 0; i < scores.length; i += BATCH_SIZE) {
+            const batch = scores.slice(i, i + BATCH_SIZE);
+            if (batch.length > 0) {
+                await db.insert(schema.idfScores).values(batch);
+            }
+        }
+    }
+    /**
+     * Recalculate TF-IDF scores for all documents using current IDF values (Memory optimization)
+     * Updates document_vectors.tfidf = document_vectors.tf * idf_scores.idf
+     */
+    async recalculateTfidfScores() {
+        await this.ensureInit();
+        // Use raw SQL for efficient batch update with JOIN
+        await this.client.execute(`
+			UPDATE document_vectors
+			SET tfidf = tf * COALESCE(
+				(SELECT idf FROM idf_scores WHERE idf_scores.term = document_vectors.term),
+				0
+			)
+		`);
+    }
+    /**
+     * Update pre-computed magnitude for all chunks (Memory optimization for search)
+     * magnitude = sqrt(sum(tfidf^2)) for each chunk
+     * Called after TF-IDF recalculation to keep magnitude in sync
+     */
+    async updateChunkMagnitudes() {
+        await this.ensureInit();
+        // Use raw SQL for efficient batch update with aggregate
+        await this.client.execute(`
+			UPDATE chunks
+			SET magnitude = COALESCE(
+				(SELECT SQRT(SUM(tfidf * tfidf)) FROM document_vectors WHERE document_vectors.chunk_id = chunks.id),
+				0
+			)
+		`);
+    }
+    /**
+     * Get terms for chunks of files (for tracking affected terms during incremental updates)
+     * When files are deleted, we need to know which terms were affected
+     */
+    async getTermsForFiles(paths) {
+        if (paths.length === 0) {
+            return new Set();
+        }
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const terms = new Set();
+        // Get file IDs
+        const files = await db
+            .select({ id: schema.files.id })
+            .from(schema.files)
+            .where(sql `${schema.files.path} IN (${sql.join(paths.map((p) => sql `${p}`), sql `, `)})`)
+            .all();
+        if (files.length === 0) {
+            return terms;
+        }
+        const fileIds = files.map((f) => f.id);
+        // Get chunk IDs for these files
+        const chunks = await db
+            .select({ id: schema.chunks.id })
+            .from(schema.chunks)
+            .where(sql `${schema.chunks.fileId} IN (${sql.join(fileIds.map((id) => sql `${id}`), sql `, `)})`)
+            .all();
+        if (chunks.length === 0) {
+            return terms;
+        }
+        const chunkIds = chunks.map((c) => c.id);
+        // Get terms for these chunks
+        const results = await db
+            .select({ term: schema.documentVectors.term })
+            .from(schema.documentVectors)
+            .where(sql `${schema.documentVectors.chunkId} IN (${sql.join(chunkIds.map((id) => sql `${id}`), sql `, `)})`)
+            .all();
+        for (const row of results) {
+            terms.add(row.term);
+        }
+        return terms;
+    }
+    /**
+     * Get all chunks with their file paths (for bulk operations)
+     */
+    async getAllChunks() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({
+            id: schema.chunks.id,
+            fileId: schema.chunks.fileId,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            metadata: schema.chunks.metadata,
+            filePath: schema.files.path,
+        })
+            .from(schema.chunks)
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id))
+            .all();
+        return results.map((r) => ({
+            id: r.id,
+            fileId: r.fileId,
+            filePath: r.filePath,
+            content: r.content,
+            type: r.type,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            metadata: r.metadata ? JSON.parse(r.metadata) : undefined,
+        }));
+    }
+    /**
+     * Close database connection
+     */
+    close() {
+        this.dbInstance.client.close();
+    }
+}
+//# sourceMappingURL=storage-persistent.js.map