codetree-claude 1.4.7 → 1.5.1

package/dist/storage/database.js

@@ -1,5 +1,26 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
+import { inflateSync } from "node:zlib";
+/**
+ * Extract lowercase trigrams from content for the trigram index.
+ * Caps at 5000 unique trigrams per file to keep the index manageable.
+ */
+function extractTrigrams(content, maxTrigrams = 5000) {
+    const trigrams = new Set();
+    const lower = content.toLowerCase();
+    for (let i = 0; i < lower.length - 2 && trigrams.size < maxTrigrams; i++) {
+        const c0 = lower.charCodeAt(i);
+        // Only index trigrams from identifier-like characters (letters, digits, underscore)
+        if ((c0 >= 97 && c0 <= 122) || (c0 >= 48 && c0 <= 57) || c0 === 95) {
+            const tri = lower.slice(i, i + 3);
+            // Skip trigrams with whitespace/newlines
+            if (!/\s/.test(tri)) {
+                trigrams.add(tri);
+            }
+        }
+    }
+    return trigrams;
+}
 const SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS files (
   path TEXT PRIMARY KEY,
@@ -123,6 +144,28 @@ CREATE TABLE IF NOT EXISTS compaction_snapshots (
 
 CREATE INDEX IF NOT EXISTS idx_compaction_session ON compaction_snapshots(session_id);
 CREATE INDEX IF NOT EXISTS idx_compaction_time ON compaction_snapshots(created_at);
+
+-- Previous content: stores old file content for diff-based reads
+CREATE TABLE IF NOT EXISTS previous_content (
+  hash TEXT PRIMARY KEY,
+  content BLOB NOT NULL,
+  saved_at INTEGER NOT NULL
+);
+
+-- Token postings: persisted inverted index for fast content search
+CREATE TABLE IF NOT EXISTS token_postings (
+  token TEXT NOT NULL,
+  file_path TEXT NOT NULL,
+  PRIMARY KEY (token, file_path)
+);
+CREATE INDEX IF NOT EXISTS idx_token_postings_file ON token_postings(file_path);
+
+-- Trigram index for fast content search (replaces sequential Grep scans)
+CREATE TABLE IF NOT EXISTS trigrams (
+  trigram TEXT NOT NULL,
+  file_path TEXT NOT NULL,
+  PRIMARY KEY (trigram, file_path)
+);
 `;
 export class Database {
     config;
@@ -154,7 +197,7 @@ export class Database {
         // Enable WAL-like behavior and foreign keys
         this.db.run("PRAGMA journal_mode=MEMORY");
         this.db.run("PRAGMA foreign_keys=ON");
-        this.db.run("PRAGMA synchronous=OFF");
+        this.db.run("PRAGMA synchronous=NORMAL");
         // Create schema
         this.db.run(SCHEMA_SQL);
         // Schema version check — force re-extraction on upgrade
@@ -162,10 +205,21 @@ export class Database {
         const result = this.db.exec("SELECT value FROM meta WHERE key = 'schema_version'");
         const existingVersion = result.length > 0 ? String(result[0].values[0][0]) : "0";
         if (existingVersion !== CURRENT_SCHEMA) {
-            // Clear all file hashes to force re-extraction on next scan
+            // Clear all file hashes AND mtime to force re-extraction on next scan.
+            // (mtime must be cleared too — otherwise indexFile's quickHash short-circuit
+            //  matches and skips re-extraction even though we want to re-extract.)
             this.db.run("DELETE FROM symbols");
             this.db.run("DELETE FROM dependencies");
-            this.db.run("UPDATE files SET hash = 'force-reindex'");
+            this.db.run("UPDATE files SET hash = 'force-reindex', mtime_ms = 0");
+            // Drop derived in-memory/persisted indexes so they get rebuilt with the new logic
+            try {
+                this.db.run("DELETE FROM token_postings");
+            }
+            catch { /* table may not exist on first run */ }
+            try {
+                this.db.run("DELETE FROM trigrams");
+            }
+            catch { /* table may not exist on first run */ }
             this.db.run(`INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', '${CURRENT_SCHEMA}')`);
             this.dirty = true;
             this.persistToDisk();
@@ -271,8 +325,33 @@ export class Database {
     searchSymbols(query, kind, limit = 50) {
         if (!this.db)
             throw new Error("Database not initialized");
-        let sql = "SELECT * FROM symbols WHERE name LIKE ?";
-        const params = [`%${query}%`];
+        // Try exact match first (uses idx_symbols_name index efficiently)
+        let sql;
+        let params;
+        const exactResults = this.searchSymbolsExact(query, kind, limit);
+        if (exactResults.length > 0)
+            return exactResults;
+        // Prefix match (still uses index for prefix portion)
+        sql = "SELECT * FROM symbols WHERE name LIKE ?";
+        params = [`${query}%`];
+        if (kind) {
+            sql += " AND kind = ?";
+            params.push(kind);
+        }
+        sql += ` LIMIT ${limit}`;
+        const prefixStmt = this.db.prepare(sql);
+        prefixStmt.bind(params);
+        const prefixResults = [];
+        while (prefixStmt.step()) {
+            const row = prefixStmt.getAsObject();
+            prefixResults.push(this.rowToSymbol(row));
+        }
+        prefixStmt.free();
+        if (prefixResults.length >= Math.min(5, limit))
+            return prefixResults;
+        // Fall back to substring match (no index benefit, but rare)
+        sql = "SELECT * FROM symbols WHERE name LIKE ?";
+        params = [`%${query}%`];
         if (kind) {
             sql += " AND kind = ?";
             params.push(kind);
@@ -282,17 +361,38 @@ export class Database {
         stmt.bind(params);
         const results = [];
         while (stmt.step()) {
-            const row = stmt.getAsObject();
-            results.push({
-                id: row.id,
-                file_path: row.file_path,
-                name: row.name,
-                kind: row.kind,
-                line_start: row.line_start,
-                line_end: row.line_end,
-                signature: row.signature,
-                exported: row.exported === 1,
-            });
+            results.push(this.rowToSymbol(stmt.getAsObject()));
+        }
+        stmt.free();
+        return results;
+    }
+    rowToSymbol(row) {
+        return {
+            id: row.id,
+            file_path: row.file_path,
+            name: row.name,
+            kind: row.kind,
+            line_start: row.line_start,
+            line_end: row.line_end,
+            signature: row.signature,
+            exported: row.exported === 1,
+        };
+    }
+    searchSymbolsExact(query, kind, limit = 50) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        let sql = "SELECT * FROM symbols WHERE name = ?";
+        const params = [query];
+        if (kind) {
+            sql += " AND kind = ?";
+            params.push(kind);
+        }
+        sql += ` LIMIT ${limit}`;
+        const stmt = this.db.prepare(sql);
+        stmt.bind(params);
+        const results = [];
+        while (stmt.step()) {
+            results.push(this.rowToSymbol(stmt.getAsObject()));
         }
         stmt.free();
         return results;
@@ -385,23 +485,90 @@ export class Database {
         const maxBytes = maxSizeMB * 1024 * 1024;
         if (totalBytes <= maxBytes)
             return;
-        // Delete oldest entries until under limit
-        const toDelete = totalBytes - maxBytes;
-        this.db.run(`
-      DELETE FROM content_cache WHERE path IN (
-        SELECT path FROM content_cache
-        ORDER BY accessed_at ASC
-        LIMIT (
-          SELECT COUNT(*) FROM content_cache
-          WHERE accessed_at <= (
-            SELECT accessed_at FROM content_cache ORDER BY accessed_at ASC
-            LIMIT 1 OFFSET (SELECT COUNT(*)/4 FROM content_cache)
-          )
+        // Iteratively delete oldest entries until under limit
+        let currentBytes = totalBytes;
+        while (currentBytes > maxBytes) {
+            // Delete the oldest 10% of entries each pass
+            const batchSize = Math.max(1, Math.ceil((this.db.exec("SELECT COUNT(*) FROM content_cache")[0]?.values[0][0] || 0) * 0.1));
+            const before = this.db.exec("SELECT SUM(LENGTH(content)) FROM content_cache");
+            this.db.run(`
+        DELETE FROM content_cache WHERE path IN (
+          SELECT path FROM content_cache
+          ORDER BY accessed_at ASC
+          LIMIT ${batchSize}
         )
-      )
-    `);
+      `);
+            const after = this.db.exec("SELECT SUM(LENGTH(content)) FROM content_cache");
+            currentBytes = after[0]?.values[0][0] || 0;
+            // Safety: if nothing was deleted, break to avoid infinite loop
+            const beforeBytes = before[0]?.values[0][0] || 0;
+            if (currentBytes >= beforeBytes)
+                break;
+        }
         this.markDirty();
     }
+    /**
+     * Look up cached content for a file that previously had a given hash.
+     * Used for diff-based reads: find old version to compute delta.
+     * Checks the previous_content table first, then falls back to content_cache.
+     * Returns decompressed content or null if not found.
+     */
+    getCachedContentByHash(hash) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        // Check previous_content table (stores last version before update)
+        const stmt = this.db.prepare("SELECT content FROM previous_content WHERE hash = ? LIMIT 1");
+        stmt.bind([hash]);
+        if (stmt.step()) {
+            const row = stmt.get();
+            stmt.free();
+            try {
+                return inflateSync(Buffer.from(row[0])).toString("utf-8");
+            }
+            catch {
+                return null;
+            }
+        }
+        stmt.free();
+        return null;
+    }
+    /**
+     * Save previous content before overwriting, keyed by hash.
+     * Used for diff-based reads.
+     *
+     * Cleanup strategy:
+     *   - TTL: 24h (long enough to survive long sessions / compaction).
+     *   - Hard cap: 200 rows; oldest evicted first.
+     * Both run AFTER the insert so we never delete the row we just saved.
+     */
+    savePreviousContent(hash, compressedContent) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        try {
+            const now = Date.now();
+            this.db.run("INSERT OR REPLACE INTO previous_content (hash, content, saved_at) VALUES (?, ?, ?)", [hash, compressedContent, now]);
+            // TTL eviction (24h). Use strict < so the just-saved row is never targeted.
+            this.db.run("DELETE FROM previous_content WHERE saved_at < ?", [now - 24 * 3600_000]);
+            // Row-count cap (oldest first). Keeps table size bounded for fast-moving repos.
+            const MAX_ROWS = 200;
+            this.db.run(`DELETE FROM previous_content WHERE hash IN (
+           SELECT hash FROM previous_content
+           ORDER BY saved_at ASC
+           LIMIT MAX(0, (SELECT COUNT(*) FROM previous_content) - ${MAX_ROWS})
+         )`);
+            this.markDirty();
+        }
+        catch { /* ignore if table doesn't exist yet */ }
+    }
+    /**
+     * Get the total size of the content_cache table in bytes (compressed content).
+     */
+    getContentCacheSize() {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const result = this.db.exec("SELECT SUM(LENGTH(content)) FROM content_cache");
+        return result.length > 0 ? (result[0].values[0][0] || 0) : 0;
+    }
     // ─── Search Operations ─────────────────────────────────────────
     searchFiles(pattern) {
         if (!this.db)
@@ -439,6 +606,23 @@ export class Database {
         this.db.run("UPDATE sessions SET ended_at = ?, last_accessed = ?, summary = ? WHERE id = ?", [Date.now(), Date.now(), summary || null, sessionId]);
         this.markDirty();
     }
+    /**
+     * Most recent read timestamp by a session for a file (any tool).
+     * Returns 0 if there is no prior read.
+     */
+    getLastSessionReadAt(sessionId, filePath) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const stmt = this.db.prepare("SELECT MAX(read_at) as last FROM session_reads WHERE session_id = ? AND file_path = ?");
+        stmt.bind([sessionId, filePath]);
+        let last = 0;
+        if (stmt.step()) {
+            const row = stmt.getAsObject();
+            last = row.last || 0;
+        }
+        stmt.free();
+        return last;
+    }
     logSessionRead(sessionId, filePath, toolName, lineStart, lineEnd) {
         if (!this.db)
             throw new Error("Database not initialized");
@@ -446,20 +630,23 @@ export class Database {
        VALUES (?, ?, ?, ?, ?, ?)`);
         stmt.run([sessionId, filePath, Date.now(), lineStart || null, lineEnd || null, toolName]);
         stmt.free();
-        // Also update the session's files_read JSON array
-        const session = this.getSession(sessionId);
-        if (session) {
-            try {
-                const filesRead = JSON.parse(session.files_read || "[]");
-                if (!filesRead.includes(filePath)) {
-                    filesRead.push(filePath);
-                    this.db.run("UPDATE sessions SET files_read = ? WHERE id = ?", [JSON.stringify(filesRead), sessionId]);
-                }
-            }
-            catch { }
-        }
         this.markDirty();
     }
+    /**
+     * Get list of unique files read in a session (from session_reads table, not JSON column).
+     */
+    getSessionFilesRead(sessionId) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const stmt = this.db.prepare("SELECT DISTINCT file_path FROM session_reads WHERE session_id = ?");
+        stmt.bind([sessionId]);
+        const paths = [];
+        while (stmt.step()) {
+            paths.push(stmt.get()[0]);
+        }
+        stmt.free();
+        return paths;
+    }
     logSessionModified(sessionId, filePath) {
         if (!this.db)
             throw new Error("Database not initialized");
@@ -492,6 +679,25 @@ export class Database {
         }
         this.markDirty();
     }
+    logSessionGrep(sessionId, pattern) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const session = this.getSession(sessionId);
+        if (session) {
+            try {
+                // Store grep patterns in the decisions JSON column (reusing existing column to avoid schema migration)
+                const decisions = JSON.parse(session.decisions || "{}");
+                const greps = Array.isArray(decisions._grep_patterns) ? decisions._grep_patterns : [];
+                if (!greps.includes(pattern)) {
+                    greps.push(pattern);
+                    decisions._grep_patterns = greps;
+                    this.db.run("UPDATE sessions SET decisions = ? WHERE id = ?", [JSON.stringify(decisions), sessionId]);
+                }
+            }
+            catch { }
+        }
+        this.markDirty();
+    }
     addSessionInsight(sessionId, category, content, filePaths) {
         if (!this.db)
             throw new Error("Database not initialized");
@@ -579,24 +785,29 @@ export class Database {
         if (!this.db)
             throw new Error("Database not initialized");
         const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
-        const results = this.db.exec(`
+        const stmt = this.db.prepare(`
       SELECT sr.file_path,
              COUNT(*) as total_reads,
              COUNT(DISTINCT sr.session_id) as sessions
       FROM session_reads sr
       JOIN sessions s ON sr.session_id = s.id
-      WHERE s.last_accessed > ${cutoff}
+      WHERE s.last_accessed > ?
       GROUP BY sr.file_path
       ORDER BY sessions DESC, total_reads DESC
-      LIMIT ${limit}
+      LIMIT ?
     `);
-        if (results.length === 0)
-            return [];
-        return results[0].values.map((row) => ({
-            file_path: row[0],
-            total_reads: row[1],
-            sessions: row[2],
-        }));
+        stmt.bind([cutoff, limit]);
+        const results = [];
+        while (stmt.step()) {
+            const row = stmt.getAsObject();
+            results.push({
+                file_path: row.file_path,
+                total_reads: row.total_reads,
+                sessions: row.sessions,
+            });
+        }
+        stmt.free();
+        return results;
     }
     /**
      * Get files that changed since a given timestamp.
@@ -711,6 +922,15 @@ export class Database {
                 fileSummaries[filePath] = `${fileRecord.language || "unknown"}, ${fileRecord.line_count} lines`;
             }
         }
+        // Extract grep patterns from session decisions
+        let grepPatterns = [];
+        try {
+            const decisions = session ? JSON.parse(session.decisions || "{}") : {};
+            if (Array.isArray(decisions._grep_patterns)) {
+                grepPatterns = decisions._grep_patterns;
+            }
+        }
+        catch { }
         return {
             filesRead: reads.map((r) => ({
                 path: r.file_path,
@@ -718,10 +938,135 @@ export class Database {
             })),
             filesModified,
             symbolsDiscovered: symbolsSearched,
-            grepPatterns: symbolsSearched, // reuse — greps are logged as symbol searches
+            grepPatterns,
             fileSummaries,
         };
     }
+    // ─── Token Postings (persisted inverted index) ──────────────
+    saveTokenPostings(filePath, tokens) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        this.db.run("DELETE FROM token_postings WHERE file_path = ?", [filePath]);
+        if (tokens.size === 0)
+            return;
+        const stmt = this.db.prepare("INSERT OR IGNORE INTO token_postings (token, file_path) VALUES (?, ?)");
+        for (const token of tokens) {
+            stmt.run([token, filePath]);
+        }
+        stmt.free();
+        this.markDirty();
+    }
+    deleteTokenPostings(filePath) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        this.db.run("DELETE FROM token_postings WHERE file_path = ?", [filePath]);
+        this.markDirty();
+    }
+    loadAllTokenPostings() {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const result = this.db.exec("SELECT token, file_path FROM token_postings");
+        const postings = new Map();
+        if (result.length === 0)
+            return postings;
+        for (const row of result[0].values) {
+            const token = row[0];
+            const filePath = row[1];
+            let set = postings.get(token);
+            if (!set) {
+                set = new Set();
+                postings.set(token, set);
+            }
+            set.add(filePath);
+        }
+        return postings;
+    }
+    getTokenPostingsFileMap() {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const result = this.db.exec("SELECT file_path, token FROM token_postings");
+        const fileTokens = new Map();
+        if (result.length === 0)
+            return fileTokens;
+        for (const row of result[0].values) {
+            const filePath = row[0];
+            const token = row[1];
+            let set = fileTokens.get(filePath);
+            if (!set) {
+                set = new Set();
+                fileTokens.set(filePath, set);
+            }
+            set.add(token);
+        }
+        return fileTokens;
+    }
+    // ─── Trigram Index ──────────────────────────────────────────
+    saveTrigrams(filePath, content) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        this.db.run("DELETE FROM trigrams WHERE file_path = ?", [filePath]);
+        const trigrams = extractTrigrams(content);
+        if (trigrams.size === 0)
+            return;
+        const stmt = this.db.prepare("INSERT OR IGNORE INTO trigrams (trigram, file_path) VALUES (?, ?)");
+        for (const tri of trigrams) {
+            stmt.run([tri, filePath]);
+        }
+        stmt.free();
+        // Don't markDirty here — called during indexing which already persists
+    }
+    deleteTrigrams(filePath) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        this.db.run("DELETE FROM trigrams WHERE file_path = ?", [filePath]);
+    }
+    /**
+     * Find files that contain ALL trigrams from the query string.
+     * This narrows candidates before doing actual content search.
+     */
+    searchByTrigrams(query, limit = 200) {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const trigrams = Array.from(extractTrigrams(query));
+        if (trigrams.length === 0)
+            return [];
+        // Intersect: files that have ALL trigrams
+        // Start with the rarest trigram to minimize set size
+        let sql = "SELECT file_path FROM trigrams WHERE trigram = ?";
+        for (let i = 1; i < Math.min(trigrams.length, 6); i++) {
+            sql += " INTERSECT SELECT file_path FROM trigrams WHERE trigram = ?";
+        }
+        sql += ` LIMIT ${limit}`;
+        const stmt = this.db.prepare(sql);
+        stmt.bind(trigrams.slice(0, 6));
+        const results = [];
+        while (stmt.step()) {
+            results.push(stmt.get()[0]);
+        }
+        stmt.free();
+        return results;
+    }
+    getSymbolCount() {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const result = this.db.exec("SELECT COUNT(*) FROM symbols");
+        return result.length > 0 ? result[0].values[0][0] : 0;
+    }
+    /**
+     * Get symbol counts per file in a single query (avoids N+1).
+     */
+    getSymbolCountsByFile() {
+        if (!this.db)
+            throw new Error("Database not initialized");
+        const result = this.db.exec("SELECT file_path, COUNT(*) as cnt FROM symbols GROUP BY file_path");
+        const counts = new Map();
+        if (result.length === 0)
+            return counts;
+        for (const row of result[0].values) {
+            counts.set(row[0], row[1]);
+        }
+        return counts;
+    }
     // ─── Lifecycle ─────────────────────────────────────────────────
     close() {
         if (this.saveTimer) {