context-mode 1.0.88 → 1.0.90

package/build/store.js

@@ -10,6 +10,7 @@
 var _a;
 import { loadDatabase, applyWALPragmas, closeDB, cleanOrphanedWALFiles, withRetry, deleteDBFiles, isSQLiteCorruptionError } from "./db-base.js";
 import { readFileSync, readdirSync, unlinkSync, existsSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 // ─────────────────────────────────────────────────────────
@@ -33,12 +34,30 @@ const STOPWORDS = new Set([
 // ─────────────────────────────────────────────────────────
 // Helpers
 // ─────────────────────────────────────────────────────────
-function sanitizeQuery(query, mode = "AND") {
-    const words = query
+/**
+ * Remove case-insensitive duplicate tokens while preserving the first
+ * occurrence's original casing. FTS5's unicode61 tokenizer lowercases on
+ * both sides, so `"Error" OR "error"` produces no extra recall — just
+ * redundant index lookups. Dedup keeps the compiled query minimal.
+ */
+function dedupeTokens(tokens) {
+    const seen = new Set();
+    const out = [];
+    for (const t of tokens) {
+        const key = t.toLowerCase();
+        if (!seen.has(key)) {
+            seen.add(key);
+            out.push(t);
+        }
+    }
+    return out;
+}
+export function sanitizeQuery(query, mode = "AND") {
+    const words = dedupeTokens(query
         .replace(/['"(){}[\]*:^~]/g, " ")
         .split(/\s+/)
         .filter((w) => w.length > 0 &&
-        !["AND", "OR", "NOT", "NEAR"].includes(w.toUpperCase()));
+        !["AND", "OR", "NOT", "NEAR"].includes(w.toUpperCase())));
     if (words.length === 0)
         return '""';
     // Filter stopwords to improve BM25 ranking — common terms like "update",
@@ -48,11 +67,11 @@ function sanitizeQuery(query, mode = "AND") {
     const final = meaningful.length > 0 ? meaningful : words;
     return final.map((w) => `"${w}"`).join(mode === "OR" ? " OR " : " ");
 }
-function sanitizeTrigramQuery(query, mode = "AND") {
+export function sanitizeTrigramQuery(query, mode = "AND") {
     const cleaned = query.replace(/["'(){}[\]*:^~]/g, "").trim();
     if (cleaned.length < 3)
         return "";
-    const words = cleaned.split(/\s+/).filter((w) => w.length >= 3);
+    const words = dedupeTokens(cleaned.split(/\s+/).filter((w) => w.length >= 3));
     if (words.length === 0)
         return "";
     const meaningful = words.filter((w) => !STOPWORDS.has(w.toLowerCase()));
@@ -280,6 +299,13 @@ export class ContentStore {
     // search performance. SQLite's built-in 'optimize' merges b-tree segments.
     #insertCount = 0;
     static OPTIMIZE_EVERY = 50;
+    // Fuzzy correction cache (process-local LRU). fuzzyCorrect() hits the vocab
+    // DB and runs levenshtein against every candidate within length tolerance,
+    // which is CPU-linear in |candidates|. Repeated queries ("erro", "erro" …)
+    // recompute the same answer. The vocabulary table is insert-only, so cache
+    // entries only become stale when new words enter — we clear on actual insert.
+    #fuzzyCache = new Map();
+    static FUZZY_CACHE_SIZE = 256;
     constructor(dbPath) {
         const Database = loadDatabase();
         this.#dbPath =
@@ -332,7 +358,9 @@ export class ContentStore {
         label TEXT NOT NULL,
         chunk_count INTEGER NOT NULL DEFAULT 0,
         code_chunk_count INTEGER NOT NULL DEFAULT 0,
-        indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
+        indexed_at TEXT NOT NULL DEFAULT (datetime('now')),
+        file_path TEXT,
+        content_hash TEXT
       );
 
       CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING fts5(
@@ -357,11 +385,20 @@ export class ContentStore {
 
       CREATE INDEX IF NOT EXISTS idx_sources_label ON sources(label);
     `);
+        // Stale detection columns — safe for existing DBs (ALTER is O(1) in SQLite)
+        try {
+            this.#db.exec("ALTER TABLE sources ADD COLUMN file_path TEXT");
+        }
+        catch { /* already exists */ }
+        try {
+            this.#db.exec("ALTER TABLE sources ADD COLUMN content_hash TEXT");
+        }
+        catch { /* already exists */ }
     }
     #prepareStatements() {
         // Write path
-        this.#stmtInsertSourceEmpty = this.#db.prepare("INSERT INTO sources (label, chunk_count, code_chunk_count) VALUES (?, 0, 0)");
-        this.#stmtInsertSource = this.#db.prepare("INSERT INTO sources (label, chunk_count, code_chunk_count) VALUES (?, ?, ?)");
+        this.#stmtInsertSourceEmpty = this.#db.prepare("INSERT INTO sources (label, chunk_count, code_chunk_count, file_path, content_hash) VALUES (?, 0, 0, ?, ?)");
+        this.#stmtInsertSource = this.#db.prepare("INSERT INTO sources (label, chunk_count, code_chunk_count, file_path, content_hash) VALUES (?, ?, ?, ?, ?)");
         this.#stmtInsertChunk = this.#db.prepare("INSERT INTO chunks (title, content, source_id, content_type) VALUES (?, ?, ?, ?)");
         this.#stmtInsertChunkTrigram = this.#db.prepare("INSERT INTO chunks_trigram (title, content, source_id, content_type) VALUES (?, ?, ?, ?)");
         this.#stmtInsertVocab = this.#db.prepare("INSERT OR IGNORE INTO vocabulary (word) VALUES (?)");
@@ -551,7 +588,7 @@ export class ContentStore {
        ORDER BY c.rowid`);
         this.#stmtSourceChunkCount = this.#db.prepare("SELECT chunk_count FROM sources WHERE id = ?");
         this.#stmtChunkContent = this.#db.prepare("SELECT content FROM chunks WHERE source_id = ?");
-        this.#stmtSourceMeta = this.#db.prepare("SELECT label, chunk_count, code_chunk_count, indexed_at FROM sources WHERE label = ?");
+        this.#stmtSourceMeta = this.#db.prepare("SELECT label, chunk_count, code_chunk_count, indexed_at, file_path, content_hash FROM sources WHERE label = ?");
         this.#stmtStats = this.#db.prepare(`
       SELECT
         (SELECT COUNT(*) FROM sources) AS sources,
@@ -572,7 +609,10 @@ export class ContentStore {
         const text = content ?? readFileSync(path, "utf-8");
         const label = source ?? path ?? "untitled";
         const chunks = this.#chunkMarkdown(text);
-        return withRetry(() => this.#insertChunks(chunks, label, text));
+        // Stale detection: store file_path + SHA-256 for file-backed sources
+        const filePath = path ?? undefined;
+        const contentHash = filePath ? createHash("sha256").update(text).digest("hex") : undefined;
+        return withRetry(() => this.#insertChunks(chunks, label, text, filePath, contentHash));
     }
     // ── Index Plain Text ──
     /**
@@ -619,7 +659,7 @@ export class ContentStore {
      * into both FTS5 tables within a transaction and extracts vocabulary.
      * Uses cached prepared statements from #prepareStatements().
      */
-    #insertChunks(chunks, label, text) {
+    #insertChunks(chunks, label, text, filePath, contentHash) {
         const codeChunks = chunks.filter((c) => c.hasCode).length;
         // Atomic dedup + insert: delete previous source with same label,
         // then insert new content — all within a single transaction.
@@ -629,10 +669,10 @@ export class ContentStore {
             this.#stmtDeleteChunksTrigramByLabel.run(label);
             this.#stmtDeleteSourcesByLabel.run(label);
             if (chunks.length === 0) {
-                const info = this.#stmtInsertSourceEmpty.run(label);
+                const info = this.#stmtInsertSourceEmpty.run(label, filePath ?? null, contentHash ?? null);
                 return Number(info.lastInsertRowid);
             }
-            const info = this.#stmtInsertSource.run(label, chunks.length, codeChunks);
+            const info = this.#stmtInsertSource.run(label, chunks.length, codeChunks, filePath ?? null, contentHash ?? null);
             const sourceId = Number(info.lastInsertRowid);
             for (const chunk of chunks) {
                 const ct = chunk.hasCode ? "code" : "prose";
@@ -733,20 +773,38 @@ export class ContentStore {
         const word = query.toLowerCase().trim();
         if (word.length < 3)
             return null;
+        // Cache hit: promote to tail (Map preserves insertion order → LRU).
+        if (this.#fuzzyCache.has(word)) {
+            const cached = this.#fuzzyCache.get(word) ?? null;
+            this.#fuzzyCache.delete(word);
+            this.#fuzzyCache.set(word, cached);
+            return cached;
+        }
         const maxDist = maxEditDistance(word.length);
         const candidates = this.#stmtFuzzyVocab.all(word.length - maxDist, word.length + maxDist);
         let bestWord = null;
         let bestDist = maxDist + 1;
+        let exactMatch = false;
         for (const { word: candidate } of candidates) {
-            if (candidate === word)
-                return null; // exact match — no correction
+            if (candidate === word) {
+                exactMatch = true;
+                break;
+            }
             const dist = levenshtein(word, candidate);
             if (dist < bestDist) {
                 bestDist = dist;
                 bestWord = candidate;
             }
         }
-        return bestDist <= maxDist ? bestWord : null;
+        const result = exactMatch ? null : bestDist <= maxDist ? bestWord : null;
+        // Evict the oldest entry before insert if we hit the size cap.
+        if (this.#fuzzyCache.size >= _a.FUZZY_CACHE_SIZE) {
+            const oldestKey = this.#fuzzyCache.keys().next().value;
+            if (oldestKey !== undefined)
+                this.#fuzzyCache.delete(oldestKey);
+        }
+        this.#fuzzyCache.set(word, result);
+        return result;
     }
     // ── Reciprocal Rank Fusion (Cormack et al. 2009) ──
     #rrfSearch(query, limit, source, contentType, sourceMatchMode = "like") {
@@ -818,6 +876,8 @@ export class ContentStore {
     }
     // ── Unified Fallback Search ──
     searchWithFallback(query, limit = 3, source, contentType, sourceMatchMode = "like") {
+        // Step 0: Auto-refresh stale file-backed sources before searching
+        this.#refreshStaleSources();
         // Step 1: RRF fusion (porter OR + trigram OR → merge)
         const rrfResults = this.#rrfSearch(query, limit, source, contentType, sourceMatchMode);
         if (rrfResults.length > 0) {
@@ -844,12 +904,44 @@ export class ContentStore {
         }
         return [];
     }
+    /** Number of sources auto-refreshed in the last searchWithFallback call. */
+    lastRefreshCount = 0;
+    /**
+     * Check all file-backed sources for staleness and auto re-index changed files.
+     * Uses mtime as a fast gate — only computes SHA-256 when mtime has advanced
+     * past indexed_at. Gracefully skips deleted files and non-file sources.
+     */
+    #refreshStaleSources() {
+        this.lastRefreshCount = 0;
+        const sources = this.#db.prepare("SELECT label, file_path, content_hash, indexed_at FROM sources WHERE file_path IS NOT NULL").all();
+        for (const src of sources) {
+            try {
+                if (!existsSync(src.file_path))
+                    continue; // file deleted — keep cached results
+                const mtime = statSync(src.file_path).mtime;
+                const indexedAt = new Date(src.indexed_at + "Z");
+                if (mtime <= indexedAt)
+                    continue; // file unchanged — fast path
+                // mtime advanced — check hash to confirm real change (not just touch)
+                const newContent = readFileSync(src.file_path, "utf-8");
+                const newHash = createHash("sha256").update(newContent).digest("hex");
+                if (newHash === src.content_hash)
+                    continue; // content identical — skip
+                // File genuinely changed — re-index
+                this.index({ path: src.file_path, source: src.label });
+                this.lastRefreshCount++;
+            }
+            catch {
+                // Graceful degradation — never break search for stale detection
+            }
+        }
+    }
     // ── Sources ──
     getSourceMeta(label) {
         const row = this.#stmtSourceMeta.get(label);
         if (!row)
             return null;
-        return { label: row.label, chunkCount: row.chunk_count, codeChunkCount: row.code_chunk_count, indexedAt: row.indexed_at };
+        return { label: row.label, chunkCount: row.chunk_count, codeChunkCount: row.code_chunk_count, indexedAt: row.indexed_at, filePath: row.file_path ?? null, contentHash: row.content_hash ?? null };
     }
     listSources() {
         return this.#stmtListSources.all();
@@ -955,11 +1047,18 @@ export class ContentStore {
             .split(/[^\p{L}\p{N}_-]+/u)
             .filter((w) => w.length >= 3 && !STOPWORDS.has(w));
         const unique = [...new Set(words)];
+        let inserted = 0;
         this.#db.transaction(() => {
             for (const word of unique) {
-                this.#stmtInsertVocab.run(word);
+                const info = this.#stmtInsertVocab.run(word);
+                inserted += info.changes;
             }
         })();
+        // Invalidate fuzzy cache when new vocab words actually land. INSERT OR
+        // IGNORE reports changes=0 for duplicates, so re-indexing identical
+        // content does not thrash the cache during iterative workflows.
+        if (inserted > 0)
+            this.#fuzzyCache.clear();
     }
     // ── Chunking ──
     #chunkMarkdown(text, maxChunkBytes = MAX_CHUNK_BYTES) {

package/build/truncate.d.ts

@@ -11,6 +11,9 @@
  * "... [truncated]" is appended. The result is NOT guaranteed to be valid
  * JSON after truncation — it is suitable only for display/logging.
  *
+ * The returned string is always <= `maxBytes` bytes. When `maxBytes` is
+ * smaller than the marker, the marker itself is byte-safely truncated.
+ *
  * @param value    - Any JSON-serializable value.
  * @param maxBytes - Maximum byte length of the returned string.
  * @param indent   - JSON indentation spaces (default 2). Pass 0 for compact.
@@ -30,6 +33,9 @@ export declare function escapeXML(str: string): string;
  * byte-safe slice with an ellipsis appended. Useful for single-value fields
  * (e.g., tool response strings) where head+tail splitting is not needed.
  *
+ * The returned string is always <= `maxBytes` bytes. When `maxBytes` is
+ * smaller than the ellipsis marker, the marker itself is byte-safely truncated.
+ *
  * @param str      - Input string.
  * @param maxBytes - Hard byte cap.
  */

package/build/truncate.js

@@ -6,6 +6,43 @@
  * consumer can import them without pulling in the full store or executor.
  */
 // ─────────────────────────────────────────────────────────
+// Internal: byte-safe prefix
+// ─────────────────────────────────────────────────────────
+/**
+ * Return the longest character-prefix of `str` whose UTF-8 encoding is at
+ * most `maxBytes` bytes. Uses binary search to avoid O(n²) scanning. Returns
+ * "" when `maxBytes` is <= 0 so callers never exceed their budget.
+ *
+ * Guards against splitting a UTF-16 surrogate pair: if the prefix would end
+ * on a lone high surrogate, back off one code unit so the result round-trips
+ * through UTF-8 without producing a U+FFFD replacement character.
+ */
+function byteSafePrefix(str, maxBytes) {
+    if (maxBytes <= 0)
+        return "";
+    if (Buffer.byteLength(str) <= maxBytes)
+        return str;
+    let lo = 0;
+    let hi = str.length;
+    while (lo < hi) {
+        const mid = (lo + hi + 1) >> 1;
+        if (Buffer.byteLength(str.slice(0, mid)) <= maxBytes) {
+            lo = mid;
+        }
+        else {
+            hi = mid - 1;
+        }
+    }
+    // If we landed between a high and low surrogate, back off so the prefix
+    // ends on a valid code point boundary.
+    if (lo > 0) {
+        const code = str.charCodeAt(lo - 1);
+        if (code >= 0xd800 && code <= 0xdbff)
+            lo -= 1;
+    }
+    return str.slice(0, lo);
+}
+// ─────────────────────────────────────────────────────────
 // JSON truncation
 // ─────────────────────────────────────────────────────────
 /**
@@ -14,6 +51,9 @@
  * "... [truncated]" is appended. The result is NOT guaranteed to be valid
  * JSON after truncation — it is suitable only for display/logging.
  *
+ * The returned string is always <= `maxBytes` bytes. When `maxBytes` is
+ * smaller than the marker, the marker itself is byte-safely truncated.
+ *
  * @param value    - Any JSON-serializable value.
  * @param maxBytes - Maximum byte length of the returned string.
  * @param indent   - JSON indentation spaces (default 2). Pass 0 for compact.
@@ -22,24 +62,13 @@ export function truncateJSON(value, maxBytes, indent = 2) {
     const serialized = JSON.stringify(value, null, indent) ?? "null";
     if (Buffer.byteLength(serialized) <= maxBytes)
         return serialized;
-    // Find the largest character slice that stays within maxBytes once encoded.
-    // Buffer.byteLength is O(n) but we only call it once per truncation.
     const marker = "... [truncated]";
     const markerBytes = Buffer.byteLength(marker);
-    const budget = maxBytes - markerBytes;
-    // Binary-search for the right character count — avoids O(n²) scanning.
-    let lo = 0;
-    let hi = serialized.length;
-    while (lo < hi) {
-        const mid = (lo + hi + 1) >> 1;
-        if (Buffer.byteLength(serialized.slice(0, mid)) <= budget) {
-            lo = mid;
-        }
-        else {
-            hi = mid - 1;
-        }
-    }
-    return serialized.slice(0, lo) + marker;
+    // Degenerate budget: can't fit serialized content + marker. Fit as much of
+    // the marker as we can so the return still honors `maxBytes`.
+    if (maxBytes <= markerBytes)
+        return byteSafePrefix(marker, maxBytes);
+    return byteSafePrefix(serialized, maxBytes - markerBytes) + marker;
 }
 // ─────────────────────────────────────────────────────────
 // XML / HTML escaping
@@ -68,6 +97,9 @@ export function escapeXML(str) {
  * byte-safe slice with an ellipsis appended. Useful for single-value fields
  * (e.g., tool response strings) where head+tail splitting is not needed.
  *
+ * The returned string is always <= `maxBytes` bytes. When `maxBytes` is
+ * smaller than the ellipsis marker, the marker itself is byte-safely truncated.
+ *
  * @param str      - Input string.
  * @param maxBytes - Hard byte cap.
  */
@@ -76,17 +108,7 @@ export function capBytes(str, maxBytes) {
         return str;
     const marker = "...";
     const markerBytes = Buffer.byteLength(marker);
-    const budget = maxBytes - markerBytes;
-    let lo = 0;
-    let hi = str.length;
-    while (lo < hi) {
-        const mid = (lo + hi + 1) >> 1;
-        if (Buffer.byteLength(str.slice(0, mid)) <= budget) {
-            lo = mid;
-        }
-        else {
-            hi = mid - 1;
-        }
-    }
-    return str.slice(0, lo) + marker;
+    if (maxBytes <= markerBytes)
+        return byteSafePrefix(marker, maxBytes);
+    return byteSafePrefix(str, maxBytes - markerBytes) + marker;
 }

package/build/types.d.ts

@@ -29,6 +29,14 @@ export interface SessionEvent {
     data: string;
     priority: number;
     data_hash: string;
+    /**
+     * Best-effort project attribution for this event.
+     * Empty string means unattributed/unknown.
+     */
+    project_dir?: string;
+    attribution_source?: string;
+    /** 0..1 confidence score for project attribution. */
+    attribution_confidence?: number;
 }
 /**
  * Result returned by PolyglotExecutor after running a code snippet.