codedeep-mcp 0.1.0 → 0.2.0

package/dist/indexer/pipeline.js

@@ -24,7 +24,10 @@ function isUnchanged(prev, mtimeMs, size, language) {
         prev.size === size &&
         prev.language === language);
 }
-function hashContent(content) {
+// Exported so the note store's staleness check can re-hash an anchored file
+// from disk with the SAME algorithm the indexer records in FileInfo.contentHash
+// — staleness compares against DISK, not the (possibly-lagging) live index.
+export function hashContent(content) {
     return createHash('sha1').update(content).digest('hex').slice(0, 16);
 }
 export class Indexer {
@@ -41,6 +44,10 @@ export class Indexer {
     // preserves unseen cached entries — the watcher must know the rescan
     // it requested may not have covered everything.
     lastScanCompleteFlag = true;
+    // Languages whose grammar-load failure was already warned — dedupes the
+    // per-file warn in processFile (one line per failure EPISODE per language,
+    // not one per file). Entries clear on the next successful load.
+    grammarWarnedLangs = new Set();
     get lastScanComplete() {
         return this.lastScanCompleteFlag;
     }
@@ -62,8 +69,10 @@ export class Indexer {
     // work and retry what deserves retrying.
     async indexAll() {
         return this.runGuarded(async () => {
-            await initParser();
+            // Scan FIRST, then load only the grammars the repo actually contains —
+            // loading all 16 up front costs ~95MB RSS on a repo that needs one.
             const { files: current, complete } = await scanProject(this.config);
+            await this.warmUpGrammars(current);
             this.lastScanCompleteFlag = complete;
             this.total = current.length;
             await this.processBatched(current);
@@ -87,7 +96,6 @@ export class Indexer {
     }
     async indexChanged() {
         return this.runGuarded(async () => {
-            await initParser();
             const { files: current, complete } = await scanProject(this.config);
             this.lastScanCompleteFlag = complete;
             const previous = new Map(this.index.getAllFiles().map((f) => [f.path, f]));
@@ -114,6 +122,9 @@ export class Indexer {
                 this.ready = true;
                 return;
             }
+            // Load only the grammars the CHANGED files need — the common warm start
+            // (few or no changes) then loads few or no grammars at all.
+            await this.warmUpGrammars(toIndex);
             this.total = toIndex.length;
             await this.processBatched(toIndex);
             await this.persist();
@@ -129,7 +140,9 @@ export class Indexer {
         return ran ? outcome : 'dropped';
     }
     async indexFileInner(rawPath) {
-        await initParser();
+        // No up-front initParser: processFile ensures the ONE grammar this file
+        // needs right before parsing (a watcher event on an unchanged file then
+        // loads nothing at all).
         // Canonicalize to a project-relative POSIX path so the cache key
         // aligns with the scanner's `src/a.ts` form regardless of whether
         // the watcher emits an absolute path, a `./`-prefix, or Windows
@@ -247,6 +260,15 @@ export class Indexer {
         this.done = 1;
         return result;
     }
+    // Parallel bulk load of the grammars `files` need. A WARM-UP, not a
+    // correctness requirement (processFile re-ensures per-file), so failure is
+    // TOLERATED: one missing/corrupt .wasm must degrade that one language
+    // (per-file 'transient' + deduped warns), not abort indexing for every
+    // other language.
+    async warmUpGrammars(files) {
+        await initParser(files.map((f) => f.language)).catch((err) => log.warn(`Indexer: bulk grammar warm-up failed (${errMsg(err)}); ` +
+            `grammars will load per-file`));
+    }
     // Resolves `false` when a run is already in flight (the request is
     // dropped, not queued); `true` when the work ran to completion.
     async runGuarded(work) {
@@ -286,6 +308,44 @@ export class Indexer {
         }
         const absPath = join(this.config.projectRoot, file.path);
         const removed = () => this.index.removeFile(file.path) ? 'removed' : 'noop';
+        // Memoized per-language ensure — a resolved-promise await after the first
+        // load. Covers the watcher path (a NEW language can appear after the
+        // startup scan chose the initial grammar set). Runs BEFORE the content
+        // read (no point reading bytes a failed grammar can't parse — a
+        // permanently corrupt .wasm would otherwise cost one full-file read per
+        // affected file per rescan) and OUTSIDE the parse try/catch below: a
+        // grammar-LOAD failure says nothing about the FILE, so it returns
+        // 'transient' (existing symbols kept — genuinely-transient causes were
+        // already retried in place by ensureLanguage's bounded backoff), never
+        // cascade-deletes them the way an unparseable file does. The warn is
+        // deduped per LANGUAGE (5,000 Python files must not produce 5,000
+        // identical lines); the dedup entry clears on the next successful load so
+        // a NEW failure episode warns again.
+        try {
+            await initParser([file.language]);
+            this.grammarWarnedLangs.delete(file.language);
+        }
+        catch (err) {
+            // Preserve the pre-existing precedence "unreadable bytes always prune
+            // the entry" even when the grammar is down: without this, a file that
+            // is itself gone/unreadable would keep serving stale symbols from the
+            // persisted cache indefinitely. One cheap access() probe, only on the
+            // (rare) grammar-failure path.
+            try {
+                await fs.access(absPath, fs.constants.R_OK);
+            }
+            catch {
+                return removed();
+            }
+            if (!this.grammarWarnedLangs.has(file.language)) {
+                this.grammarWarnedLangs.add(file.language);
+                log.warn(`Indexer: grammar load failed for ${file.language} (first: ${file.path}): ` +
+                    `${errMsg(err)}. Files of this language are missing or stale in the ` +
+                    `index until the grammar loads (fix the installation; probed again on ` +
+                    `the next change or rescan). Existing symbols are kept.`);
+            }
+            return 'transient';
+        }
         let content;
         try {
             content = await fs.readFile(absPath, 'utf8');

package/dist/indexer/scanner.js

@@ -1,8 +1,13 @@
 import { open, readdir, stat } from 'node:fs/promises';
-import { join, relative, sep, posix } from 'node:path';
+import { join, relative, posix } from 'node:path';
 import picomatch from 'picomatch';
+import { toPosix } from '../fs-util.js';
 import { LANGUAGE_UNKNOWN } from '../types.js';
 import { log } from '../logger.js';
+// Re-exported for the scanner's indexer siblings (pipeline/watcher) — the
+// implementation lives in the neutral fs-util module so config.ts can use it
+// without importing the indexer layer.
+export { toPosix };
 const BYTE_CHECK_BUF_SIZE = 8192;
 const LANGUAGE_BY_EXT = {
     '.ts': 'typescript',
@@ -77,9 +82,6 @@ const BINARY_EXT = new Set([
     '.class', '.pyc', '.pyo', '.jar', '.war',
 ]);
 const GLOB_CHARS = /[*?[\]{}!]/;
-export function toPosix(p) {
-    return sep === '/' ? p : p.split(sep).join('/');
-}
 export function detectLanguage(filename) {
     const ext = posix.extname(toPosix(filename)).toLowerCase();
     return LANGUAGE_BY_EXT[ext] ?? null;

package/dist/indexer/watcher.js

@@ -320,6 +320,15 @@ export class Watcher {
                     this.pending.add(rel); // guard drop — retry
                 else if (outcome === 'cap-skipped')
                     capSkipped.push(rel);
+                // 'transient' (grammar-load failure) is deliberately NOT re-queued:
+                // the loader already retried in place (parser.ts's bounded
+                // backoff inside ensureLanguage covers the genuinely-transient case
+                // for EVERY caller), so a failure surviving that is durable — a
+                // corrupt/missing .wasm re-queued here would just cycle the retry
+                // tick. The file's existing symbols were kept; recovery rides the
+                // next fs event or rescan (langLoads self-resets). A per-path retry
+                // budget was tried and removed — it swallowed edits landing
+                // mid-budget and its counters leaked across interleaved outcomes.
             }
             catch (err) {
                 log.warn(`watcher: failed to index ${rel}: ${errMsg(err)}`);