@duckmind/dm-darwin-x64 0.66.4 → 0.66.6

package/extensions/dm-lens/clients/tree-sitter-client.js

@@ -68,6 +68,9 @@ const TYPESCRIPT_SQL_KNOWN_PACKAGES = new Set([
   "@prisma/client"
 ]);
 const TYPESCRIPT_SQL_IMPORTS = new WeakMap;
+export function wasmQueryInput(queryKey) {
+  return { languageId: "query", source: queryKey };
+}
 const NOT_PARSED = { parsed: false };
 function createParserCounters() {
   return {
@@ -99,10 +102,27 @@ function grammarFileStamp(filePath) {
     return;
   }
 }
-function isTreeSitterWasmAbortError(error) {
-  const message = error instanceof Error ? error.message : String(error);
-  return message.includes("Aborted") || message.includes("abort()");
+const UNRESOLVED_IMPORT_MESSAGE = "resolved is not a function";
+const WASM_TRAP_MESSAGES = [
+  "memory access out of bounds",
+  "table index is out of bounds",
+  "null function or function signature mismatch"
+];
+export function classifyTreeSitterWasmError(thrown) {
+  const message = thrown instanceof Error ? thrown.message : String(thrown);
+  if (message.includes("Aborted") || message.includes("abort()")) {
+    return "abort";
+  }
+  const { WebAssembly } = globalThis;
+  if (thrown instanceof WebAssembly.RuntimeError)
+    return "trap";
+  if (thrown instanceof TypeError && message === UNRESOLVED_IMPORT_MESSAGE) {
+    return "trap";
+  }
+  return WASM_TRAP_MESSAGES.some((trap) => message.includes(trap)) ? "trap" : undefined;
 }
+export const WASM_TRAP_BUDGET = 3;
+export const GRAMMAR_TRAP_LATCH_THRESHOLD = WASM_TRAP_BUDGET - 1;
 const RESOLUTION_ERROR_CODES = new Set([
   "ERR_MODULE_NOT_FOUND",
   "MODULE_NOT_FOUND",
@@ -161,6 +181,9 @@ export class TreeSitterClient {
   static QUERY_BATCH_CACHE_MAX_ENTRIES = 256;
   queryCache = new BoundedFifoMap(TreeSitterClient.QUERY_CACHE_MAX_ENTRIES);
   queryBatchCache = new BoundedFifoMap(TreeSitterClient.QUERY_BATCH_CACHE_MAX_ENTRIES);
+  queryBatchBuilds = new Map;
+  queryBatchHealEpoch = 0;
+  queryBatchInputs = new Map;
   queryCacheCap() {
     const value = Number.parseInt(process.env.DM_LENS_TREE_SITTER_QUERY_CACHE_CAP ?? "", 10);
     return Number.isSafeInteger(value) && value > 0 ? value : TreeSitterClient.QUERY_CACHE_MAX_ENTRIES;
@@ -176,12 +199,54 @@ export class TreeSitterClient {
     for (const [, dropped] of evicted)
       dropped?.query?.delete?.();
   }
-  cacheQueryBatch(key, value) {
+  cacheQueryBatch(key, value, inputKeys = []) {
+    const replaced = this.queryBatchCache.get(key);
     this.queryBatchCache.delete(key);
+    if (replaced)
+      this.retireQueryBatch(replaced);
     const evicted = this.queryBatchCache.setMaxEntries(this.queryBatchCacheCap());
     evicted.push(...this.queryBatchCache.set(key, value));
+    for (const [evictedKey] of evicted)
+      this.queryBatchInputs.delete(evictedKey);
+    this.queryBatchInputs.set(key, new Set(inputKeys));
     for (const [, dropped] of evicted)
-      dropped?.query?.delete?.();
+      this.retireQueryBatch(dropped);
+  }
+  disposeQueryBatch(batch) {
+    if (batch.disposed)
+      return;
+    batch.disposed = true;
+    batch.query?.delete?.();
+  }
+  retireQueryBatch(batch) {
+    if (!batch)
+      return;
+    batch.retired = true;
+    if (batch.users === 0) {
+      this.disposeQueryBatch(batch);
+    } else {
+      incrementDegradationCount({
+        kind: "wasm-query-batch-disposal-deferred",
+        subject: "web-tree-sitter",
+        reason: "deferred batch disposal while a scan is active"
+      });
+    }
+  }
+  retainQueryBatch(batch) {
+    batch.users++;
+  }
+  releaseQueryBatch(batch) {
+    batch.users--;
+    if (batch.retired && batch.users === 0)
+      this.disposeQueryBatch(batch);
+  }
+  takeQueryBatchLeases(lease, batch) {
+    lease.batch = batch;
+    if (!batch)
+      return null;
+    batch.users += lease.count;
+    lease.count = 0;
+    return batch;
   }
   queryBatchLoadFailures = new Map;
   queryLoader = new TreeSitterQueryLoader;
@@ -191,6 +256,12 @@ export class TreeSitterClient {
   activeMeasurements = 0;
   onWasmAbort;
   wasmAborted = false;
+  wasmTraps = 0;
+  reportedTraps = new WeakSet;
+  trappedInputs = new Map;
+  grammarTrapInputs = new Map;
+  retiredGrammars = new Map;
+  activeWasmInput;
   grammarDirResolutionDeps;
   constructor(verbose = false, onWasmAbort, grammarDirResolutionDeps) {
     this.grammarDirResolutionDeps = grammarDirResolutionDeps ?? {
@@ -214,20 +285,110 @@ export class TreeSitterClient {
     if (measurement)
       measurement[key] += amount;
   }
-  reportWasmAbort(error) {
-    if (!isTreeSitterWasmAbortError(error))
+  reportWasmAbort(thrown, input = this.activeWasmInput) {
+    const failure = classifyTreeSitterWasmError(thrown);
+    if (!failure)
       return false;
+    const message = thrown instanceof Error ? thrown.message : String(thrown);
+    if (failure === "trap") {
+      if (typeof thrown === "object" && thrown !== null) {
+        if (this.reportedTraps.has(thrown))
+          return this.wasmAborted;
+        this.reportedTraps.add(thrown);
+      }
+      if (input) {
+        const key = this.wasmInputKey(input);
+        this.countGrammarTrapInput(input.languageId, key);
+        const entry = this.trappedInputs.get(key);
+        if (entry) {
+          entry.traps++;
+          incrementDegradationCount({
+            kind: "wasm-trap",
+            subject: "web-tree-sitter",
+            reason: `input charged: ${message}`
+          });
+          return false;
+        }
+        this.trappedInputs.set(key, {
+          traps: 1,
+          by: input.caller,
+          source: input.languageId === "query" ? input.source : undefined
+        });
+      }
+      if (++this.wasmTraps <= WASM_TRAP_BUDGET) {
+        incrementDegradationCount({
+          kind: "wasm-trap",
+          subject: "web-tree-sitter",
+          reason: message
+        });
+        this.parsers.clear();
+        this.treeCache.clear();
+        return false;
+      }
+    }
     if (!this.wasmAborted) {
       this.wasmAborted = true;
       recordDegradation({
         kind: "wasm-abort",
         subject: "web-tree-sitter",
-        reason: error instanceof Error ? error.message : String(error)
+        reason: failure === "trap" ? `${WASM_TRAP_BUDGET} wasm traps absorbed, next one: ${message}` : message
       });
       this.onWasmAbort?.();
     }
     return true;
   }
+  countGrammarTrapInput(languageId, key) {
+    const grammarFile = LANGUAGE_TO_GRAMMAR[languageId];
+    if (!grammarFile)
+      return;
+    let inputs = this.grammarTrapInputs.get(languageId);
+    if (!inputs)
+      this.grammarTrapInputs.set(languageId, inputs = new Set);
+    inputs.add(key);
+    if (inputs.size >= GRAMMAR_TRAP_LATCH_THRESHOLD) {
+      this.retiredGrammars.set(languageId, grammarFile);
+      this.recordGrammarBlocked(grammarFile);
+    }
+  }
+  wasmInputKey(input) {
+    input.key ??= crypto.createHash("sha256").update(input.languageId).update("\x00").update(input.source).digest("hex");
+    return input.key;
+  }
+  wasmInputTraps(input) {
+    if (!input || this.trappedInputs.size === 0)
+      return 0;
+    return this.trappedInputs.get(this.wasmInputKey(input))?.traps ?? 0;
+  }
+  clearWasmInput(input, internalBatchHeal = false) {
+    if (!input || this.trappedInputs.size === 0)
+      return;
+    const key = this.wasmInputKey(input);
+    const entry = this.trappedInputs.get(key);
+    const source = entry?.source;
+    if (entry?.by === input.caller) {
+      this.trappedInputs.delete(key);
+      this.grammarTrapInputs.get(input.languageId)?.delete(key);
+      if (!internalBatchHeal)
+        this.queryBatchHealEpoch++;
+      for (const [batchKey, batch] of this.queryBatchCache) {
+        const inputs = this.queryBatchInputs.get(batchKey);
+        if (!source || !inputs?.has(source))
+          continue;
+        this.queryBatchCache.delete(batchKey);
+        this.queryBatchInputs.delete(batchKey);
+        this.retireQueryBatch(batch);
+      }
+    }
+  }
+  notParsed(thrown, input) {
+    if (!input || classifyTreeSitterWasmError(thrown) !== "trap") {
+      return NOT_PARSED;
+    }
+    return {
+      parsed: false,
+      wasmTrap: this.wasmInputTraps(input) > 1 ? "charged" : "retry"
+    };
+  }
   getRuntimeStats() {
     return {
       languagesLoaded: this.languages.size,
@@ -373,6 +534,16 @@ export class TreeSitterClient {
     this.grammarLastNotifiedDelayMs.clear();
     this.poisonedGrammarPaths.clear();
     this.staleReportedGrammarPaths.clear();
+    for (const grammarFile of this.retiredGrammars.values()) {
+      this.recordGrammarBlocked(grammarFile);
+    }
+  }
+  recordGrammarBlocked(grammarFile) {
+    recordDegradationOnce({
+      kind: "grammar-blocked",
+      subject: grammarFile,
+      reason: `grammar retired after ${GRAMMAR_TRAP_LATCH_THRESHOLD} distinct WASM traps; ` + "structural analysis is unavailable for this language until restart"
+    });
   }
   reportPoisonedGrammarFile(candidate, grammarFile) {
     this.refreshGrammarSessionLatches();
@@ -643,11 +814,14 @@ export class TreeSitterClient {
   }
   resetLoadStateForSession() {
     this.webTreeSitterLoadFailed = false;
+    this.refreshGrammarSessionLatches();
   }
   async loadLanguage(languageId) {
     if (this.wasmAborted)
       return null;
     this.dbg(`Loading language: ${languageId}`);
+    if (this.retiredGrammars.has(languageId))
+      return null;
     if (this.languages.has(languageId)) {
       this.dbg(`Language ${languageId} already loaded`);
       return this.languages.get(languageId);
@@ -695,7 +869,9 @@ export class TreeSitterClient {
       }
       return language;
     } catch (err) {
-      if (!this.reportWasmAbort(err)) {
+      if (classifyTreeSitterWasmError(err) === "abort") {
+        this.reportWasmAbort(err);
+      } else {
         this.recordGrammarLoadFailure(grammarPath, grammarFile, err);
       }
       this.dbg(`Language load error: ${err}`);
@@ -717,13 +893,13 @@ export class TreeSitterClient {
     return parser;
   }
   async parseFile(filePath, languageId, contentOverride) {
-    const outcome = await this.parseFileAndUse(filePath, languageId, contentOverride, (tree) => tree);
+    const outcome = await this.parseFileAndUse(filePath, languageId, contentOverride, (tree) => tree, "parseFile");
     return outcome.parsed ? outcome.value : null;
   }
-  async withParsedTree(filePath, languageId, contentOverride, consume) {
-    return this.parseFileAndUse(filePath, languageId, contentOverride, consume);
+  async withParsedTree(filePath, languageId, contentOverride, consume, caller = "withParsedTree") {
+    return this.parseFileAndUse(filePath, languageId, contentOverride, consume, caller);
   }
-  async parseFileAndUse(filePath, languageId, contentOverride, consume) {
+  async parseFileAndUse(filePath, languageId, contentOverride, consume, caller) {
     this.dbg(`Parsing ${filePath} with language ${languageId}`);
     const parser = await this.getParser(languageId);
     if (!parser) {
@@ -731,9 +907,14 @@ export class TreeSitterClient {
       return NOT_PARSED;
     }
     let tree;
+    let input;
     try {
       const content = contentOverride ?? fs.readFileSync(filePath, "utf-8");
       this.dbg(`File content length: ${content.length}`);
+      input = { languageId, source: content };
+      if (this.wasmInputTraps(input) > 1) {
+        return { parsed: false, wasmTrap: "charged" };
+      }
       const cachedTree = this.treeCache.get(filePath, content, languageId);
       if (cachedTree) {
         this.dbg(`Using cached tree for ${filePath}`);
@@ -753,15 +934,28 @@ export class TreeSitterClient {
         this.treeCache.set(filePath, content, languageId, tree);
       }
     } catch (err) {
-      this.reportWasmAbort(err);
+      this.reportWasmAbort(err, input);
       this.dbg(`Parse error: ${err}`);
-      return NOT_PARSED;
+      return this.notParsed(err, input);
     }
+    this.clearWasmInput(input);
+    input = { ...input, caller };
     try {
-      return { parsed: true, value: consume(tree) };
-    } catch (error) {
-      this.reportWasmAbort(error);
-      throw error;
+      this.activeWasmInput = input;
+      const trapsBefore = this.wasmInputTraps(input);
+      const value = consume(tree);
+      if (this.wasmInputTraps(input) <= trapsBefore) {
+        this.clearWasmInput(input);
+      }
+      return { parsed: true, value };
+    } catch (thrown) {
+      this.reportWasmAbort(thrown);
+      if (classifyTreeSitterWasmError(thrown)) {
+        return this.notParsed(thrown, input);
+      }
+      throw thrown;
+    } finally {
+      this.activeWasmInput = undefined;
     }
   }
   extractInjections(filePath, content) {
@@ -821,7 +1015,7 @@ export class TreeSitterClient {
     return language !== null;
   }
   getLanguage(languageId) {
-    if (this.wasmAborted)
+    if (this.wasmAborted || this.retiredGrammars.has(languageId))
       return null;
     return this.languages.get(languageId) || null;
   }
@@ -864,7 +1058,7 @@ export class TreeSitterClient {
     const compiled = await this.compileRawQuery(queryDef.id, queryDef.query, queryDef.metavars, languageId, queryDef.post_filter, queryDef.post_filter_params);
     if (!compiled)
       return [];
-    const matches = await this.searchFileWithQuery(filePath, compiled.query, compiled.metavars, languageId, queryDef.id, compiled.postFilter, compiled.postFilterParams, contentOverride);
+    const matches = await this.searchFileWithQuery(filePath, compiled.query, compiled.metavars, languageId, `${queryDef.id}\x00${queryDef.query}`, compiled.postFilter, compiled.postFilterParams, contentOverride);
     const maxResults = options.maxResults ?? 50;
     return matches.slice(0, maxResults);
   }
@@ -877,7 +1071,7 @@ export class TreeSitterClient {
         return [];
     }
     const maxResults = options.maxResults ?? 50;
-    const batch = await this.compileQueryBatch(queryDefs, languageId);
+    const batch = await this.compileQueryBatch(queryDefs, languageId, true);
     if (!batch) {
       const results = [];
       for (const queryDef of queryDefs) {
@@ -931,7 +1125,7 @@ export class TreeSitterClient {
         this.reportWasmAbort(err);
         this.dbg(`Batched query matching error: ${err}`);
       }
-    });
+    }, `runQueriesOnFile\x00${batch.key}`).finally(() => this.releaseQueryBatch(batch));
     const results = [];
     for (let i = 0;i < batch.entries.length; i++) {
       for (const match of perQuery.get(i) ?? []) {
@@ -940,42 +1134,93 @@ export class TreeSitterClient {
     }
     return results;
   }
-  async compileQueryBatch(queryDefs, languageId) {
+  async compileQueryBatch(queryDefs, languageId, retain = false) {
     const identity = crypto.createHash("sha256").update(JSON.stringify(queryDefs)).digest("hex");
     const cacheKey = this.getQueryCacheKey(`batch:${identity}`, languageId);
     const cached = this.queryBatchCache.get(cacheKey);
     if (cached !== undefined) {
       this.queryBatchCache.delete(cacheKey);
       this.queryBatchCache.set(cacheKey, cached);
+      if (retain && cached)
+        this.retainQueryBatch(cached);
       return cached;
     }
+    const inFlight = this.queryBatchBuilds.get(cacheKey);
+    if (inFlight) {
+      if (retain) {
+        const decided = inFlight.lease.batch;
+        if (decided === undefined)
+          inFlight.lease.count++;
+        else if (decided)
+          this.retainQueryBatch(decided);
+      }
+      return inFlight.promise;
+    }
+    const lease = { count: retain ? 1 : 0, batch: undefined };
+    const promise = this.compileQueryBatchOnce(queryDefs, languageId, lease);
+    const build = { promise, lease };
+    this.queryBatchBuilds.set(cacheKey, build);
+    try {
+      return await promise;
+    } finally {
+      if (this.queryBatchBuilds.get(cacheKey) === build) {
+        this.queryBatchBuilds.delete(cacheKey);
+      }
+    }
+  }
+  async compileQueryBatchOnce(queryDefs, languageId, lease) {
+    const identity = crypto.createHash("sha256").update(JSON.stringify(queryDefs)).digest("hex");
+    const cacheKey = this.getQueryCacheKey(`batch:${identity}`, languageId);
+    const inputKeys = [
+      cacheKey,
+      ...queryDefs.map((queryDef) => this.getQueryCacheKey(`raw:${queryDef.id}:${queryDef.query}`, languageId))
+    ];
+    const cached = this.queryBatchCache.get(cacheKey);
+    if (cached !== undefined) {
+      this.queryBatchCache.delete(cacheKey);
+      this.queryBatchCache.set(cacheKey, cached);
+      return this.takeQueryBatchLeases(lease, cached);
+    }
+    const healEpoch = this.queryBatchHealEpoch;
     const language = await this.loadLanguage(languageId);
     if (!language) {
       const failures = (this.queryBatchLoadFailures.get(cacheKey) ?? 0) + 1;
       if (failures >= QUERY_BATCH_MAX_LOAD_FAILURES) {
         this.dbg(`Batch: grammar for ${languageId} failed to load ${failures} times — caching miss`);
         this.queryBatchLoadFailures.delete(cacheKey);
-        this.cacheQueryBatch(cacheKey, null);
+        this.cacheQueryBatch(cacheKey, null, inputKeys);
       } else {
         this.queryBatchLoadFailures.set(cacheKey, failures);
       }
-      return null;
+      return this.takeQueryBatchLeases(lease, null);
     }
     this.queryBatchLoadFailures.delete(cacheKey);
+    let trapped = false;
+    const batchInput = wasmQueryInput(cacheKey);
     const build = async () => {
+      if (this.wasmInputTraps(batchInput) > 1)
+        return null;
       const Query = (await loadWebTreeSitter()).Query;
       const entries = [];
       const sources = [];
       const ownerOfPattern = [];
       for (const queryDef of queryDefs) {
         let patternCount;
+        const probeInput = wasmQueryInput(this.getQueryCacheKey(`raw:${queryDef.id}:${queryDef.query}`, languageId));
+        if (this.wasmInputTraps(probeInput) > 1) {
+          this.dbg(`Batch: skipping ${queryDef.id}, its compile trapped twice`);
+          continue;
+        }
         try {
           const probe = new Query(language, queryDef.query);
           patternCount = probe.patternCount();
           probe.delete?.();
+          this.clearWasmInput(probeInput, true);
         } catch (err) {
-          if (this.reportWasmAbort(err))
+          if (this.reportWasmAbort(err, probeInput))
             return null;
+          if (classifyTreeSitterWasmError(err) === "trap")
+            trapped = true;
           this.dbg(`Batch: skipping ${queryDef.id} for ${languageId}: ${err}`);
           this.reportQueryCompileFailure(queryDef.id, languageId, err);
           continue;
@@ -1000,16 +1245,33 @@ export class TreeSitterClient {
           this.dbg(`Batch pattern count mismatch for ${languageId} (${query.patternCount()} vs ${ownerOfPattern.length}) — falling back`);
           return null;
         }
-        return { query, entries, ownerOfPattern };
+        this.clearWasmInput(batchInput, true);
+        return {
+          query,
+          entries,
+          ownerOfPattern,
+          key: cacheKey,
+          users: 0,
+          retired: false,
+          disposed: false
+        };
       } catch (err) {
-        if (this.reportWasmAbort(err))
+        if (this.reportWasmAbort(err, batchInput))
           return null;
+        if (classifyTreeSitterWasmError(err) === "trap")
+          trapped = true;
         this.dbg(`Batch compile failed for ${languageId}: ${err}`);
         return null;
       }
     };
     const batch = await build();
-    this.cacheQueryBatch(cacheKey, batch);
+    const published = !trapped && this.queryBatchHealEpoch === healEpoch;
+    this.takeQueryBatchLeases(lease, batch);
+    if (published) {
+      this.cacheQueryBatch(cacheKey, batch, inputKeys);
+    } else {
+      this.retireQueryBatch(batch);
+    }
     return batch;
   }
   patternToQuery(pattern, languageId) {
@@ -1100,6 +1362,9 @@ export class TreeSitterClient {
       this.queryCache.set(cacheKey, cached);
       return cached;
     }
+    const input = wasmQueryInput(cacheKey);
+    if (this.wasmInputTraps(input) > 1)
+      return null;
     const language = await this.loadLanguage(languageId);
     if (!language) {
       this.dbg(`Could not load language ${languageId}`);
@@ -1117,10 +1382,11 @@ export class TreeSitterClient {
       const query = new Query(language, queryStr);
       this.dbg(`Query compiled with ${query.patternCount()} patterns`);
       const result = { query, metavars, postFilter, postFilterParams };
+      this.clearWasmInput(input);
       this.cacheQuery(cacheKey, result);
       return result;
     } catch (err) {
-      this.reportWasmAbort(err);
+      this.reportWasmAbort(err, input);
       this.dbg(`Query compilation failed: ${err}`);
       return null;
     }
@@ -1133,6 +1399,9 @@ export class TreeSitterClient {
       this.queryCache.set(cacheKey, cached);
       return cached;
     }
+    const input = wasmQueryInput(cacheKey);
+    if (this.wasmInputTraps(input) > 1)
+      return null;
     const language = await this.loadLanguage(languageId);
     if (!language)
       return null;
@@ -1140,10 +1409,11 @@ export class TreeSitterClient {
       const Query = (await loadWebTreeSitter()).Query;
       const query = new Query(language, queryStr);
       const result = { query, metavars, postFilter, postFilterParams };
+      this.clearWasmInput(input);
       this.cacheQuery(cacheKey, result);
       return result;
     } catch (err) {
-      this.reportWasmAbort(err);
+      this.reportWasmAbort(err, input);
       this.dbg(`Raw query compilation failed (${queryId}): ${err}`);
       this.reportQueryCompileFailure(queryId, languageId, err);
       return null;
@@ -2724,7 +2994,7 @@ export class TreeSitterClient {
     const predicates = query.textPredicates?.[match.patternIndex] ?? [];
     return predicates.every((fn) => fn(match.captures));
   }
-  async searchFileWithQuery(filePath, query, metavars, languageId, _originalPattern, postFilter, postFilterParams, contentOverride) {
+  async searchFileWithQuery(filePath, query, metavars, languageId, queryKey, postFilter, postFilterParams, contentOverride) {
     const matches = [];
     await this.parseFileAndUse(filePath, languageId, contentOverride, (tree) => {
       try {
@@ -2766,7 +3036,7 @@ export class TreeSitterClient {
         this.reportWasmAbort(err);
         this.dbg(`Query matching error: ${err}`);
       }
-    });
+    }, `runQueryOnFile\x00${queryKey}`);
     return matches;
   }
   collectFiles(dir, languageId, fileFilter) {