@wrongstack/tools 0.265.1 → 0.268.0

package/dist/codebase-index/worker.js

@@ -2013,6 +2013,7 @@ async function loadGitignoreMatcher(projectRoot) {
 
 // src/codebase-index/indexer.ts
 var YIELD_EVERY_N = 50;
+var PARALLEL_BATCH = 20;
 function yieldEventLoop() {
   return new Promise((resolve2) => setImmediate(resolve2));
 }
@@ -2146,97 +2147,108 @@ async function runIndexerWithStore(store, opts) {
   if (!force) {
     for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
   }
-  for (let fi = 0; fi < files.length; fi++) {
-    const file = expectDefined(files[fi]);
-    opts.onProgress?.(fi + 1, files.length);
-    if (fi > 0 && fi % YIELD_EVERY_N === 0) {
+  for (let batchStart = 0; batchStart < files.length; batchStart += PARALLEL_BATCH) {
+    const batchEnd = Math.min(batchStart + PARALLEL_BATCH, files.length);
+    const batchFiles = files.slice(batchStart, batchEnd);
+    opts.onProgress?.(batchEnd, files.length);
+    if (batchStart > 0 && batchStart % YIELD_EVERY_N === 0) {
       await yieldEventLoop();
       throwIfAborted(signal);
     }
-    let stat2;
-    try {
-      const statOpts = signal ? { signal } : {};
-      stat2 = await fs3.stat(file, statOpts);
-    } catch (e) {
-      if (isAbortError(e)) throw e;
-      store.deleteFile(file);
-      continue;
-    }
-    if (!stat2.isFile()) continue;
-    const lang = detectLang(file);
-    if (!lang) continue;
-    const meta = existingMeta.get(file);
-    if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
-      langStats[lang] = (langStats[lang] ?? 0) + meta.symbolCount;
-      symbolsIndexed += meta.symbolCount;
-      filesIndexed++;
-      continue;
-    }
-    store.deleteRefsForFile(file);
-    store.deleteSymbolsForFile(file);
-    let content;
-    try {
-      content = await fs3.readFile(file, { encoding: "utf8", signal });
-    } catch (e) {
-      if (isAbortError(e)) throw e;
-      errors.push(`read error: ${file}: ${e instanceof Error ? e.message : String(e)}`);
-      continue;
-    }
-    let parsed;
-    try {
-      parsed = await parseFile(file, content, lang);
-    } catch (e) {
-      errors.push(`parse error: ${file}: ${e instanceof Error ? e.message : String(e)}`);
-      continue;
-    }
-    if (parsed.symbols.length === 0) {
-      store.upsertFile({
-        file,
-        lang,
-        mtimeMs: Math.floor(stat2.mtimeMs),
-        symbolCount: 0,
-        lastIndexed: Date.now()
-      });
-      filesIndexed++;
-      continue;
-    }
-    const nextId = store.getMaxSymbolId() + 1;
-    const symbolsWithIds = parsed.symbols.map((s, i) => ({ ...s, id: nextId + i }));
-    store.insertSymbols(symbolsWithIds, nextId);
-    const count = symbolsWithIds.length;
-    symbolsIndexed += count;
-    langStats[lang] = (langStats[lang] ?? 0) + count;
-    if (parsed.refs && parsed.refs.length > 0) {
-      const refsByLine = /* @__PURE__ */ new Map();
-      for (const r of parsed.refs) {
-        let arr = refsByLine.get(r.line);
-        if (!arr) {
-          arr = [];
-          refsByLine.set(r.line, arr);
+    const statOpts = signal ? { signal } : {};
+    const statReadParse = await Promise.allSettled(
+      batchFiles.map(async (file) => {
+        let stat2;
+        try {
+          stat2 = await fs3.stat(file, statOpts);
+        } catch (e) {
+          if (isAbortError(e)) throw e;
+          return { file, stat: null, lang: "", parsed: null, error: `stat error: ${e instanceof Error ? e.message : String(e)}` };
+        }
+        if (!stat2.isFile()) return { file, stat: stat2, lang: "", parsed: null };
+        const lang = detectLang(file);
+        if (!lang) return { file, stat: stat2, lang: "", parsed: null };
+        let content;
+        try {
+          content = await fs3.readFile(file, { encoding: "utf8", signal });
+        } catch (e) {
+          if (isAbortError(e)) throw e;
+          return { file, stat: stat2, lang, parsed: null, error: `read error: ${e instanceof Error ? e.message : String(e)}` };
         }
-        arr.push(r);
+        let parsed;
+        try {
+          parsed = await parseFile(file, content, lang);
+        } catch (e) {
+          return { file, stat: stat2, lang, parsed: null, error: `parse error: ${e instanceof Error ? e.message : String(e)}` };
+        }
+        return { file, stat: stat2, lang, parsed, content };
+      })
+    );
+    for (let fi = 0; fi < statReadParse.length; fi++) {
+      const settled = statReadParse[fi];
+      const file = expectDefined(batchFiles[fi]);
+      if (settled.status === "rejected") {
+        const err = settled.reason;
+        if (err instanceof Error && isAbortError(err)) throw err;
+        errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
+        continue;
       }
-      const batch = [];
-      for (const sym of symbolsWithIds) {
-        const symRefs = refsByLine.get(sym.line);
-        if (symRefs) {
-          for (const r of symRefs) {
-            batch.push({ ...r, fromId: sym.id });
-          }
+      const result = settled.value;
+      if (result.error) {
+        if (result.stat) store.deleteFile(file);
+        if (result.error.includes("error:")) errors.push(result.error);
+        continue;
+      }
+      const { stat: stat2, lang, parsed } = result;
+      if (!lang || !parsed) {
+        if (lang) {
+          store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: 0, lastIndexed: Date.now() });
+          filesIndexed++;
         }
+        continue;
+      }
+      const meta = existingMeta.get(file);
+      if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
+        langStats[lang] = (langStats[lang] ?? 0) + meta.symbolCount;
+        symbolsIndexed += meta.symbolCount;
+        filesIndexed++;
+        continue;
+      }
+      store.deleteRefsForFile(file);
+      store.deleteSymbolsForFile(file);
+      if (parsed.symbols.length === 0) {
+        store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: 0, lastIndexed: Date.now() });
+        filesIndexed++;
+        continue;
       }
-      if (batch.length > 0) {
-        store.insertRefsBatch(batch);
+      const nextId = store.getMaxSymbolId() + 1;
+      const symbolsWithIds = parsed.symbols.map((s, i) => ({ ...s, id: nextId + i }));
+      store.insertSymbols(symbolsWithIds, nextId);
+      const count = symbolsWithIds.length;
+      symbolsIndexed += count;
+      langStats[lang] = (langStats[lang] ?? 0) + count;
+      if (parsed.refs && parsed.refs.length > 0) {
+        const refsByLine = /* @__PURE__ */ new Map();
+        for (const r of parsed.refs) {
+          let arr = refsByLine.get(r.line);
+          if (!arr) {
+            arr = [];
+            refsByLine.set(r.line, arr);
+          }
+          arr.push(r);
+        }
+        const batch = [];
+        for (const sym of symbolsWithIds) {
+          const symRefs = refsByLine.get(sym.line);
+          if (symRefs) {
+            for (const r of symRefs) batch.push({ ...r, fromId: sym.id });
+          }
+        }
+        if (batch.length > 0) store.insertRefsBatch(batch);
       }
+      store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: count, lastIndexed: Date.now() });
+      filesIndexed++;
     }
-    store.upsertFile({
-      file,
-      lang,
-      mtimeMs: Math.floor(stat2.mtimeMs),
-      symbolCount: count,
-      lastIndexed: Date.now()
-    });
-    filesIndexed++;
   }
   for (const [file_] of existingMeta) {
     try {