@wrongstack/tools 0.275.1 → 0.276.3

package/dist/codebase-index/worker.js

@@ -727,6 +727,104 @@ var IndexStore = class {
       }
     });
   }
+  /**
+   * Commit a batch of file-level symbol/refs/upserts in a single transaction.
+   *
+   * Used by the indexer to amortize SQLite commit overhead across many files.
+   * Before this, the indexer issued one transaction per file (BEGIN IMMEDIATE
+   * for symbols, plus per-file deletes and an upsertFile call), so a 20-file
+   * parallel batch cost ~5+ transactions × 20 files = 100+ commits. With
+   * this entry point we do exactly one BEGIN/COMMIT per parallel batch.
+   *
+   * Each entry must already be a fully-parsed FileSymbols (symbols + refs).
+   * The caller is responsible for the per-file prefix accounting
+   * (refsByLine → flat list with `fromId` populated). `deleteForFiles` lets
+   * the caller clear stale symbols/refs for any files being re-indexed before
+   * the inserts run (required to keep refs → symbols FK invariants).
+   *
+   * Returns the symbols back with their assigned `id` (same shape as
+   * {@link insertSymbols}) so callers can build final per-file results.
+   */
+  commitBatch(entries, options = {}) {
+    if (entries.length === 0 && (options.deleteForFiles?.length ?? 0) === 0) {
+      return [];
+    }
+    return this.runWithRetry(() => {
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        if (options.deleteForFiles && options.deleteForFiles.length > 0) {
+          const placeholders = options.deleteForFiles.map(() => "?").join(",");
+          if (this.ftsAvailable) {
+            this.db.prepare(
+              `DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
+            ).run(...options.deleteForFiles);
+          }
+          this.db.prepare(
+            `DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
+          ).run(...options.deleteForFiles);
+          this.db.prepare(`DELETE FROM symbols WHERE file IN (${placeholders})`).run(...options.deleteForFiles);
+        }
+        const maxRows = this.db.prepare("SELECT MAX(id) AS m FROM symbols").all();
+        let nextId = (maxRows[0]?.m ?? 0) + 1;
+        const symStmt = this.db.prepare(
+          `INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text, file_fk)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+        );
+        const ftsStmt = this.ftsAvailable ? this.db.prepare("INSERT INTO symbols_fts(rowid, text) VALUES (?, ?)") : null;
+        const allInserted = [];
+        const refsToInsert = [];
+        for (const entry of entries) {
+          for (const s of entry.symbols) {
+            const id = nextId++;
+            symStmt.run(
+              id,
+              s.lang,
+              s.kind,
+              s.name,
+              s.file,
+              s.line,
+              s.col,
+              s.signature,
+              s.docComment,
+              s.scope,
+              s.text,
+              s.file
+            );
+            ftsStmt?.run(id, buildIndexableText(s.name, s.signature, s.docComment));
+            allInserted.push({ ...s, id });
+          }
+          for (const r of entry.refs) refsToInsert.push(r);
+        }
+        if (refsToInsert.length > 0) {
+          const refStmt = this.db.prepare(
+            `INSERT INTO refs(from_id, to_name, to_id, call_type, line)
+             VALUES (?, ?, ?, ?, ?)`
+          );
+          for (const ref of refsToInsert) {
+            refStmt.run(ref.fromId, ref.toName, ref.toId ?? null, ref.callType, ref.line);
+          }
+        }
+        const upsertStmt = this.db.prepare(
+          `INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
+           VALUES (?, ?, ?, ?, ?)
+           ON CONFLICT(file) DO UPDATE SET
+             lang = excluded.lang,
+             mtime_ms = excluded.mtime_ms,
+             symbol_count = excluded.symbol_count,
+             last_indexed = excluded.last_indexed`
+        );
+        const now = Date.now();
+        for (const entry of entries) {
+          upsertStmt.run(entry.file, entry.lang, entry.mtimeMs, entry.symbolCount, now);
+        }
+        this.db.exec("COMMIT");
+        return allInserted;
+      } catch (err) {
+        this.db.exec("ROLLBACK");
+        throw err;
+      }
+    });
+  }
   /**
    * Delete all refs whose source symbols are in a given file.
    * Used when re-indexing a file to clear stale refs.
@@ -2258,6 +2356,8 @@ async function runIndexerWithStore(store, opts) {
         return { file, stat: stat2, lang, parsed, content };
       })
     );
+    const batchEntries = [];
+    const deleteForFiles = [];
     for (let fi = 0; fi < statReadParse.length; fi++) {
       const settled = statReadParse[fi];
       const file = expectDefined(batchFiles[fi]);
@@ -2282,43 +2382,116 @@ async function runIndexerWithStore(store, opts) {
       }
       if (!lang || !parsed) {
         if (lang) {
-          store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: 0, lastIndexed: Date.now() });
+          store.upsertFile({
+            file,
+            lang,
+            mtimeMs: Math.floor(stat2.mtimeMs),
+            symbolCount: 0,
+            lastIndexed: Date.now()
+          });
           filesIndexed++;
         }
         continue;
       }
-      store.deleteRefsForFile(file);
-      store.deleteSymbolsForFile(file);
       if (parsed.symbols.length === 0) {
-        store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: 0, lastIndexed: Date.now() });
+        store.upsertFile({
+          file,
+          lang,
+          mtimeMs: Math.floor(stat2.mtimeMs),
+          symbolCount: 0,
+          lastIndexed: Date.now()
+        });
         filesIndexed++;
         continue;
       }
-      const symbolsWithIds = store.insertSymbols(parsed.symbols);
-      const count = symbolsWithIds.length;
-      symbolsIndexed += count;
-      langStats[lang] = (langStats[lang] ?? 0) + count;
+      const refs = [];
       if (parsed.refs && parsed.refs.length > 0) {
-        const refsByLine = /* @__PURE__ */ new Map();
-        for (const r of parsed.refs) {
-          let arr = refsByLine.get(r.line);
-          if (!arr) {
-            arr = [];
-            refsByLine.set(r.line, arr);
+        for (const r of parsed.refs) refs.push({ ...r, fromId: 0 });
+      }
+      batchEntries.push({
+        file,
+        lang,
+        symbols: parsed.symbols,
+        refs,
+        mtimeMs: Math.floor(stat2.mtimeMs),
+        symbolCount: parsed.symbols.length
+      });
+      deleteForFiles.push(file);
+    }
+    if (batchEntries.length > 0) {
+      try {
+        const inserted = store.commitBatch(batchEntries, { deleteForFiles });
+        let cursor = 0;
+        for (const entry of batchEntries) {
+          const count = entry.symbols.length;
+          const symbolsWithIds = inserted.slice(cursor, cursor + count);
+          cursor += count;
+          symbolsIndexed += count;
+          langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
+          filesIndexed++;
+          if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
+            const refsByLine = /* @__PURE__ */ new Map();
+            for (let i = 0; i < symbolsWithIds.length; i++) {
+              const sym = symbolsWithIds[i];
+              let arr = refsByLine.get(sym.line);
+              if (!arr) {
+                arr = [];
+                refsByLine.set(sym.line, arr);
+              }
+              arr.push(i);
+            }
+            for (const ref of entry.refs) {
+              const indices = refsByLine.get(ref.line);
+              if (indices && indices.length > 0) {
+                const idx = indices.shift();
+                ref.fromId = symbolsWithIds[idx].id;
+              }
+            }
           }
-          arr.push(r);
         }
-        const batch = [];
-        for (const sym of symbolsWithIds) {
-          const symRefs = refsByLine.get(sym.line);
-          if (symRefs) {
-            for (const r of symRefs) batch.push({ ...r, fromId: sym.id });
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        errors.push(`commitBatch failed: ${message} \u2014 falling back to per-file writes`);
+        for (const entry of batchEntries) {
+          try {
+            store.deleteRefsForFile(entry.file);
+            store.deleteSymbolsForFile(entry.file);
+            const symbolsWithIds = store.insertSymbols(entry.symbols);
+            symbolsIndexed += symbolsWithIds.length;
+            langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
+            filesIndexed++;
+            if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
+              const refsByLine = /* @__PURE__ */ new Map();
+              for (const sym of symbolsWithIds) {
+                let arr = refsByLine.get(sym.line);
+                if (!arr) {
+                  arr = [];
+                  refsByLine.set(sym.line, arr);
+                }
+                arr.push(sym);
+              }
+              const fallbackBatch = [];
+              for (const ref of entry.refs) {
+                const syms = refsByLine.get(ref.line);
+                if (syms && syms.length > 0) {
+                  const sym = syms.shift();
+                  fallbackBatch.push({ ...ref, fromId: sym.id });
+                }
+              }
+              if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
+            }
+            store.upsertFile({
+              file: entry.file,
+              lang: entry.lang,
+              mtimeMs: entry.mtimeMs,
+              symbolCount: entry.symbolCount,
+              lastIndexed: Date.now()
+            });
+          } catch (innerErr) {
+            errors.push(`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`);
           }
         }
-        if (batch.length > 0) store.insertRefsBatch(batch);
       }
-      store.upsertFile({ file, lang, mtimeMs: Math.floor(stat2.mtimeMs), symbolCount: count, lastIndexed: Date.now() });
-      filesIndexed++;
     }
   }
   if (discoveredFiles) {