@wrongstack/tools 0.68.0 → 0.77.0

package/dist/codebase-index/index.d.ts

@@ -1,6 +1,29 @@
-import { d as Symbol, F as FileMeta, e as SymbolKind, f as SymbolLang, c as SearchResult, b as IndexStats, R as Ref } from '../codebase-stats-tool-C8ApERbn.js';
-export { a as FileSymbols, I as IndexResult, S as SCHEMA_VERSION, g as codebaseIndexTool, h as codebaseSearchTool, i as codebaseStatsTool } from '../codebase-stats-tool-C8ApERbn.js';
-import '@wrongstack/core';
+import { I as IndexResult, d as Symbol, F as FileMeta, e as SymbolKind, f as SymbolLang, c as SearchResult, b as IndexStats, R as Ref } from '../background-indexer-C70RD7LU.js';
+export { a as FileSymbols, S as SCHEMA_VERSION, g as cancelPendingReindexes, h as codebaseIndexTool, i as codebaseSearchTool, j as codebaseStatsTool, k as enqueueReindex, l as getIndexState, m as isIndexReady, n as isIndexableFile, o as isIndexing, p as onIndexStateChange, r as runStartupIndex } from '../background-indexer-C70RD7LU.js';
+import { Context } from '@wrongstack/core';
+
+/**
+ * Main indexing orchestrator.
+ *
+ * Given a project root and a list of files:
+ * 1. Parse each file with the appropriate parser (TS, Go, Python, Rust, JSON, YAML)
+ * 2. Delete old symbols for changed/deleted files
+ * 3. Insert new symbols
+ * 4. Update file metadata
+ * 5. Return index statistics
+ */
+
+interface IndexerOptions {
+    projectRoot: string;
+    files?: string[];
+    force?: boolean;
+    langs?: string[];
+    ignore?: string[];
+    /** Override the index directory (default: the global per-project dir). */
+    indexDir?: string;
+}
+/** Run a full or incremental index and return statistics. */
+declare function runIndexer(_ctx: Context, opts: IndexerOptions): Promise<IndexResult>;
 
 /**
  * SQLite storage layer for the codebase index.
@@ -50,6 +73,14 @@ declare class IndexStore {
         id: number;
         text: string;
     }>;
+    /**
+     * Largest symbol id currently in the table (0 when empty). New ids must be
+     * allocated from this, NOT from `COUNT(*)`: incremental reindexes delete a
+     * changed file's rows, so the row count drops below the max id and a
+     * count-based id would collide with a surviving row (UNIQUE constraint on
+     * `symbols.id`). Ids may have gaps — that is fine.
+     */
+    getMaxSymbolId(): number;
     getStats(): IndexStats;
     setLastIndexed(ts: number): void;
     clearAll(): void;
@@ -137,4 +168,4 @@ declare function lspKindToInternalKind(k: number): SymbolKind | null;
  */
 declare function internalKindToLspKind(k: SymbolKind): number | null;
 
-export { FileMeta, IndexStats, IndexStore, SearchResult, Symbol, SymbolKind, SymbolLang, buildBm25Index, buildIndexableText, codebaseIndexDirOverride, internalKindToLspKind, lspKindToInternalKind, resolveIndexDir, tokenise };
+export { FileMeta, IndexResult, IndexStats, IndexStore, SearchResult, Symbol, SymbolKind, SymbolLang, buildBm25Index, buildIndexableText, codebaseIndexDirOverride, internalKindToLspKind, lspKindToInternalKind, resolveIndexDir, runIndexer, tokenise };

package/dist/codebase-index/index.js

@@ -1,4 +1,4 @@
-import * as fs2 from 'node:fs/promises';
+import * as fs3 from 'node:fs/promises';
 import * as path4 from 'node:path';
 import { resolveWstackPaths, compileGlob } from '@wrongstack/core';
 import { createRequire } from 'node:module';
@@ -290,6 +290,17 @@ var IndexStore = class {
       ({ id, text }) => ({ id, text })
     );
   }
+  /**
+   * Largest symbol id currently in the table (0 when empty). New ids must be
+   * allocated from this, NOT from `COUNT(*)`: incremental reindexes delete a
+   * changed file's rows, so the row count drops below the max id and a
+   * count-based id would collide with a surviving row (UNIQUE constraint on
+   * `symbols.id`). Ids may have gaps — that is fine.
+   */
+  getMaxSymbolId() {
+    const rows = this.db.prepare("SELECT MAX(id) AS m FROM symbols").all();
+    return rows[0]?.m ?? 0;
+  }
   // ─── Stats ───────────────────────────────────────────────────────────────────
   getStats() {
     const sizeBytes = this.sizeBytes();
@@ -1608,8 +1619,181 @@ function makeSymbol2(opts) {
     text: `${opts.name} ${opts.signature}`.trim()
   };
 }
+function globBody(glob) {
+  return compileGlob(glob).source.replace(/^\^/, "").replace(/\$$/, "");
+}
+function compileGitignore(lines) {
+  const rules = [];
+  for (const raw of lines) {
+    let line = raw.replace(/\r$/, "");
+    if (!line.trim() || line.trimStart().startsWith("#")) continue;
+    line = line.trim();
+    let negated = false;
+    if (line.startsWith("!")) {
+      negated = true;
+      line = line.slice(1);
+    }
+    let dirOnly = false;
+    if (line.endsWith("/")) {
+      dirOnly = true;
+      line = line.slice(0, -1);
+    }
+    if (!line) continue;
+    const anchored = line.startsWith("/") || line.includes("/");
+    if (line.startsWith("/")) line = line.slice(1);
+    const body = globBody(line);
+    const prefix = anchored ? "^" : "(?:^|.*/)";
+    rules.push({
+      eqOrUnder: new RegExp(`${prefix}${body}(?:/.*)?$`),
+      under: new RegExp(`${prefix}${body}/.*$`),
+      negated,
+      dirOnly
+    });
+  }
+  return (relPath, isDir) => {
+    const p = relPath.replace(/\\/g, "/").replace(/^\/+/, "");
+    let ignored = false;
+    for (const r of rules) {
+      const re = r.dirOnly && !isDir ? r.under : r.eqOrUnder;
+      if (re.test(p)) ignored = !r.negated;
+    }
+    return ignored;
+  };
+}
+async function loadGitignoreMatcher(projectRoot) {
+  let lines = [];
+  try {
+    const raw = await fs3.readFile(path4.join(projectRoot, ".gitignore"), "utf8");
+    lines = raw.split("\n");
+  } catch {
+  }
+  return compileGitignore(lines);
+}
+
+// src/codebase-index/background-indexer.ts
+var _ready = false;
+var _indexing = false;
+var _currentFile = 0;
+var _totalFiles = 0;
+var _lastError = null;
+function isIndexReady() {
+  return _ready;
+}
+function setIndexReady() {
+  _ready = true;
+}
+function isIndexing() {
+  return _indexing;
+}
+function getIndexState() {
+  return {
+    ready: _ready,
+    indexing: _indexing,
+    currentFile: _currentFile,
+    totalFiles: _totalFiles,
+    lastError: _lastError
+  };
+}
+var _listeners = [];
+function onIndexStateChange(listener) {
+  _listeners.push(listener);
+  return () => {
+    _listeners = _listeners.filter((l) => l !== listener);
+  };
+}
+function emitState() {
+  const state = getIndexState();
+  for (const l of _listeners) l(state);
+}
+function _setIndexProgress(current, total) {
+  _currentFile = current;
+  _totalFiles = total;
+  emitState();
+}
+function stubCtx(projectRoot) {
+  return {
+    projectRoot,
+    cwd: projectRoot,
+    messages: [],
+    todos: [],
+    readFiles: /* @__PURE__ */ new Set(),
+    fileMtimes: /* @__PURE__ */ new Map()
+  };
+}
+var chain = Promise.resolve();
+function withMutex(job) {
+  const run = chain.then(job, job);
+  chain = run.then(
+    () => void 0,
+    () => void 0
+  );
+  return run;
+}
+var DEFAULT_DEBOUNCE_MS = 400;
+var debounceTimers = /* @__PURE__ */ new Map();
+function debounceKey(indexDir, file) {
+  return `${indexDir ?? ""}|${file}`;
+}
+function isIndexableFile(filePath) {
+  return detectLang(filePath) !== null;
+}
+async function runStartupIndex(opts) {
+  _indexing = true;
+  _currentFile = 0;
+  _totalFiles = 0;
+  _lastError = null;
+  emitState();
+  try {
+    const result = await withMutex(
+      () => runIndexer(stubCtx(opts.projectRoot), {
+        projectRoot: opts.projectRoot,
+        indexDir: opts.indexDir,
+        force: opts.force
+      })
+    );
+    _ready = true;
+    return result;
+  } catch (err) {
+    _lastError = err instanceof Error ? err.message : String(err);
+    _ready = true;
+    throw err;
+  } finally {
+    _indexing = false;
+    emitState();
+  }
+}
+function enqueueReindex(opts) {
+  const files = opts.files.filter(isIndexableFile);
+  if (files.length === 0) return;
+  const ms = opts.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+  for (const file of files) {
+    const key = debounceKey(opts.indexDir, file);
+    const existing = debounceTimers.get(key);
+    if (existing) clearTimeout(existing);
+    const timer = setTimeout(() => {
+      debounceTimers.delete(key);
+      void withMutex(
+        () => runIndexer(stubCtx(opts.projectRoot), {
+          projectRoot: opts.projectRoot,
+          files: [file],
+          indexDir: opts.indexDir
+        })
+      ).catch((err) => opts.onError?.(err));
+    }, ms);
+    timer.unref?.();
+    debounceTimers.set(key, timer);
+  }
+}
+function cancelPendingReindexes() {
+  for (const t of debounceTimers.values()) clearTimeout(t);
+  debounceTimers.clear();
+}
 
 // src/codebase-index/indexer.ts
+var YIELD_EVERY_N = 50;
+function yieldEventLoop() {
+  return new Promise((resolve2) => setImmediate(resolve2));
+}
 var DEFAULT_IGNORE = [
   "node_modules",
   ".git",
@@ -1621,7 +1805,7 @@ var DEFAULT_IGNORE = [
   "__snapshots__",
   ".nyc_output"
 ];
-async function findSourceFiles(projectRoot, ignore) {
+async function findSourceFiles(projectRoot, ignore, isGitIgnored) {
   const results = [];
   const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
   const globs = [
@@ -1639,17 +1823,19 @@ async function findSourceFiles(projectRoot, ignore) {
   const walk = async (dir) => {
     let entries;
     try {
-      entries = await fs2.readdir(dir, { withFileTypes: true });
+      entries = await fs3.readdir(dir, { withFileTypes: true });
     } catch {
       return;
     }
     for (const e of entries) {
       if (ignoreSet.has(e.name)) continue;
       const full = path4.join(dir, e.name);
+      const rel = path4.relative(projectRoot, full).replace(/\\/g, "/");
       if (e.isDirectory()) {
+        if (isGitIgnored(rel, true)) continue;
         await walk(full);
       } else if (e.isFile()) {
-        const rel = path4.relative(projectRoot, full).replace(/\\/g, "/");
+        if (isGitIgnored(rel, false)) continue;
         const ext = path4.extname(e.name);
         for (const { ext: extName, pat } of globs) {
           if (ext === extName && (pat.test(rel) || pat.test(e.name))) {
@@ -1692,11 +1878,12 @@ async function runIndexer(_ctx, opts) {
   const langStats = {};
   let filesIndexed = 0;
   let symbolsIndexed = 0;
+  const isGitIgnored = await loadGitignoreMatcher(projectRoot);
   let files;
   if (opts.files && opts.files.length > 0) {
-    files = opts.files.map((f) => path4.resolve(projectRoot, f));
+    files = opts.files.map((f) => path4.resolve(projectRoot, f)).filter((f) => !isGitIgnored(path4.relative(projectRoot, f).replace(/\\/g, "/"), false));
   } else {
-    files = await findSourceFiles(projectRoot, ignore);
+    files = await findSourceFiles(projectRoot, ignore, isGitIgnored);
   }
   if (langs && langs.length > 0) {
     const langSet = new Set(langs);
@@ -1710,10 +1897,15 @@ async function runIndexer(_ctx, opts) {
   if (!force) {
     for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
   }
-  for (const file of files) {
+  for (let fi = 0; fi < files.length; fi++) {
+    const file = files[fi];
+    _setIndexProgress(fi + 1, files.length);
+    if (fi > 0 && fi % YIELD_EVERY_N === 0) {
+      await yieldEventLoop();
+    }
     let stat2;
     try {
-      stat2 = await fs2.stat(file);
+      stat2 = await fs3.stat(file);
     } catch {
       store.deleteFile(file);
       continue;
@@ -1728,11 +1920,11 @@ async function runIndexer(_ctx, opts) {
       filesIndexed++;
       continue;
     }
-    store.deleteSymbolsForFile(file);
     store.deleteRefsForFile(file);
+    store.deleteSymbolsForFile(file);
     let content;
     try {
-      content = await fs2.readFile(file, "utf8");
+      content = await fs3.readFile(file, "utf8");
     } catch (e) {
       errors.push(`read error: ${file}: ${e instanceof Error ? e.message : String(e)}`);
       continue;
@@ -1755,7 +1947,7 @@ async function runIndexer(_ctx, opts) {
       filesIndexed++;
       continue;
     }
-    const nextId = store.getStats().totalSymbols + 1;
+    const nextId = store.getMaxSymbolId() + 1;
     const symbolsWithIds = parsed.symbols.map((s, i) => ({ ...s, id: nextId + i }));
     store.insertSymbols(symbolsWithIds, nextId);
     const count = symbolsWithIds.length;
@@ -1782,7 +1974,7 @@ async function runIndexer(_ctx, opts) {
   }
   for (const [file_] of existingMeta) {
     try {
-      await fs2.stat(file_);
+      await fs3.stat(file_);
     } catch {
       store.deleteFile(file_);
     }
@@ -1824,12 +2016,23 @@ var codebaseIndexTool = {
     }
   },
   async execute(input, ctx) {
+    if (isIndexing()) {
+      return {
+        filesIndexed: 0,
+        symbolsIndexed: 0,
+        langStats: {},
+        durationMs: 0,
+        errors: [],
+        note: "A full index is already in progress. Retry codebase-index after it completes (check codebase-stats)."
+      };
+    }
     const result = await runIndexer(ctx, {
       projectRoot: ctx.projectRoot,
       force: input.force ?? false,
       langs: input.langs,
       indexDir: codebaseIndexDirOverride(ctx)
     });
+    setIndexReady();
     return result;
   }
 };
@@ -1965,6 +2168,31 @@ var codebaseSearchTool = {
     required: ["query"]
   },
   async execute(input, ctx) {
+    const state = getIndexState();
+    if (!state.ready) {
+      return {
+        results: [],
+        total: 0,
+        query: input.query,
+        indexStatus: state.indexing ? `Indexing in progress (${state.currentFile}/${state.totalFiles} files) \u2014 retry in a moment.` : "Index not yet built. The codebase is being indexed at startup \u2014 search will be available shortly."
+      };
+    }
+    if (state.indexing) {
+      return {
+        results: [],
+        total: 0,
+        query: input.query,
+        indexStatus: `Index refresh in progress (${state.currentFile}/${state.totalFiles} files). Results may be incomplete.`
+      };
+    }
+    if (state.lastError) {
+      return {
+        results: [],
+        total: 0,
+        query: input.query,
+        indexStatus: `Index build failed: ${state.lastError}. Try /codebase-reindex.`
+      };
+    }
     const store = new IndexStore(ctx.projectRoot, { indexDir: codebaseIndexDirOverride(ctx) });
     try {
       const limit = Math.min(input.limit ?? 20, 100);
@@ -2022,6 +2250,32 @@ var codebaseStatsTool = {
     additionalProperties: false
   },
   async execute(_input, ctx) {
+    const idxState = getIndexState();
+    if (!idxState.ready) {
+      return {
+        totalSymbols: 0,
+        totalFiles: 0,
+        byLang: {},
+        byKind: {},
+        lastIndexed: null,
+        sizeBytes: 0,
+        indexPath: "",
+        version: SCHEMA_VERSION,
+        indexStatus: idxState.indexing ? `Indexing in progress (${idxState.currentFile}/${idxState.totalFiles} files).` : "Index not yet built."
+      };
+    }
+    if (idxState.indexing) {
+      const store2 = new IndexStore(ctx.projectRoot, { indexDir: codebaseIndexDirOverride(ctx) });
+      try {
+        const stats = store2.getStats();
+        return {
+          ...stats,
+          indexStatus: `Index refresh in progress (${idxState.currentFile}/${idxState.totalFiles} files). Stats may be incomplete.`
+        };
+      } finally {
+        store2.close();
+      }
+    }
     const store = new IndexStore(ctx.projectRoot, { indexDir: codebaseIndexDirOverride(ctx) });
     try {
       const stats = store.getStats();
@@ -2041,6 +2295,6 @@ var codebaseStatsTool = {
   }
 };
 
-export { IndexStore, SCHEMA_VERSION, buildBm25Index, buildIndexableText, codebaseIndexDirOverride, codebaseIndexTool, codebaseSearchTool, codebaseStatsTool, internalKindToLspKind, lspKindToInternalKind, resolveIndexDir, tokenise };
+export { IndexStore, SCHEMA_VERSION, buildBm25Index, buildIndexableText, cancelPendingReindexes, codebaseIndexDirOverride, codebaseIndexTool, codebaseSearchTool, codebaseStatsTool, enqueueReindex, getIndexState, internalKindToLspKind, isIndexReady, isIndexableFile, isIndexing, lspKindToInternalKind, onIndexStateChange, resolveIndexDir, runIndexer, runStartupIndex, tokenise };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map