@wrongstack/tools 0.269.0 → 0.272.0

package/dist/codebase-index/worker.js

@@ -1,13 +1,12 @@
 import { parentPort } from 'node:worker_threads';
 import { expectDefined, resolveWstackPaths, compileGlob, truncate } from '@wrongstack/core';
-import * as fs3 from 'node:fs/promises';
+import * as fs6 from 'node:fs/promises';
 import * as path4 from 'node:path';
 import { toErrorMessage } from '@wrongstack/core/utils';
 import { createRequire } from 'node:module';
 import * as fs from 'node:fs';
-import { writeFileSync, mkdirSync } from 'node:fs';
 import * as ts from 'typescript';
-import { execFileSync, spawnSync } from 'node:child_process';
+import { execFileSync, spawn } from 'node:child_process';
 import * as os from 'node:os';
 
 // src/codebase-index/worker.ts
@@ -385,33 +384,53 @@ var IndexStore = class {
     }
   }
   // ─── Symbol CRUD ─────────────────────────────────────────────────────────────
-  insertSymbols(symbols, nextId) {
+  /**
+   * Insert symbols, assigning IDs atomically inside `BEGIN IMMEDIATE` /
+   * `COMMIT`. The ID allocation (`SELECT MAX(id)`) and all `INSERT`s share
+   * the same transaction, preventing UNIQUE constraint violations when two
+   * processes index concurrently (each would see a different `MAX(id)` and
+   * neither can insert with the other's IDs).
+   *
+   * @returns The symbols array with `id` fields populated so the caller can
+   *          use them for refs without re-reading from the DB.
+   */
+  insertSymbols(symbols) {
     return this.runWithRetry(() => {
-      const stmt = this.db.prepare(
-        `INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text, file_fk)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
-      );
-      const ftsStmt = this.ftsAvailable ? this.db.prepare("INSERT INTO symbols_fts(rowid, text) VALUES (?, ?)") : null;
-      let id = nextId;
-      for (const s of symbols) {
-        stmt.run(
-          id,
-          s.lang,
-          s.kind,
-          s.name,
-          s.file,
-          s.line,
-          s.col,
-          s.signature,
-          s.docComment,
-          s.scope,
-          s.text,
-          s.file
+      this.db.exec("BEGIN IMMEDIATE");
+      try {
+        const maxRows = this.db.prepare("SELECT MAX(id) AS m FROM symbols").all();
+        let nextId = (maxRows[0]?.m ?? 0) + 1;
+        const stmt = this.db.prepare(
+          `INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text, file_fk)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
         );
-        ftsStmt?.run(id, buildIndexableText(s.name, s.signature, s.docComment));
-        id++;
+        const ftsStmt = this.ftsAvailable ? this.db.prepare("INSERT INTO symbols_fts(rowid, text) VALUES (?, ?)") : null;
+        const result = [];
+        for (const s of symbols) {
+          const id = nextId++;
+          stmt.run(
+            id,
+            s.lang,
+            s.kind,
+            s.name,
+            s.file,
+            s.line,
+            s.col,
+            s.signature,
+            s.docComment,
+            s.scope,
+            s.text,
+            s.file
+          );
+          ftsStmt?.run(id, buildIndexableText(s.name, s.signature, s.docComment));
+          result.push({ ...s, id });
+        }
+        this.db.exec("COMMIT");
+        return result;
+      } catch (err) {
+        this.db.exec("ROLLBACK");
+        throw err;
       }
-      return id;
     });
   }
   deleteSymbolsForFile(file) {
@@ -965,10 +984,10 @@ function detectLang(file) {
   if (idx < 0) return null;
   return extToLang(file.slice(idx));
 }
-function parseSymbols2(opts) {
+async function parseSymbols2(opts) {
   const { file, content, lang } = opts;
   try {
-    return syncGoParse(file, content, lang);
+    return await syncGoParse(file, content, lang);
   } catch {
     return { file, lang, symbols: [], mtimeMs: Date.now() };
   }
@@ -1205,19 +1224,34 @@ func formatType(t ast.Expr) string {
 	}
 }
 `;
-function syncGoParse(filePath, content, lang) {
+async function syncGoParse(filePath, content, lang) {
   const tmpDir = path4.join(os.tmpdir(), "ws-go-parse");
   try {
-    mkdirSync(tmpDir, { recursive: true });
+    await fs6.mkdir(tmpDir, { recursive: true });
     const scriptPath = path4.join(tmpDir, "parse.go");
-    writeFileSync(scriptPath, GO_PARSE_SCRIPT, "utf8");
-    const stdout = execFileSync("go", ["run", scriptPath], {
-      input: content,
-      timeout: 15e3,
-      encoding: "utf8",
+    await fs6.writeFile(scriptPath, GO_PARSE_SCRIPT, "utf8");
+    const proc = spawn("go", ["run", scriptPath], {
+      stdio: ["pipe", "pipe", "pipe"],
       windowsHide: true
     });
-    if (!stdout.trim()) {
+    let stdout = "";
+    proc.stdout?.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    proc.stdin?.write(content);
+    proc.stdin?.end();
+    const { code } = await Promise.race([
+      new Promise((resolve2) => {
+        proc.on("close", (c) => resolve2({ code: c }));
+      }),
+      new Promise(
+        (_, reject) => setTimeout(() => {
+          proc.kill("SIGKILL");
+          reject(new Error("timeout"));
+        }, 15e3)
+      )
+    ]).catch(() => ({ code: -1 }));
+    if (code !== 0 || !stdout.trim()) {
       return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
     }
     const raw = JSON.parse(stdout.trim());
@@ -1239,10 +1273,10 @@ function syncGoParse(filePath, content, lang) {
     return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
   }
 }
-function parseSymbols3(opts) {
+async function parseSymbols3(opts) {
   const { file, lang } = opts;
   try {
-    return syncPyParse(file, lang);
+    return await syncPyParse(file, lang);
   } catch {
     return { file, lang, symbols: [], mtimeMs: Date.now() };
   }
@@ -1451,18 +1485,32 @@ visitor.visit(tree)
 
 print(json.dumps([s.to_dict() for s in syms]))
 `;
-function syncPyParse(filePath, lang) {
+async function syncPyParse(filePath, lang) {
   try {
     const tmpDir = path4.join(os.tmpdir(), "ws-py-parse");
-    mkdirSync(tmpDir, { recursive: true });
+    await fs6.mkdir(tmpDir, { recursive: true });
     const scriptPath = path4.join(tmpDir, "parse.py");
-    writeFileSync(scriptPath, PY_PARSE_SCRIPT, "utf8");
-    const stdout = execFileSync("python", [scriptPath, filePath], {
-      timeout: 15e3,
-      encoding: "utf8",
+    await fs6.writeFile(scriptPath, PY_PARSE_SCRIPT, "utf8");
+    const proc = spawn("python", [scriptPath, filePath], {
+      stdio: ["pipe", "pipe", "pipe"],
       windowsHide: true
     });
-    if (!stdout.trim()) {
+    let stdout = "";
+    proc.stdout?.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    const { code } = await Promise.race([
+      new Promise((resolve2) => {
+        proc.on("close", (c) => resolve2({ code: c }));
+      }),
+      new Promise(
+        (_, reject) => setTimeout(() => {
+          proc.kill("SIGKILL");
+          reject(new Error("timeout"));
+        }, 15e3)
+      )
+    ]).catch(() => ({ code: -1 }));
+    if (code !== 0 || !stdout.trim()) {
       return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
     }
     const raw = JSON.parse(stdout.trim());
@@ -1484,11 +1532,11 @@ function syncPyParse(filePath, lang) {
     return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
   }
 }
-function parseSymbols4(opts) {
+async function parseSymbols4(opts) {
   const { file, content, lang } = opts;
   const nativeAvailable = checkNativeParser();
   if (nativeAvailable) {
-    const result = tryNativeParse(file, content);
+    const result = await tryNativeParse(file, content);
     if (result) return result;
   }
   return regexParse({ file, content, lang });
@@ -1518,25 +1566,34 @@ function checkNativeParser() {
     return false;
   }
 }
-function tryNativeParse(file, content) {
+async function tryNativeParse(file, content) {
   try {
     const toolsDir = path4.join(process.cwd(), "tools");
     const crateDir = path4.join(toolsDir, "syn-parser");
     const tmpFile = path4.join(crateDir, "src", "input.rs");
-    writeFileSync(tmpFile, content, "utf8");
-    const result = spawnSync(
-      "cargo",
-      ["run", "--manifest-path", path4.join(toolsDir, "Cargo.toml")],
-      {
-        cwd: process.cwd(),
-        encoding: "utf8",
-        timeout: 15e3,
-        stdio: ["pipe", "pipe", "pipe"],
-        windowsHide: true
-      }
-    );
-    if (result.status === 0 && result.stdout) {
-      const symbols = JSON.parse(result.stdout);
+    await fs6.writeFile(tmpFile, content, "utf8");
+    const proc = spawn("cargo", ["run", "--manifest-path", path4.join(toolsDir, "Cargo.toml")], {
+      cwd: process.cwd(),
+      stdio: ["pipe", "pipe", "pipe"],
+      windowsHide: true
+    });
+    let stdout = "";
+    proc.stdout?.on("data", (chunk) => {
+      stdout += chunk.toString();
+    });
+    const { code } = await Promise.race([
+      new Promise((resolve2) => {
+        proc.on("close", (c) => resolve2({ code: c }));
+      }),
+      new Promise(
+        (_, reject) => setTimeout(() => {
+          proc.kill("SIGKILL");
+          reject(new Error("timeout"));
+        }, 15e3)
+      )
+    ]).catch(() => ({ code: -1 }));
+    if (code === 0 && stdout.trim()) {
+      const symbols = JSON.parse(stdout.trim());
       return {
         file,
         lang: "rs",
@@ -2005,7 +2062,7 @@ function compileGitignore(lines) {
 async function loadGitignoreMatcher(projectRoot) {
   let lines = [];
   try {
-    const raw = await fs3.readFile(path4.join(projectRoot, ".gitignore"), "utf8");
+    const raw = await fs6.readFile(path4.join(projectRoot, ".gitignore"), "utf8");
     lines = raw.split("\n");
   } catch {
   }
@@ -2063,7 +2120,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
     }
     let entries;
     try {
-      entries = await fs3.readdir(dir, { withFileTypes: true });
+      entries = await fs6.readdir(dir, { withFileTypes: true });
     } catch {
       return;
     }
@@ -2161,7 +2218,7 @@ async function runIndexerWithStore(store, opts) {
       batchFiles.map(async (file) => {
         let stat2;
         try {
-          stat2 = await fs3.stat(file, statOpts);
+          stat2 = await fs6.stat(file, statOpts);
         } catch (e) {
           if (isAbortError(e)) throw e;
           return { file, stat: null, lang: "", parsed: null, error: `stat error: ${e instanceof Error ? e.message : String(e)}` };
@@ -2175,7 +2232,7 @@ async function runIndexerWithStore(store, opts) {
         }
         let content;
         try {
-          content = await fs3.readFile(file, { encoding: "utf8", signal });
+          content = await fs6.readFile(file, { encoding: "utf8", signal });
         } catch (e) {
           if (isAbortError(e)) throw e;
           return { file, stat: stat2, lang, parsed: null, error: `read error: ${e instanceof Error ? e.message : String(e)}` };
@@ -2225,9 +2282,7 @@ async function runIndexerWithStore(store, opts) {
         filesIndexed++;
         continue;
       }
-      const nextId = store.getMaxSymbolId() + 1;
-      const symbolsWithIds = parsed.symbols.map((s, i) => ({ ...s, id: nextId + i }));
-      store.insertSymbols(symbolsWithIds, nextId);
+      const symbolsWithIds = store.insertSymbols(parsed.symbols);
       const count = symbolsWithIds.length;
       symbolsIndexed += count;
       langStats[lang] = (langStats[lang] ?? 0) + count;
@@ -2256,7 +2311,7 @@ async function runIndexerWithStore(store, opts) {
   }
   for (const [file_] of existingMeta) {
     try {
-      await fs3.stat(file_);
+      await fs6.stat(file_);
     } catch {
       store.deleteFile(file_);
     }