@wrongstack/tools 0.273.1 → 0.274.0

package/dist/index.js

@@ -778,7 +778,27 @@ async function globNative(pattern, base, extraGlob) {
   await walk(base);
   return results;
 }
+
+// src/_concurrency.ts
+async function mapWithConcurrency(items, limit, fn) {
+  if (items.length === 0) return [];
+  const effectiveLimit = Math.max(1, Math.min(limit | 0, items.length));
+  const results = new Array(items.length);
+  let nextIndex = 0;
+  const worker2 = async () => {
+    while (true) {
+      const i = nextIndex++;
+      if (i >= items.length) return;
+      results[i] = await fn(items[i]);
+    }
+  };
+  await Promise.all(Array.from({ length: effectiveLimit }, worker2));
+  return results;
+}
+
+// src/glob.ts
 var DEFAULT_IGNORE2 = ["node_modules", ".git", "dist", "build", ".next", "coverage", ".turbo"];
+var WALK_CONCURRENCY = 16;
 var globTool = {
   name: "glob",
   category: "Filesystem",
@@ -816,6 +836,22 @@ var globTool = {
     const re = compileGlob(input.pattern);
     const results = [];
     let truncated = false;
+    const pushResult = async (full) => {
+      if (truncated || results.length >= limit) {
+        truncated = true;
+        return;
+      }
+      try {
+        const st = await fs7.stat(full);
+        if (truncated || results.length >= limit) {
+          truncated = true;
+          return;
+        }
+        results.push({ rel: full, mtime: st.mtimeMs });
+        if (results.length >= limit) truncated = true;
+      } catch {
+      }
+    };
     const walk = async (dir, relPrefix) => {
       if (results.length >= limit) {
         truncated = true;
@@ -827,6 +863,8 @@ var globTool = {
       } catch {
         return;
       }
+      const subdirs = [];
+      const matchedFiles = [];
       for (const e of entries) {
         const name = e.name;
         if (DEFAULT_IGNORE2.includes(name)) continue;
@@ -834,22 +872,36 @@ var globTool = {
         const rel = relPrefix ? `${relPrefix}/${name}` : name;
         const full = path.join(dir, name);
         if (e.isDirectory()) {
-          await walk(full, rel);
-          if (truncated) return;
+          subdirs.push({ full, rel });
         } else if (e.isFile()) {
-          if (re.test(rel) || re.test(name)) {
-            try {
-              const st = await fs7.stat(full);
-              results.push({ rel: full, mtime: st.mtimeMs });
-              if (results.length >= limit) {
-                truncated = true;
-                return;
-              }
-            } catch {
+          re.lastIndex = 0;
+          const relMatch = re.test(rel);
+          re.lastIndex = 0;
+          const nameMatch = re.test(name);
+          if (relMatch || nameMatch) {
+            matchedFiles.push(full);
+          }
+        } else if (e.isSymbolicLink()) {
+          try {
+            const st = await fs7.stat(full);
+            if (st.isDirectory()) {
+              subdirs.push({ full, rel });
+            } else if (st.isFile()) {
+              re.lastIndex = 0;
+              const relMatch = re.test(rel);
+              re.lastIndex = 0;
+              const nameMatch = re.test(name);
+              if (relMatch || nameMatch) matchedFiles.push(full);
             }
+          } catch {
           }
         }
+        if (truncated) return;
       }
+      await mapWithConcurrency(matchedFiles, WALK_CONCURRENCY, pushResult);
+      if (truncated) return;
+      const remainingSubdirs = truncated ? [] : subdirs;
+      await mapWithConcurrency(remainingSubdirs, WALK_CONCURRENCY, ({ full, rel }) => walk(full, rel));
     };
     await walk(base, "");
     results.sort((a, b) => b.mtime - a.mtime);
@@ -866,6 +918,8 @@ async function readGitignore(dir) {
 }
 var DEFAULT_IGNORE3 = ["node_modules", ".git", "dist", "build", ".next", "coverage"];
 var NATIVE_SCAN_CONCURRENCY = 32;
+var NATIVE_READ_CHUNK_BYTES = 64 * 1024;
+var NATIVE_MAX_FILE_BYTES = 1e6;
 var grepTool = {
   name: "grep",
   category: "Search",
@@ -1090,7 +1144,8 @@ async function runNative(input, base, mode, limit, signal) {
   const re = compiled.regex;
   const globRe = input.glob ? compileGlob(input.glob) : null;
   const matches = [];
-  const fileMatches = /* @__PURE__ */ new Map();
+  const countOnlyFirstHit = mode === "count" && limit === 1;
+  const maxBytes = mode === "content" ? NATIVE_MAX_FILE_BYTES : Math.min(NATIVE_MAX_FILE_BYTES, 256 * 1024);
   let total = 0;
   let stopped = false;
   const scanFile = async (full, name) => {
@@ -1099,34 +1154,79 @@ async function runNative(input, base, mode, limit, signal) {
     if (globRe) globRe.lastIndex = 0;
     try {
       const stat11 = await fs7.stat(full);
-      if (stat11.size > 1e6 || stopped || signal.aborted) return;
-      const head = await fs7.readFile(full);
-      if (isBinaryBuffer(head) || stopped || signal.aborted) return;
-      const text = head.toString("utf8");
-      const lines = text.split(/\r?\n/);
-      let fileHits = 0;
-      for (let i = 0; i < lines.length; i++) {
-        if (stopped || signal.aborted) break;
-        const ln = capSubject(lines[i] ?? "");
-        re.lastIndex = 0;
-        if (re.test(ln)) {
-          fileHits++;
-          total++;
-          if (mode === "content" && matches.length < limit) {
-            matches.push(`${full}:${i + 1}:${ln}`);
+      if (!stat11.isFile() || stat11.size > maxBytes || stopped || signal.aborted) return;
+      const file = await fs7.open(full, "r");
+      try {
+        let bytesReadTotal = 0;
+        let lineNumber = 0;
+        let fileHits = 0;
+        let leftover = "";
+        let binaryChecked = false;
+        const buffer = Buffer.allocUnsafe(Math.min(NATIVE_READ_CHUNK_BYTES, maxBytes));
+        while (!stopped && !signal.aborted && bytesReadTotal < maxBytes) {
+          const remaining = maxBytes - bytesReadTotal;
+          const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, remaining), null);
+          if (bytesRead === 0) break;
+          const chunk = buffer.subarray(0, bytesRead);
+          if (!binaryChecked) {
+            binaryChecked = true;
+            if (isBinaryBuffer(chunk)) return;
+          }
+          bytesReadTotal += bytesRead;
+          const text = leftover + chunk.toString("utf8");
+          const lines = text.split(/\r?\n/);
+          leftover = lines.pop() ?? "";
+          for (const rawLine of lines) {
+            if (stopped || signal.aborted) break;
+            lineNumber++;
+            const ln = capSubject(rawLine);
+            re.lastIndex = 0;
+            if (!re.test(ln)) continue;
+            fileHits++;
+            total++;
+            if (mode === "content") {
+              if (matches.length < limit) matches.push(`${full}:${lineNumber}:${ln}`);
+            } else if (mode === "files_with_matches") {
+              if (fileHits === 1 && matches.length < limit) matches.push(full);
+              break;
+            } else if (fileHits === 1 && matches.length < limit) {
+              matches.push(`${full}:${countOnlyFirstHit ? 1 : 0}`);
+            }
+            if (countOnlyFirstHit || mode !== "content" && matches.length >= limit) {
+              stopped = true;
+              break;
+            }
           }
+          if (mode === "files_with_matches" && fileHits > 0) break;
         }
-      }
-      if (fileHits > 0) {
-        fileMatches.set(full, fileHits);
-        if (mode === "files_with_matches" && matches.length < limit) {
-          matches.push(full);
+        if (!stopped && !signal.aborted && leftover.length > 0) {
+          lineNumber++;
+          const ln = capSubject(leftover);
+          re.lastIndex = 0;
+          if (re.test(ln)) {
+            fileHits++;
+            total++;
+            if (mode === "content") {
+              if (matches.length < limit) matches.push(`${full}:${lineNumber}:${ln}`);
+            } else if (mode === "files_with_matches") {
+              if (matches.length < limit) matches.push(full);
+            } else if (matches.length < limit) {
+              matches.push(`${full}:${countOnlyFirstHit ? 1 : fileHits}`);
+            }
+          }
         }
-        if (mode === "count" && matches.length < limit) {
-          matches.push(`${full}:${fileHits}`);
+        if (fileHits > 0) {
+          if (mode === "count") {
+            const idx = matches.findIndex((entry) => entry.startsWith(`${full}:`));
+            if (idx !== -1) matches[idx] = `${full}:${fileHits}`;
+          }
+          if (mode === "files_with_matches" && matches.length >= limit) stopped = true;
         }
+        if (mode === "content" && matches.length >= limit) stopped = true;
+        if (mode === "count" && matches.length >= limit && (countOnlyFirstHit || mode !== "count")) stopped = true;
+      } finally {
+        await file.close();
       }
-      if (matches.length >= limit) stopped = true;
     } catch {
     }
   };
@@ -1139,18 +1239,20 @@ async function runNative(input, base, mode, limit, signal) {
       return;
     }
     const files = [];
+    const subdirs = [];
     for (const e of entries) {
       if (stopped) return;
       if (DEFAULT_IGNORE3.includes(e.name)) continue;
       if (e.isSymbolicLink()) continue;
       const full = path.join(dir, e.name);
       if (e.isDirectory()) {
-        await walk(full);
+        subdirs.push(full);
       } else if (e.isFile()) {
         files.push({ full, name: e.name });
       }
     }
     await mapWithConcurrency(files, NATIVE_SCAN_CONCURRENCY, ({ full, name }) => scanFile(full, name));
+    await mapWithConcurrency(subdirs, Math.min(16, NATIVE_SCAN_CONCURRENCY), walk);
   };
   await walk(base);
   return {
@@ -1160,20 +1262,6 @@ async function runNative(input, base, mode, limit, signal) {
     used: "native"
   };
 }
-async function mapWithConcurrency(items, concurrency, fn) {
-  if (items.length === 0) return;
-  let next = 0;
-  const workerCount = Math.min(Math.max(1, concurrency), items.length);
-  const workers = Array.from({ length: workerCount }, async () => {
-    for (; ; ) {
-      const idx = next++;
-      if (idx >= items.length) return;
-      const item = items[idx];
-      if (item !== void 0) await fn(item);
-    }
-  });
-  await Promise.all(workers);
-}
 var SPOOL_RETENTION_MS = 7 * 24 * 60 * 60 * 1e3;
 var SPOOL_WRITE_HWM_BYTES = 4 * 1024 * 1024;
 var sweepStarted = false;
@@ -1215,7 +1303,7 @@ function createOutputSpool(opts) {
   let filePath = null;
   let failed = false;
   let finalized = false;
-  const open = () => {
+  const open2 = () => {
     if (stream || failed) return;
     try {
       const dir = toolOutputDir();
@@ -1248,7 +1336,7 @@ function createOutputSpool(opts) {
           return;
         }
         head += text;
-        open();
+        open2();
         head = "";
         return;
       }
@@ -4161,13 +4249,13 @@ ${formatTaskList(taskFile.tasks)}`
   }
 };
 function mkResult(plan, ok, message, todos) {
-  const open = plan.items.filter((i) => i.status !== "done").length;
+  const open2 = plan.items.filter((i) => i.status !== "done").length;
   const result = {
     ok,
     message,
     plan: formatPlan(plan),
     count: plan.items.length,
-    open
+    open: open2
   };
   if (todos !== void 0) result.todos = todos;
   return result;
@@ -8039,6 +8127,8 @@ var IndexStore = class {
     this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_kind ON symbols(kind)");
     this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_lang ON symbols(lang)");
     this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_file ON symbols(file)");
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_lang_kind ON symbols(lang, kind)");
+    this.db.exec("CREATE INDEX IF NOT EXISTS idx_s_file_fk ON symbols(file_fk)");
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS refs (
         id INTEGER PRIMARY KEY,
@@ -8951,9 +9041,9 @@ async function syncGoParse(filePath, content, lang) {
   }
 }
 async function parseSymbols3(opts) {
-  const { file, lang } = opts;
+  const { file, content, lang } = opts;
   try {
-    return await syncPyParse(file, lang);
+    return await syncPyParse(file, content, lang);
   } catch {
     return { file, lang, symbols: [], mtimeMs: Date.now() };
   }
@@ -9021,8 +9111,7 @@ syms = []
 errors = []
 
 try:
-    with open(sys.argv[1], "r", encoding="utf-8") as f:
-        source = f.read()
+    source = sys.stdin.read()
     tree = ast.parse(source, filename=sys.argv[1])
 except Exception as e:
     errors.append(str(e))
@@ -9162,16 +9251,21 @@ visitor.visit(tree)
 
 print(json.dumps([s.to_dict() for s in syms]))
 `;
-async function syncPyParse(filePath, lang) {
+var _cachedScriptPath = null;
+async function syncPyParse(filePath, content, lang) {
   try {
-    const tmpDir = path.join(os2.tmpdir(), "ws-py-parse");
-    await fs7.mkdir(tmpDir, { recursive: true });
-    const scriptPath = path.join(tmpDir, "parse.py");
-    await fs7.writeFile(scriptPath, PY_PARSE_SCRIPT, "utf8");
-    const proc = spawn("python", [scriptPath, filePath], {
+    if (!_cachedScriptPath) {
+      const tmpDir = path.join(os2.tmpdir(), "ws-py-parse");
+      await fs7.mkdir(tmpDir, { recursive: true });
+      _cachedScriptPath = path.join(tmpDir, "parse.py");
+      await fs7.writeFile(_cachedScriptPath, PY_PARSE_SCRIPT, "utf8");
+    }
+    const proc = spawn("python", [_cachedScriptPath, filePath], {
       stdio: ["pipe", "pipe", "pipe"],
       windowsHide: true
     });
+    proc.stdin?.write(content);
+    proc.stdin?.end();
     let stdout = "";
     proc.stdout?.on("data", (chunk) => {
       stdout += chunk.toString();
@@ -9838,7 +9932,7 @@ async function parseFile(file, content, lang) {
     case "go":
       return parseSymbols2({ file, content, lang: "go" });
     case "py":
-      return parseSymbols3({ file, lang: "py" });
+      return parseSymbols3({ file, content, lang: "py" });
     case "rs":
       return parseSymbols4({ file, content, lang: "rs" });
     case "json":
@@ -9869,10 +9963,12 @@ async function runIndexerWithStore(store, opts) {
   let symbolsIndexed = 0;
   const isGitIgnored = await loadGitignoreMatcher(projectRoot);
   let files;
+  let discoveredFiles = null;
   if (opts.files && opts.files.length > 0) {
     files = opts.files.map((f) => path.resolve(projectRoot, f)).filter((f) => !isGitIgnored(path.relative(projectRoot, f).replace(/\\/g, "/"), false));
   } else {
     files = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
+    discoveredFiles = new Set(files);
   }
   if (langs && langs.length > 0) {
     const langSet = new Set(langs);
@@ -9990,11 +10086,11 @@ async function runIndexerWithStore(store, opts) {
       filesIndexed++;
     }
   }
-  for (const [file_] of existingMeta) {
-    try {
-      await fs7.stat(file_);
-    } catch {
-      store.deleteFile(file_);
+  if (discoveredFiles) {
+    for (const [file_] of existingMeta) {
+      if (!discoveredFiles.has(file_)) {
+        store.deleteFile(file_);
+      }
     }
   }
   const durationMs = Date.now() - startMs;