@wrongstack/tools 0.296.4 → 0.298.0

package/dist/grep.js

@@ -1,10 +1,14 @@
 // src/grep.ts
-import { expectDefined } from "@wrongstack/core/utils";
 import { spawn } from "node:child_process";
 import * as fs2 from "node:fs/promises";
 import * as path3 from "node:path";
 import { ToolValidationError } from "@wrongstack/core/types";
-import { buildChildEnv, compileGlob as compileGlob2, DEFAULT_WALK_IGNORE_DIRS } from "@wrongstack/core/utils";
+import {
+  buildChildEnv,
+  compileGlob as compileGlob2,
+  DEFAULT_WALK_IGNORE_DIRS,
+  expectDefined
+} from "@wrongstack/core/utils";
 
 // src/_concurrency.ts
 async function mapWithConcurrency(items, limit, fn) {
@@ -68,9 +72,41 @@ function capSubject(line) {
   return line.length > MAX_SUBJECT_LEN ? line.slice(0, MAX_SUBJECT_LEN) : line;
 }
 
+// src/_util.ts
+import * as path from "node:path";
+import * as Core from "@wrongstack/core/utils";
+function resolvePath(input, ctx) {
+  return path.isAbsolute(input) ? path.normalize(input) : path.resolve(ctx.workingDir ?? ctx.cwd, input);
+}
+function allowedRoots(ctx) {
+  return [path.resolve(ctx.projectRoot), path.resolve(Core.wstackGlobalRoot())];
+}
+function isInsideAny(target, roots) {
+  return roots.some((root) => {
+    const rel = path.relative(root, target);
+    return rel === "" || !rel.startsWith("..") && !path.isAbsolute(rel);
+  });
+}
+function ensureInsideRoot(absPath, ctx) {
+  const target = path.resolve(absPath);
+  if (ctx.allowOutsideProjectRoot) return target;
+  if (isInsideAny(target, allowedRoots(ctx))) return target;
+  throw new Error(`Path "${absPath}" is outside project root "${path.resolve(ctx.projectRoot)}"`);
+}
+function safeResolve(input, ctx) {
+  return ensureInsideRoot(resolvePath(input, ctx), ctx);
+}
+function isBinaryBuffer(buf) {
+  const len = Math.min(buf.length, 8192);
+  for (let i = 0; i < len; i++) {
+    if (buf[i] === 0) return true;
+  }
+  return false;
+}
+
 // src/codebase-index/gitignore.ts
 import * as fs from "node:fs/promises";
-import * as path from "node:path";
+import * as path2 from "node:path";
 import { compileGlob } from "@wrongstack/core/utils";
 function globBody(glob) {
   return compileGlob(glob).source.replace(/^\^/, "").replace(/\$$/, "");
@@ -116,50 +152,20 @@ function compileGitignore(lines) {
 async function loadGitignoreMatcher(projectRoot) {
   let lines = [];
   try {
-    const raw = await fs.readFile(path.join(projectRoot, ".gitignore"), "utf8");
+    const raw = await fs.readFile(path2.join(projectRoot, ".gitignore"), "utf8");
     lines = raw.split("\n");
   } catch {
   }
   return compileGitignore(lines);
 }
 
-// src/_util.ts
-import * as path2 from "node:path";
-import * as Core from "@wrongstack/core/utils";
-function resolvePath(input, ctx) {
-  return path2.isAbsolute(input) ? path2.normalize(input) : path2.resolve(ctx.workingDir ?? ctx.cwd, input);
-}
-function allowedRoots(ctx) {
-  return [path2.resolve(ctx.projectRoot), path2.resolve(Core.wstackGlobalRoot())];
-}
-function isInsideAny(target, roots) {
-  return roots.some((root) => {
-    const rel = path2.relative(root, target);
-    return rel === "" || !rel.startsWith("..") && !path2.isAbsolute(rel);
-  });
-}
-function ensureInsideRoot(absPath, ctx) {
-  const target = path2.resolve(absPath);
-  if (ctx.allowOutsideProjectRoot) return target;
-  if (isInsideAny(target, allowedRoots(ctx))) return target;
-  throw new Error(`Path "${absPath}" is outside project root "${path2.resolve(ctx.projectRoot)}"`);
-}
-function safeResolve(input, ctx) {
-  return ensureInsideRoot(resolvePath(input, ctx), ctx);
-}
-function isBinaryBuffer(buf) {
-  const len = Math.min(buf.length, 8192);
-  for (let i = 0; i < len; i++) {
-    if (buf[i] === 0) return true;
-  }
-  return false;
-}
-
 // src/grep.ts
 var DEFAULT_IGNORE = DEFAULT_WALK_IGNORE_DIRS;
 var NATIVE_SCAN_CONCURRENCY = 32;
 var NATIVE_READ_CHUNK_BYTES = 64 * 1024;
 var NATIVE_MAX_FILE_BYTES = 1e6;
+var RG_MAX_QUEUE_CHUNKS = 128;
+var RG_MAX_QUEUE_CHARS = 8 * 1024 * 1024;
 var grepTool = {
   name: "grep",
   category: "Search",
@@ -287,9 +293,19 @@ async function* runRgStream(input, base, mode, limit, signal) {
   const FLUSH_AT = 16;
   const MAX_BUF_BYTES = 1e6;
   let bufOverflow = false;
-  const child = spawn("rg", args, { signal, env: buildChildEnv(), stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
+  const child = spawn("rg", args, {
+    signal,
+    env: buildChildEnv(),
+    // rg diagnostics are not part of the tool result. Ignoring stderr avoids
+    // an unread pipe filling up and pinning the child when rg reports many
+    // filesystem errors.
+    stdio: ["ignore", "pipe", "ignore"],
+    windowsHide: true
+  });
   const queue = [];
+  let queuedChars = 0;
   let waiter;
+  let paused = false;
   const wake = () => {
     if (waiter) {
       const w = waiter;
@@ -297,93 +313,133 @@ async function* runRgStream(input, base, mode, limit, signal) {
       w();
     }
   };
-  child.stdout?.on("data", (c) => {
-    queue.push({ kind: "out", data: c.toString() });
+  const pauseIfFlooded = () => {
+    if (paused || queue.length < RG_MAX_QUEUE_CHUNKS && queuedChars < RG_MAX_QUEUE_CHARS)
+      return;
+    paused = true;
+    child.stdout?.pause();
+  };
+  const resumeIfDrained = () => {
+    if (!paused || queue.length >= RG_MAX_QUEUE_CHUNKS || queuedChars >= RG_MAX_QUEUE_CHARS)
+      return;
+    paused = false;
+    child.stdout?.resume();
+  };
+  const onStdoutData = (c) => {
+    const data = c.toString();
+    queue.push({ kind: "out", data });
+    queuedChars += data.length;
     wake();
-  });
-  child.on("error", (e) => {
+    pauseIfFlooded();
+  };
+  const onError = (e) => {
     queue.push({ kind: "error", data: e.message });
     wake();
-  });
-  child.on("close", () => {
+  };
+  const onClose = () => {
     queue.push({ kind: "close", data: "" });
     wake();
-  });
+  };
+  child.stdout?.on("data", onStdoutData);
+  child.on("error", onError);
+  child.on("close", onClose);
   let pendingBatch = [];
   let errored = false;
-  for (; ; ) {
-    while (queue.length === 0) {
-      await new Promise((r) => {
-        waiter = r;
-      });
-    }
-    const c = expectDefined(queue.shift());
-    if (c.kind === "error") {
-      errored = true;
-      continue;
-    }
-    if (c.kind === "close") break;
-    buf += c.data;
-    if (buf.length > MAX_BUF_BYTES && !bufOverflow) {
-      bufOverflow = true;
-      buf = buf.slice(-MAX_BUF_BYTES);
-      try {
-        child.kill("SIGTERM");
-      } catch {
+  let closed = false;
+  try {
+    for (; ; ) {
+      while (queue.length === 0) {
+        await new Promise((r) => {
+          waiter = r;
+        });
+      }
+      const c = expectDefined(queue.shift());
+      queuedChars = Math.max(0, queuedChars - c.data.length);
+      resumeIfDrained();
+      if (c.kind === "error") {
+        errored = true;
+        continue;
+      }
+      if (c.kind === "close") {
+        closed = true;
+        break;
+      }
+      buf += c.data;
+      if (buf.length > MAX_BUF_BYTES && !bufOverflow) {
+        bufOverflow = true;
+        buf = buf.slice(-MAX_BUF_BYTES);
+        try {
+          child.kill("SIGTERM");
+        } catch {
+        }
+      }
+      const idx = buf.lastIndexOf("\n");
+      if (idx === -1) continue;
+      const ready = buf.slice(0, idx);
+      buf = buf.slice(idx + 1);
+      for (const line of ready.split("\n")) {
+        if (!line) continue;
+        totalLines++;
+        if (mode === "count") totalCount += parseRgCountLine(line);
+        if (matches.length < limit) {
+          matches.push(line);
+          pendingBatch.push(line);
+          batchSinceFlush++;
+        }
+      }
+      if (batchSinceFlush >= FLUSH_AT) {
+        yield {
+          type: "partial_output",
+          text: pendingBatch.join("\n"),
+          data: { matches_so_far: matches.length }
+        };
+        pendingBatch = [];
+        batchSinceFlush = 0;
       }
     }
-    const idx = buf.lastIndexOf("\n");
-    if (idx === -1) continue;
-    const ready = buf.slice(0, idx);
-    buf = buf.slice(idx + 1);
-    for (const line of ready.split("\n")) {
-      if (!line) continue;
-      totalLines++;
-      if (mode === "count") totalCount += parseRgCountLine(line);
-      if (matches.length < limit) {
-        matches.push(line);
-        pendingBatch.push(line);
-        batchSinceFlush++;
+    if (buf.trim()) {
+      for (const line of buf.split("\n")) {
+        if (!line) continue;
+        totalLines++;
+        if (mode === "count") totalCount += parseRgCountLine(line);
+        if (matches.length < limit) {
+          matches.push(line);
+          pendingBatch.push(line);
+        }
       }
     }
-    if (batchSinceFlush >= FLUSH_AT) {
+    if (pendingBatch.length > 0) {
       yield {
         type: "partial_output",
         text: pendingBatch.join("\n"),
         data: { matches_so_far: matches.length }
       };
-      pendingBatch = [];
-      batchSinceFlush = 0;
     }
-  }
-  if (buf.trim()) {
-    for (const line of buf.split("\n")) {
-      if (!line) continue;
-      totalLines++;
-      if (mode === "count") totalCount += parseRgCountLine(line);
-      if (matches.length < limit) {
-        matches.push(line);
-        pendingBatch.push(line);
-      }
-    }
-  }
-  if (pendingBatch.length > 0) {
+    if (errored) throw new Error("rg: spawn error");
     yield {
-      type: "partial_output",
-      text: pendingBatch.join("\n"),
-      data: { matches_so_far: matches.length }
+      type: "final",
+      output: {
+        matches,
+        count: mode === "count" ? totalCount : totalLines,
+        truncated: totalLines > limit || bufOverflow,
+        used: "rg"
+      }
     };
-  }
-  if (errored) throw new Error("rg: spawn error");
-  yield {
-    type: "final",
-    output: {
-      matches,
-      count: mode === "count" ? totalCount : totalLines,
-      truncated: totalLines > limit || bufOverflow,
-      used: "rg"
+  } finally {
+    child.stdout?.off("data", onStdoutData);
+    child.off("error", onError);
+    child.off("close", onClose);
+    child.stdout?.destroy();
+    queue.length = 0;
+    queuedChars = 0;
+    waiter = void 0;
+    if (!closed && child.exitCode === null) {
+      try {
+        child.kill("SIGTERM");
+      } catch {
+      }
     }
-  };
+  }
 }
 function parseRgCountLine(line) {
   const idx = line.lastIndexOf(":");