mixdog 0.9.38 → 0.9.39

package/src/runtime/agent/orchestrator/tools/builtin/rg-runner.mjs

@@ -1,9 +1,13 @@
 import { spawn, execFile } from 'child_process';
 import { existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { resolve } from 'node:path';
+import { accessSync, constants, statSync } from 'node:fs';
 import { promisify } from 'node:util';
 import os from 'node:os';
-import { acquire as acquireChildSpawnSlot } from '../../../../shared/child-spawn-gate.mjs';
+import {
+    acquire as acquireChildSpawnSlot,
+    hasSpareCapacity as hasRgSpareCapacity,
+} from '../../../../shared/child-spawn-gate.mjs';
 
 const execFileAsync = promisify(execFile);
 
@@ -28,15 +32,48 @@ const execFileAsync = promisify(execFile);
 // ── rg --threads cap ─────────────────────────────────────────────────────
 // Each rg process otherwise fans out across EVERY core; with several agents
 // running grep at once the box over-saturates and the whole batch trips the
-// 20s deadline together. Cap rg's worker threads to a fraction of the host so
-// concurrent greps share the machine. Internal dynamic default; env override
-// only — deliberately NOT exposed on any tool schema / parameter surface.
-function _rgThreadCap() {
+// 20s deadline together. Use the larger half-CPU cap only when the spawn gate
+// has spare capacity; otherwise retain the conservative quarter-CPU cap.
+// Internal dynamic default; env override only — deliberately NOT exposed on
+// any tool schema / parameter surface.
+let _rgDefaultThreadCap = null;
+export function _rgThreadCap(spareCapacity = false) {
     const override = Number(process.env.MIXDOG_RG_THREADS);
     if (Number.isFinite(override) && override >= 1) return Math.floor(override);
-    let cpus = 0;
-    try { cpus = os.cpus()?.length || 0; } catch { cpus = 0; }
-    return Math.max(2, Math.ceil((cpus || 4) / 4));
+    if (_rgDefaultThreadCap === null) {
+        let cpus = 0;
+        try { cpus = os.cpus()?.length || 0; } catch { cpus = 0; }
+        _rgDefaultThreadCap = cpus || 4;
+    }
+    const cpus = _rgDefaultThreadCap;
+    return spareCapacity
+        ? Math.max(4, Math.ceil(cpus / 2))
+        : Math.max(2, Math.ceil(cpus / 4));
+}
+
+// Walk rg options once for all thread-argument helpers. Values of these
+// flags are opaque patterns/paths and must not be mistaken for options.
+const _rgValueFlags = new Set([
+    '-e', '--glob', '--iglob', '--type', '-A', '-B', '-C',
+    '--max-depth', '--max-columns',
+]);
+function _walkRgArgs(argsList, visit) {
+    let options = true;
+    for (let i = 0; i < argsList.length; i++) {
+        const a = argsList[i];
+        if (options && a === '--') {
+            visit(a, i, null, true);
+            options = false;
+            continue;
+        }
+        if (options && typeof a === 'string'
+            && (_rgValueFlags.has(a) || a === '-j' || a === '--threads')) {
+            visit(a, i, i + 1, true);
+            i++;
+            continue;
+        }
+        visit(a, i, null, options);
+    }
 }
 
 // Single source of truth for "did the caller already pin rg's thread count?".
@@ -44,23 +81,23 @@ function _rgThreadCap() {
 // is the NEXT arg, short-attached (`-j8`), and long-attached (`--threads=8`).
 // Used by both the _withRgThreads injection guard and the EAGAIN retry guard
 // so the two never disagree.
-function _hasRgThreadArg(argsList) {
-    for (const a of argsList) {
-        if (typeof a !== 'string') continue;
-        if (a === '-j' || a === '--threads') return true;
-        if (/^-j\d+$/.test(a)) return true;
-        if (a.startsWith('--threads=')) return true;
-    }
-    return false;
+export function _hasRgThreadArg(argsList) {
+    let found = false;
+    _walkRgArgs(argsList, (a, _index, _valueIndex, options) => {
+        if (!options || typeof a !== 'string') return;
+        if (a === '-j' || a === '--threads'
+            || /^-j\d+$/.test(a) || a.startsWith('--threads=')) found = true;
+    });
+    return found;
 }
 
 // Inject `--threads N` unless the caller already pinned thread count
 // (`-j`/`--threads`/`-jN`/`--threads=N`, e.g. the EAGAIN `-j 1` retry). Does
 // not mutate; may return the ORIGINAL array unchanged when already
 // thread-pinned (callers must not assume a fresh copy).
-function _withRgThreads(argsList) {
+export function _withRgThreads(argsList, spareCapacity = false) {
     if (_hasRgThreadArg(argsList)) return argsList;
-    return ['--threads', String(_rgThreadCap()), ...argsList];
+    return ['--threads', String(_rgThreadCap(spareCapacity)), ...argsList];
 }
 
 // Build the EAGAIN single-thread retry args. If the caller already pinned a
@@ -68,14 +105,15 @@ function _withRgThreads(argsList) {
 // retry is unambiguously single-threaded (never `-j 8 ... -j 1`). Drops both the
 // separated form (`-j N` / `--threads N` → flag + following count) and the
 // attached forms (`-jN` / `--threads=N`).
-function _rgEagainRetryArgs(argsList) {
+export function _rgEagainRetryArgs(argsList) {
     const out = [];
-    for (let i = 0; i < argsList.length; i++) {
-        const a = argsList[i];
-        if (a === '-j' || a === '--threads') { i++; continue; } // skip flag + its count
-        if (typeof a === 'string' && (/^-j\d+$/.test(a) || a.startsWith('--threads='))) continue;
+    _walkRgArgs(argsList, (a, index, valueIndex, options) => {
+        if (options && typeof a === 'string'
+            && (a === '-j' || a === '--threads'
+                || /^-j\d+$/.test(a) || a.startsWith('--threads='))) return;
         out.push(a);
-    }
+        if (valueIndex !== null && valueIndex < argsList.length) out.push(argsList[valueIndex]);
+    });
     return ['-j', '1', ...out];
 }
 
@@ -83,43 +121,75 @@ function _rgEagainRetryArgs(argsList) {
 // retry would change nothing. Matches the separated `-j 1` / `--threads 1` form
 // and the attached `-j1` / `--threads=1` form. Any other thread pin (e.g. -j8)
 // is NOT "single-thread pinned": the retry should still run and downshift it.
-function _isRgSingleThreadPinned(argsList) {
-    for (let i = 0; i < argsList.length; i++) {
-        const a = argsList[i];
-        if (typeof a !== 'string') continue;
-        if ((a === '-j' || a === '--threads') && String(argsList[i + 1]).trim() === '1') return true;
-        if (a === '-j1' || a === '--threads=1') return true;
-    }
-    return false;
+export function _isRgSingleThreadPinned(argsList) {
+    let found = false;
+    _walkRgArgs(argsList, (a, _index, valueIndex, options) => {
+        if (!options || typeof a !== 'string') return;
+        if ((a === '-j' || a === '--threads')
+            && valueIndex !== null && String(argsList[valueIndex]).trim() === '1') found = true;
+        if (a === '-j1' || a === '--threads=1') found = true;
+    });
+    return found;
 }
 
 let _rgExecutableResolved = null;
+let _rgExecutableResolutionKey = null;
+let _rgResolvePromise = null;
+let _rgResolvePromiseKey = null;
+let _rgResolutionGeneration = 0;
 
-async function _resolveRgExecutable() {
-    const isWin = process.platform === 'win32';
+function _rgResolutionKey() {
+    return [
+        process.platform,
+        process.cwd(),
+        String(process.env.PATH || ''),
+        String(process.env.PATHEXT || ''),
+    ].join('\0');
+}
+
+function _usableRgCandidate(candidate, isWin = process.platform === 'win32') {
     try {
-        const cmd = isWin ? 'where' : 'which';
-        const { stdout: out } = await execFileAsync(cmd, ['rg'], { encoding: 'utf8', windowsHide: true });
-        const first = out.split(/\r?\n/).map((l) => l.trim()).find(Boolean);
-        if (first && existsSync(first)) return first;
-    } catch { /* fall through to PATH scan */ }
+        const info = statSync(candidate);
+        if (!info.isFile()) return false;
+        if (isWin) {
+            const ext = candidate.slice(candidate.lastIndexOf('.')).toLowerCase();
+            return ext === '.exe' || ext === '.com';
+        }
+        accessSync(candidate, constants.X_OK);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+export async function _resolveRgExecutable() {
+    const isWin = process.platform === 'win32';
     const pathSep = isWin ? ';' : ':';
-    const dirs = String(process.env.PATH || '').split(pathSep).filter(Boolean);
-    const pathext = isWin
-        ? String(process.env.PATHEXT || '.EXE;.CMD;.BAT').split(';').map((e) => e.toLowerCase())
-        : [''];
-    const names = isWin ? ['rg.exe', 'rg'] : ['rg'];
-    for (const dir of dirs) {
+    const dirs = String(process.env.PATH || '').split(pathSep);
+    // Avoid `where`/`which` on the normal path: on Windows it costs tens of
+    // milliseconds before doing the same filesystem search. Windows command
+    // lookup checks the current directory before PATH and applies PATHEXT in
+    // order; POSIX lookup is just the PATH order.
+    const searchDirs = isWin ? [process.cwd(), ...dirs] : dirs;
+    const names = isWin
+        ? String(process.env.PATHEXT || '.EXE;.CMD;.BAT').split(';')
+            .map((ext) => ext.trim().toLowerCase())
+            .filter((ext) => ext === '.exe' || ext === '.com')
+            .map((ext) => `rg${ext}`)
+        : ['rg'];
+    for (const dir of searchDirs) {
         for (const name of names) {
-            const candidate = join(dir, name);
-            if (!existsSync(candidate)) continue;
-            if (isWin) {
-                const ext = candidate.slice(candidate.lastIndexOf('.')).toLowerCase();
-                if (ext && !pathext.includes(ext)) continue;
-            }
-            return candidate;
+            const candidate = resolve(dir, name);
+            if (existsSync(candidate) && _usableRgCandidate(candidate, isWin)) return candidate;
         }
     }
+    try {
+        const cmd = isWin ? 'where' : 'which';
+        const { stdout: out } = await execFileAsync(cmd, ['rg'], { encoding: 'utf8', windowsHide: true });
+        const first = out.split(/\r?\n/).map((l) => l.trim()).find(Boolean);
+        const candidate = first ? resolve(first) : '';
+        if (candidate && _usableRgCandidate(candidate, isWin)) return candidate;
+    } catch { /* preserve bare-rg fallback */ }
     return 'rg';
 }
 
@@ -134,12 +204,23 @@ function rgExecutable() {
 // blocking the event loop ~90ms on the first grep/glob/find of a session
 // (freezing the TUI). Resolve asynchronously and cache the result; subsequent
 // calls return immediately.
-let _rgResolvePromise = null;
-async function ensureRgResolved() {
-    if (_rgExecutableResolved !== null) return _rgExecutableResolved;
-    if (!_rgResolvePromise) {
+export async function ensureRgResolved() {
+    const key = _rgResolutionKey();
+    const cachedUsable = _rgExecutableResolved !== null
+        && _rgExecutableResolved !== 'rg'
+        && _usableRgCandidate(_rgExecutableResolved);
+    if (_rgExecutableResolutionKey === key && cachedUsable) return _rgExecutableResolved;
+    if (_rgExecutableResolved !== null && (_rgExecutableResolutionKey !== key || !cachedUsable)) {
+        _rgResolutionGeneration++;
+        _rgExecutableResolved = null;
+        _rgExecutableResolutionKey = null;
+        _rgResolvePromise = null;
+    }
+    if (!_rgResolvePromise || _rgResolvePromiseKey !== key) {
+        _rgResolvePromiseKey = key;
         _rgResolvePromise = _resolveRgExecutable().then((resolved) => {
             _rgExecutableResolved = resolved;
+            _rgExecutableResolutionKey = key;
             return resolved;
         });
     }
@@ -155,7 +236,6 @@ const RG_FALLBACK_FAIL_TTL_MS = 30000;
 let _rgFallbackUsable = null;
 let _rgFallbackFailAt = 0;
 async function rgFallbackUsable() {
-    if (_rgFallbackUsable === true) return true;
     if (_rgFallbackUsable === false && (Date.now() - _rgFallbackFailAt) < RG_FALLBACK_FAIL_TTL_MS) {
         return false;
     }
@@ -195,12 +275,16 @@ const RG_PCRE2_FAIL_TTL_MS = 30000;
 let _rgPcre2Supported = null;
 let _rgPcre2FailAt = 0;
 export async function rgSupportsPcre2() {
-    if (_rgPcre2Supported === true) return true;
-    if (_rgPcre2Supported === false && (Date.now() - _rgPcre2FailAt) < RG_PCRE2_FAIL_TTL_MS) {
-        return false;
+    await ensureRgResolved();
+    const cacheKey = `${_rgResolutionGeneration}:${rgExecutable()}`;
+    if (rgSupportsPcre2._cacheKey !== cacheKey) {
+        _rgPcre2Supported = null;
+        _rgPcre2FailAt = 0;
+        rgSupportsPcre2._cacheKey = cacheKey;
     }
+    if (_rgPcre2Supported === true) return true;
+    if (_rgPcre2Supported === false && (Date.now() - _rgPcre2FailAt) < RG_PCRE2_FAIL_TTL_MS) return false;
     try {
-        await ensureRgResolved();
         await execFileAsync(rgExecutable(), ['--pcre2-version'], { windowsHide: true, timeout: 3000 });
         _rgPcre2Supported = true;
     } catch {
@@ -311,7 +395,9 @@ function spawnRg(argsList, execOptions) {
     // node guardian is started: rg owns its full SIGTERM→grace→force-kill +
     // force-settle teardown, so the 1:1 guardian process was pure overhead.
     return acquireChildSpawnSlot(gateSignal).then((releaseSlot) => new Promise((resolve, reject) => {
-        const proc = spawn(rgExecutable(), _withRgThreads(argsList), {
+        // Probe after acquisition: this spawn counts toward inflight. The
+        // answer is best-effort and only selects the per-process rg cap.
+        const proc = spawn(rgExecutable(), _withRgThreads(argsList, hasRgSpareCapacity()), {
             cwd: execOptions?.cwd,
             env: execOptions?.env || process.env,
             windowsHide: true,
@@ -456,7 +542,8 @@ function spawnRgWindowedLines(argsList, execOptions, opts = {}) {
     // excluded from the rg timeout (armed after spawn). releaseSlot is fired
     // once via .finally on every settle path.
     return acquireChildSpawnSlot(gateSignal).then((releaseSlot) => new Promise((resolve, reject) => {
-        const proc = spawn(rgExecutable(), _withRgThreads(argsList), {
+        // As above, probe only after this spawn has acquired its gate slot.
+        const proc = spawn(rgExecutable(), _withRgThreads(argsList, hasRgSpareCapacity()), {
             cwd: execOptions?.cwd,
             env: execOptions?.env || process.env,
             windowsHide: true,

package/src/runtime/agent/orchestrator/tools/builtin/search-path-diagnostics.mjs

@@ -67,6 +67,24 @@ const NOT_FOUND_CODES = new Set(['ENOENT', 'ENOTDIR']);
 
 export const ENOENT_FIND_NUDGE = 'Locate with find on the basename before retrying.';
 
+// Per-invocation memo for the ENOENT recovery fs scans. A single grep/glob
+// ENOENT surfaces the SAME missing path through tryReadFamilyEnoentRedirect
+// (resolveUniqueEnoentRedirect) AND buildNotFoundHint (which re-runs
+// resolveUniqueEnoentRedirect plus its own findFileByBasename). Threading an
+// optional cache object keyed on that single path collapses the repeated
+// suffix-strip / basename BFS into one scan. Absent cache → scan fresh
+// (unchanged behavior for callers that don't pass one).
+function cachedSuffixStrip(workDir, missingPath, cache) {
+    if (!cache) return findBySuffixStrip(workDir, missingPath);
+    if (!('suffixHit' in cache)) cache.suffixHit = findBySuffixStrip(workDir, missingPath);
+    return cache.suffixHit;
+}
+function cachedFileByBasename(workDir, missingPath, cache) {
+    if (!cache) return findFileByBasename(workDir, missingPath);
+    if (!('fileHits' in cache)) cache.fileHits = findFileByBasename(workDir, missingPath);
+    return cache.fileHits;
+}
+
 function isDirectoryPathGuess(missingPath) {
     const base = basename(String(missingPath || '').replace(/\\/g, '/'));
     if (!base || /[*?[\]{}]/.test(base)) return false;
@@ -112,11 +130,11 @@ export function finalizeReadFamilyEnoentTail(hint, requestedPath, errCode = 'ENO
     return appendEnoentFindNudge(String(hint || '') + spaceJoinedPathHint(requestedPath));
 }
 
-export function resolveUniqueEnoentRedirect(workDir, missingPath, errCode = 'ENOENT') {
+export function resolveUniqueEnoentRedirect(workDir, missingPath, errCode = 'ENOENT', cache = null) {
     if (!NOT_FOUND_CODES.has(String(errCode || 'ENOENT'))) return null;
-    const suffixHit = findBySuffixStrip(workDir, missingPath);
+    const suffixHit = cachedSuffixStrip(workDir, missingPath, cache);
     if (suffixHit) return suffixHit;
-    const elsewhere = findFileByBasename(workDir, missingPath);
+    const elsewhere = cachedFileByBasename(workDir, missingPath, cache);
     if (elsewhere.length === 1) return elsewhere[0];
     if (isDirectoryPathGuess(missingPath)) {
         const dirHits = findDirectoryByBasename(workDir, missingPath, { limit: 3 });
@@ -136,19 +154,20 @@ export async function tryReadFamilyEnoentRedirect({
     errCode,
     options,
     rerun,
+    cache = null,
 }) {
     if (options?._enoentRedirectFrom) return null;
-    const target = resolveUniqueEnoentRedirect(workDir, resolvedPath, errCode);
+    const target = resolveUniqueEnoentRedirect(workDir, resolvedPath, errCode, cache);
     if (!target) return null;
     const body = await rerun(target, { ...options, _enoentRedirectFrom: resolvedPath });
     const shown = requestedPath ?? resolvedPath;
     return redirectedFromPrefix(shown) + body;
 }
 
-export function buildNotFoundHint(workDir, missingPath, actionVerb, errCode = 'ENOENT') {
+export function buildNotFoundHint(workDir, missingPath, actionVerb, errCode = 'ENOENT', cache = null) {
     if (!NOT_FOUND_CODES.has(String(errCode || 'ENOENT'))) return '';
-    if (resolveUniqueEnoentRedirect(workDir, missingPath, errCode)) return '';
-    const elsewhere = findFileByBasename(workDir, missingPath);
+    if (resolveUniqueEnoentRedirect(workDir, missingPath, errCode, cache)) return '';
+    const elsewhere = cachedFileByBasename(workDir, missingPath, cache);
     if (elsewhere.length) {
         return ` Not found at this path; the same filename exists at: ${elsewhere.map((p) => `"${normalizeOutputPath(p)}"`).join(', ')}. ${actionVerb} that path directly.`;
     }
@@ -211,15 +230,20 @@ export function uncRefusalMessage(toolName, original, resolved) {
     return `Error: ${toolName} refuses UNC/SMB path ${JSON.stringify(shown)}; remote share access is blocked to prevent NTLM credential leaks`;
 }
 
-export function basePathDiagnostic(basePaths, workDir) {
+export function basePathDiagnostic(basePaths, workDir, statCache = null) {
     return basePaths.map((basePath) => {
         const resolved = resolveSearchScope(basePath, workDir);
-        try {
-            const st = statSync(resolved);
+        // Reuse the caller's call-scoped stat cache (glob preflight + per-group
+        // rg runs already stat'd this same resolved root) instead of re-stating.
+        let st = null;
+        let err = null;
+        const cached = statCache && statCache.get(resolved);
+        if (cached) { st = cached.st; err = cached.err; }
+        else { try { st = statSync(resolved); } catch (e) { err = e; } }
+        if (!err) {
             return `${normalizeOutputPath(basePath)}: ${st.isDirectory() ? 'path exists (dir)' : 'path exists (file)'}`;
-        } catch (err) {
-            return `${normalizeOutputPath(basePath)}: path does not exist (${err?.code || 'ENOENT'})`
-                + finalizeReadFamilyEnoentTail(buildNotFoundHint(workDir, resolved, 'Search', err?.code), basePath, err?.code);
         }
+        return `${normalizeOutputPath(basePath)}: path does not exist (${err?.code || 'ENOENT'})`
+            + finalizeReadFamilyEnoentTail(buildNotFoundHint(workDir, resolved, 'Search', err?.code), basePath, err?.code);
     }).join('; ');
 }