edsger 0.55.4 → 0.56.1

package/dist/phases/quality-benchmark/tool-runner.js

@@ -0,0 +1,399 @@
+/**
+ * Tool execution layer for the quality-benchmark phase.
+ *
+ * Three operations exposed to the rest of the phase (and ultimately to
+ * the LLM via the MCP server in `mcp-server.ts`):
+ *
+ *   probeTool(id, ctx)    -> { available, version, install_command }
+ *   installTool(id, ctx)  -> { installed, version, error }
+ *   executeTool(id, ctx)  -> { summary, run, parser_output }
+ *
+ * All commands come from the authoritative `TOOL_CATALOG` — callers may
+ * not pass arbitrary command strings. Outputs are saved to a per-run
+ * scratch directory so the LLM never has to ferry megabytes of JSON
+ * through its context; only the small `ParsedToolOutput` flows back.
+ *
+ * Safety rails enforced here (in addition to those documented in the
+ * rubric):
+ *   - All commands run with cwd = repo root (or an explicit override)
+ *   - Per-command timeout; SIGKILL on overrun
+ *   - No sudo / no system package managers (refused at install time)
+ *   - Captured outputs truncated at 16 MiB to bound disk usage
+ */
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync, promises as fsp, readFileSync } from 'node:fs';
+import { homedir, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { parseToolOutput } from './parsers.js';
+import { ALL_INSTALL_COMMANDS, TOOL_CATALOG_BY_ID } from './tool-catalog.js';
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+/** Maximum captured stdout/stderr per command (16 MiB). */
+const MAX_CAPTURE_BYTES = 16 * 1024 * 1024;
+/** Buffer threshold for streaming output to disk — keeps RSS low on huge logs. */
+const STREAM_FLUSH_BYTES = 1 * 1024 * 1024;
+/** Default timeout for installer commands (5 minutes). */
+const INSTALL_TIMEOUT_MS = 5 * 60 * 1000;
+/** Probe commands should always be near-instant; clamp to 30 s. */
+const PROBE_TIMEOUT_MS = 30 * 1000;
+/**
+ * Forbidden installer patterns. The catalog itself never contains these,
+ * but we re-check at runtime as a defense in depth — a future PR cannot
+ * silently introduce `sudo apt install foo`.
+ */
+const FORBIDDEN_INSTALL_PATTERNS = [
+    /\bsudo\b/i,
+    /\bapt(-get)?\b/i,
+    /\bbrew\b/i,
+    /\byum\b/i,
+    /\bdnf\b/i,
+    /\bpacman\b/i,
+    /\bzypper\b/i,
+    /\bapk\b/i,
+    /\bchoco(latey)?\b/i,
+    /\bscoop\b/i,
+    /\bwinget\b/i,
+];
+/**
+ * Forbidden flags in tool invocations: anything that mutates the repo.
+ * Caught on top of catalog whitelisting because future maintainers might
+ * accidentally introduce `--fix`.
+ */
+const FORBIDDEN_RUN_PATTERNS = [
+    /\B--fix\b/,
+    /\B--auto-?fix\b/,
+    /\B--write\b/,
+    /\bgit\s+(commit|push|reset\s+--hard|clean\s+-f)/i,
+    /\brm\s+-rf\s+\//,
+];
+// ---------------------------------------------------------------------------
+// Scan-dir helpers
+// ---------------------------------------------------------------------------
+export function createRunnerContext(opts) {
+    const runId = opts.run_id ?? new Date().toISOString().replace(/[:.]/g, '-');
+    const base = opts.base_dir ?? join(homedir() || tmpdir(), '.edsger', 'quality-runs');
+    const scanDir = join(base, runId);
+    mkdirSync(scanDir, { recursive: true });
+    return {
+        repo_root: opts.repo_root,
+        package_manager: opts.package_manager,
+        install_enabled: opts.install_enabled ?? true,
+        scan_dir: scanDir,
+    };
+}
+// ---------------------------------------------------------------------------
+// Command template resolution
+// ---------------------------------------------------------------------------
+function resolvePlaceholders(template, ctx) {
+    return template
+        .replaceAll('%REPO_ROOT%', ctx.repo_root)
+        .replaceAll('%PKG_MANAGER%', ctx.package_manager ?? 'npm')
+        .replaceAll('%SCAN_DIR%', ctx.scan_dir);
+}
+function getEntry(id) {
+    const entry = TOOL_CATALOG_BY_ID.get(id);
+    if (!entry) {
+        throw new Error(`Unknown tool id: ${id} (not in TOOL_CATALOG)`);
+    }
+    return entry;
+}
+async function runCommand(command, opts) {
+    const started = Date.now();
+    const outPath = opts.outputPath ??
+        join(opts.cwd, `.runner-${Math.random().toString(36).slice(2, 10)}.out`);
+    const child = spawn('bash', ['-lc', command], {
+        cwd: opts.cwd,
+        env: { ...process.env, ...(opts.env ?? {}) },
+        stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let stdout = '';
+    let stderr = '';
+    let stdoutBytes = 0;
+    let stderrBytes = 0;
+    let truncatedStdout = false;
+    // Open file for streaming stdout to disk (audit trail).
+    const fileHandle = await fsp.open(outPath, 'w');
+    let bufferedToFlush = '';
+    const flushBuffer = async () => {
+        if (bufferedToFlush.length === 0) {
+            return;
+        }
+        await fileHandle.write(bufferedToFlush);
+        bufferedToFlush = '';
+    };
+    child.stdout?.on('data', (chunk) => {
+        const str = chunk.toString('utf8');
+        stdoutBytes += chunk.byteLength;
+        bufferedToFlush += str;
+        if (bufferedToFlush.length >= STREAM_FLUSH_BYTES) {
+            void flushBuffer();
+        }
+        // Keep in-memory copy bounded
+        if (!truncatedStdout && stdout.length + str.length <= MAX_CAPTURE_BYTES) {
+            stdout += str;
+        }
+        else if (!truncatedStdout) {
+            stdout += str.slice(0, MAX_CAPTURE_BYTES - stdout.length);
+            truncatedStdout = true;
+        }
+    });
+    child.stderr?.on('data', (chunk) => {
+        const str = chunk.toString('utf8');
+        stderrBytes += chunk.byteLength;
+        if (stderr.length + str.length <= MAX_CAPTURE_BYTES) {
+            stderr += str;
+        }
+        else if (stderr.length < MAX_CAPTURE_BYTES) {
+            stderr += str.slice(0, MAX_CAPTURE_BYTES - stderr.length);
+        }
+    });
+    let timedOut = false;
+    const timer = setTimeout(() => {
+        timedOut = true;
+        child.kill('SIGKILL');
+    }, opts.timeoutMs);
+    const closePromise = new Promise((resolve) => {
+        child.on('close', (code, signal) => {
+            resolve({ exit_code: code ?? -1, signal });
+        });
+    });
+    const { exit_code, signal } = await closePromise;
+    clearTimeout(timer);
+    await flushBuffer();
+    await fileHandle.close();
+    return {
+        stdout,
+        stderr,
+        exit_code,
+        signal,
+        duration_ms: Date.now() - started,
+        timed_out: timedOut,
+        raw_output_path: outPath,
+        // Audit aids (not used by parsers)
+        // @ts-expect-error - intentionally extending capture with stats
+        bytes_captured: { stdout: stdoutBytes, stderr: stderrBytes },
+    };
+}
+// ---------------------------------------------------------------------------
+// Probing
+// ---------------------------------------------------------------------------
+/** Extract a version-looking token from a probe command's output. */
+function extractVersion(out) {
+    const m = out.match(/v?(\d+\.\d+(?:\.\d+)?(?:[-+][\w.-]+)?)/);
+    return m ? m[1] : null;
+}
+export async function probeTool(id, ctx) {
+    const entry = getEntry(id);
+    const cmd = resolvePlaceholders(entry.probe, ctx);
+    const cap = await runCommand(cmd, {
+        cwd: ctx.repo_root,
+        timeoutMs: PROBE_TIMEOUT_MS,
+    });
+    // `command -v` returns non-zero when not found; combined probes that run
+    // both `command -v X` and `X --version` need an exit-zero check.
+    if (cap.timed_out || cap.exit_code !== 0) {
+        return {
+            available: false,
+            version: null,
+            install_command: entry.install,
+            install_prereq: entry.install_prereq,
+            reason: cap.timed_out ? 'probe_failed' : 'not_found',
+        };
+    }
+    const version = extractVersion(cap.stdout) ?? extractVersion(cap.stderr);
+    return {
+        available: true,
+        version,
+        install_command: entry.install,
+        install_prereq: entry.install_prereq,
+    };
+}
+export async function probeInstaller(prereq, ctx) {
+    if (prereq === null) {
+        return true;
+    }
+    const probeMap = {
+        pipx: 'command -v pipx',
+        go: 'command -v go',
+        cargo: 'command -v cargo',
+        npx: 'command -v npx',
+        gem: 'command -v gem',
+    };
+    const cap = await runCommand(probeMap[prereq], {
+        cwd: ctx.repo_root,
+        timeoutMs: PROBE_TIMEOUT_MS,
+    });
+    return cap.exit_code === 0;
+}
+// ---------------------------------------------------------------------------
+// Installing
+// ---------------------------------------------------------------------------
+function assertInstallSafe(cmd) {
+    // Whitelist: must be byte-for-byte in the catalog. Defense in depth.
+    if (!ALL_INSTALL_COMMANDS.includes(cmd)) {
+        throw new Error(`Refused install command not present in TOOL_CATALOG: ${cmd.slice(0, 80)}`);
+    }
+    for (const pat of FORBIDDEN_INSTALL_PATTERNS) {
+        if (pat.test(cmd)) {
+            throw new Error(`Refused install command matching forbidden pattern ${String(pat)}: ${cmd.slice(0, 80)}`);
+        }
+    }
+}
+export async function installTool(id, ctx) {
+    const entry = getEntry(id);
+    if (!entry.install) {
+        return { installed: false, version: null, error: 'no_install_command' };
+    }
+    if (!ctx.install_enabled) {
+        return { installed: false, version: null, error: 'install_disabled' };
+    }
+    const prereqOk = await probeInstaller(entry.install_prereq, ctx);
+    if (!prereqOk) {
+        return {
+            installed: false,
+            version: null,
+            error: `prereq_missing: ${entry.install_prereq}`,
+        };
+    }
+    assertInstallSafe(entry.install);
+    const cap = await runCommand(entry.install, {
+        cwd: ctx.repo_root,
+        timeoutMs: INSTALL_TIMEOUT_MS,
+    });
+    if (cap.exit_code !== 0 || cap.timed_out) {
+        return {
+            installed: false,
+            version: null,
+            error: tailString(cap.stderr || cap.stdout, 500),
+        };
+    }
+    // Re-probe to confirm installation took
+    const re = await probeTool(id, ctx);
+    return {
+        installed: re.available,
+        version: re.version,
+        error: re.available ? undefined : 'reprobe_failed',
+    };
+}
+// ---------------------------------------------------------------------------
+// Executing
+// ---------------------------------------------------------------------------
+function assertRunSafe(cmd) {
+    for (const pat of FORBIDDEN_RUN_PATTERNS) {
+        if (pat.test(cmd)) {
+            throw new Error(`Refused run command matching forbidden pattern ${String(pat)}: ${cmd.slice(0, 120)}`);
+        }
+    }
+}
+function buildParserContext(ctx) {
+    return { repo_root: ctx.repo_root };
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+function tailString(s, n) {
+    if (!s) {
+        return '';
+    }
+    return s.length <= n ? s : s.slice(-n);
+}
+export async function executeTool(id, ctx) {
+    const entry = getEntry(id);
+    const cmd = resolvePlaceholders(entry.command, ctx);
+    assertRunSafe(cmd);
+    const outPath = join(ctx.scan_dir, `${entry.id}.out`);
+    const ranAt = nowIso();
+    const cap = await runCommand(cmd, {
+        cwd: ctx.repo_root,
+        timeoutMs: entry.timeout_minutes * 60 * 1000,
+        outputPath: outPath,
+    });
+    // Some tools genuinely succeed with non-zero exit (e.g. linters with findings).
+    const isOk = !cap.timed_out &&
+        (cap.exit_code === 0 ||
+            (entry.tolerate_nonzero_exit === true && cap.stdout.length > 0));
+    // If output was streamed to disk and the in-memory copy was truncated,
+    // re-read the head of the file for parsing.
+    const stdoutForParser = cap.stdout.length > 0 ? cap.stdout : safeReadHead(outPath);
+    const parsed = isOk
+        ? parseToolOutput(id, stdoutForParser, cap.stderr, buildParserContext(ctx))
+        : {
+            tool_id: id,
+            summary: {
+                tier: 'counts',
+                counts: { errors: 0, warnings: 0, info: 0 },
+            },
+            oneliner: cap.timed_out
+                ? `timed out after ${entry.timeout_minutes}m`
+                : `exit ${cap.exit_code}`,
+        };
+    const run = {
+        tool_id: id,
+        ran_at: ranAt,
+        duration_ms: cap.duration_ms,
+        exit_code: cap.exit_code,
+        findings_count: findingsCount(parsed),
+        summary: parsed.oneliner,
+        parsed: isOk,
+        stderr_tail: isOk ? undefined : tailString(cap.stderr, 500),
+        raw_output_path: outPath,
+    };
+    return { parsed, run, ok: isOk };
+}
+function findingsCount(p) {
+    switch (p.summary.tier) {
+        case 'counts':
+            return p.summary.counts.errors + p.summary.counts.warnings;
+        case 'findings':
+            return p.summary.counts.total;
+        case 'metrics':
+            return 0;
+        default:
+            return 0;
+    }
+}
+function safeReadHead(path, maxBytes = MAX_CAPTURE_BYTES) {
+    try {
+        if (!existsSync(path)) {
+            return '';
+        }
+        const buf = readFileSync(path);
+        const slice = buf.byteLength <= maxBytes ? buf : buf.subarray(0, maxBytes);
+        return slice.toString('utf8');
+    }
+    catch {
+        return '';
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers exported for the MCP server / phase index
+// ---------------------------------------------------------------------------
+/** Convert a failed probe into the UnavailableTool record stored in DB. */
+export function probeToUnavailable(id, probe, reason) {
+    const entry = TOOL_CATALOG_BY_ID.get(id);
+    return {
+        name: id,
+        category: entry?.category ?? 'lint',
+        install_command: probe.install_command,
+        reason,
+    };
+}
+/** Convert an install failure to UnavailableTool. */
+export function installFailureToUnavailable(id, res) {
+    const entry = TOOL_CATALOG_BY_ID.get(id);
+    let reason = 'install_failed';
+    if (res.error?.startsWith('prereq_missing')) {
+        reason = 'prereq_missing';
+    }
+    else if (res.error === 'install_disabled') {
+        reason = 'install_disabled';
+    }
+    return {
+        name: id,
+        category: entry?.category ?? 'lint',
+        install_command: entry?.install ?? null,
+        reason,
+        detail: res.error,
+    };
+}