edsger 0.55.4 → 0.56.1

package/dist/phases/quality-benchmark/mcp-server.js

@@ -0,0 +1,252 @@
+/**
+ * In-process MCP server exposing the quality-benchmark execution
+ * primitives to the Claude Agent SDK session.
+ *
+ * The single SDK session reasons about the rubric in its system prompt
+ * and calls these tools to drive the 6-phase pipeline:
+ *
+ *   list_applicable_tools(detected_context) — Phase 1 -> 2 handoff
+ *   probe_tool(tool_id)                     — Phase 2
+ *   install_tool(tool_id)                   — Phase 2.5
+ *   run_tool(tool_id)                       — Phase 3 (returns parsed summary)
+ *   verify_finding(file, line, snippet?)    — Phase 5
+ *   record_progress(phase, message)         — UI streaming side channel
+ *
+ * Every command and install step is whitelisted by `tool-catalog.ts`
+ * and re-checked at runtime by `tool-runner.ts`. The MCP server is a
+ * thin adapter between the SDK's tool-call protocol and that runner —
+ * it adds no new privilege, no new commands, and no new side effects.
+ */
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { createSdkMcpServer, tool } from '@anthropic-ai/claude-agent-sdk';
+import { z } from 'zod';
+import { selectToolsForContext, TOOL_CATALOG_BY_ID } from './tool-catalog.js';
+import { executeTool, installFailureToUnavailable, installTool, probeTool, probeToUnavailable, } from './tool-runner.js';
+export function createEmptyRunState() {
+    return {
+        tool_versions: {},
+        unavailable_tools: [],
+        tool_outputs: {},
+        parsed_summaries: {},
+        dropped_findings: 0,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function textResult(data, isError = false) {
+    const text = typeof data === 'string' ? data : JSON.stringify(data);
+    return {
+        content: [{ type: 'text', text }],
+        ...(isError ? { isError: true } : {}),
+    };
+}
+function emit(deps, event) {
+    deps.onProgress?.(event);
+}
+// ---------------------------------------------------------------------------
+// Server factory
+// ---------------------------------------------------------------------------
+export function createQualityBenchmarkMcpServer(deps, state) {
+    const listApplicable = tool('list_applicable_tools', 'List the catalog tools that apply to the detected repo. Pass the languages / package-managers / frameworks you identified in Phase 1; the server returns the subset of tools whose `applies_to` and `requires` gates are satisfied. Use this to decide what to probe in Phase 2.', {
+        languages: z
+            .array(z.string())
+            .describe('Detected language tags (e.g. ["ts","py"]).'),
+        package_managers: z
+            .array(z.string())
+            .optional()
+            .describe('Detected package managers (npm/pnpm/yarn/...).'),
+        frameworks: z
+            .array(z.string())
+            .optional()
+            .describe('Detected frameworks (rails/django/...).'),
+        files_present: z
+            .array(z.string())
+            .optional()
+            .describe('Manifest / config filenames present at the repo root (e.g. "tsconfig.json").'),
+    }, async (args) => {
+        const tools = selectToolsForContext({
+            languages: args.languages,
+            package_managers: args.package_managers,
+            frameworks: args.frameworks,
+            files_present: args.files_present,
+        });
+        return textResult({
+            tools: tools.map((t) => ({
+                id: t.id,
+                label: t.label,
+                category: t.category,
+                install_prereq: t.install_prereq,
+                subscores: t.subscores,
+                requires: t.requires,
+            })),
+        });
+    });
+    const probe = tool('probe_tool', 'Probe whether a tool from the catalog is installed and reachable. Returns availability + version + the install command that would install it. Use this in Phase 2 before deciding to install / skip.', {
+        tool_id: z.string().describe('Catalog tool id (e.g. "semgrep").'),
+    }, async (args) => {
+        if (!TOOL_CATALOG_BY_ID.has(args.tool_id)) {
+            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
+        }
+        const result = await probeTool(args.tool_id, deps.runner);
+        if (result.available && result.version) {
+            state.tool_versions[args.tool_id] = result.version;
+        }
+        emit(deps, {
+            phase: 'probing',
+            message: `${args.tool_id}: ${result.available ? `available (${result.version ?? '?'})` : 'not found'}`,
+            data: { tool_id: args.tool_id, available: result.available },
+        });
+        return textResult(result);
+    });
+    const install = tool('install_tool', 'Install a missing catalog tool to user-space. Only the install command registered in the catalog is allowed — `sudo`, `apt`, `brew`, `yum`, etc. are rejected. Returns {installed, version, error?}. If the user has disabled installation (--no-install), this returns install_disabled.', {
+        tool_id: z.string().describe('Catalog tool id to install.'),
+    }, async (args) => {
+        const entry = TOOL_CATALOG_BY_ID.get(args.tool_id);
+        if (!entry) {
+            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
+        }
+        emit(deps, {
+            phase: 'installation',
+            message: `Installing ${args.tool_id}...`,
+            data: { tool_id: args.tool_id },
+        });
+        const result = await installTool(args.tool_id, deps.runner);
+        if (result.installed && result.version) {
+            state.tool_versions[args.tool_id] = result.version;
+        }
+        else {
+            state.unavailable_tools.push(installFailureToUnavailable(args.tool_id, result));
+        }
+        emit(deps, {
+            phase: 'installation',
+            message: `${args.tool_id}: ${result.installed ? `installed (${result.version ?? '?'})` : `failed (${result.error ?? 'unknown'})`}`,
+        });
+        return textResult(result);
+    });
+    const run = tool('run_tool', 'Execute a catalog tool against the repo. Returns the **parsed summary only** (counts / top-N findings / metrics) — never raw output, which is saved to disk and referenced by tool_outputs.raw_output_path. The summary is one of three shapes: counts (style linters), findings (security/correctness), or metrics (LOC/complexity).', {
+        tool_id: z.string().describe('Catalog tool id to run.'),
+    }, async (args) => {
+        const entry = TOOL_CATALOG_BY_ID.get(args.tool_id);
+        if (!entry) {
+            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
+        }
+        // If we never confirmed the tool is available, refuse — caller must probe first.
+        if (!state.tool_versions[args.tool_id]) {
+            const probed = await probeTool(args.tool_id, deps.runner);
+            if (!probed.available) {
+                state.unavailable_tools.push(probeToUnavailable(args.tool_id, probed, 'not_found'));
+                emit(deps, {
+                    phase: 'execution',
+                    message: `${args.tool_id}: skipped (not installed)`,
+                });
+                return textResult({
+                    ran: false,
+                    reason: 'not_available',
+                    install_command: probed.install_command,
+                });
+            }
+            if (probed.version) {
+                state.tool_versions[args.tool_id] = probed.version;
+            }
+        }
+        emit(deps, {
+            phase: 'execution',
+            message: `Running ${args.tool_id}...`,
+            data: { tool_id: args.tool_id },
+        });
+        const { parsed, run: runMeta, ok, } = await executeTool(args.tool_id, deps.runner);
+        state.parsed_summaries[args.tool_id] = parsed;
+        state.tool_outputs[args.tool_id] = runMeta;
+        emit(deps, {
+            phase: 'execution',
+            message: `${args.tool_id}: ${parsed.oneliner}`,
+        });
+        return textResult({
+            ran: true,
+            ok,
+            oneliner: parsed.oneliner,
+            summary: parsed.summary,
+            meta: {
+                duration_ms: runMeta.duration_ms,
+                exit_code: runMeta.exit_code,
+                raw_output_path: runMeta.raw_output_path,
+            },
+        });
+    });
+    const verify = tool('verify_finding', 'Validate that a `file:line` claim is real before including it as evidence in the final report. Returns { exists, line_in_range, snippet_matches } so the LLM can drop hallucinated entries. Used during Phase 5.', {
+        file: z.string().describe('Repo-relative path of the file.'),
+        line: z.number().int().min(1).describe('1-based line number.'),
+        snippet: z
+            .string()
+            .optional()
+            .describe('Optional snippet that should appear within +/-3 lines of `line`.'),
+    }, async (args) => {
+        const result = verifyFinding(deps.runner.repo_root, args);
+        if (!result.exists ||
+            !result.line_in_range ||
+            result.snippet_matches === false) {
+            state.dropped_findings += 1;
+        }
+        return textResult(result);
+    });
+    const progress = tool('record_progress', 'Send a progress message to the UI (and the DB). Does not affect scoring. Use it to keep users informed during long phases.', {
+        phase: z
+            .enum([
+            'detection',
+            'probing',
+            'installation',
+            'execution',
+            'external_signals',
+            'verification',
+            'synthesis',
+        ])
+            .describe('Which phase the message belongs to.'),
+        message: z.string().describe('Human-readable status update.'),
+    }, async (args) => {
+        emit(deps, { phase: args.phase, message: args.message });
+        return textResult({ acknowledged: true });
+    });
+    return createSdkMcpServer({
+        name: 'quality-benchmark',
+        version: '1.0.0',
+        tools: [listApplicable, probe, install, run, verify, progress],
+    });
+}
+function verifyFinding(repoRoot, args) {
+    const abs = join(repoRoot, args.file);
+    if (!existsSync(abs)) {
+        return { exists: false, line_in_range: false };
+    }
+    let stat;
+    try {
+        stat = statSync(abs);
+    }
+    catch {
+        return { exists: false, line_in_range: false };
+    }
+    if (!stat.isFile()) {
+        return { exists: false, line_in_range: false };
+    }
+    let content;
+    try {
+        content = readFileSync(abs, 'utf8');
+    }
+    catch {
+        return { exists: true, line_in_range: false };
+    }
+    const lines = content.split(/\r?\n/);
+    const lineInRange = args.line >= 1 && args.line <= lines.length;
+    if (!args.snippet) {
+        return { exists: true, line_in_range: lineInRange };
+    }
+    if (!lineInRange) {
+        return { exists: true, line_in_range: false, snippet_matches: false };
+    }
+    const start = Math.max(0, args.line - 4);
+    const end = Math.min(lines.length, args.line + 3);
+    const window = lines.slice(start, end).join('\n');
+    const snippet_matches = window.includes(args.snippet.trim().slice(0, 80));
+    return { exists: true, line_in_range: true, snippet_matches };
+}

package/dist/phases/quality-benchmark/parsers.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Tool output parsers. Each parser extracts the minimum information the
+ * LLM needs to score the dimension — either counts (Tier 1), counts plus
+ * top findings (Tier 2), or domain-specific metrics (Tier 3).
+ *
+ * Full raw outputs are saved to disk by the tool-runner and never enter
+ * the LLM context — these parsers operate on the stdout/stderr strings
+ * captured during execution.
+ *
+ * Severity mapping:
+ *   tool-specific → our 4-tier enum (critical | high | medium | low)
+ *
+ * All parsers must be:
+ *   - Defensive: never throw. Return `parsed: false` (via a counts-zero
+ *     summary) if the output is malformed.
+ *   - Cheap: simple JSON parse + small map. No regex over megabytes.
+ *   - Stable: same input → same output (no randomness, no clocks).
+ */
+import type { ParsedToolOutput, ParserContext, ParserFn } from './types.js';
+export declare const PARSERS: Record<string, ParserFn>;
+/** Run the parser for a tool, defensively swallowing errors. */
+export declare function parseToolOutput(toolId: string, stdout: string, stderr: string, ctx: ParserContext): ParsedToolOutput;