edsger 0.77.0 → 0.78.0

package/dist/phases/quality-benchmark/index.js

@@ -1,208 +0,0 @@
-/**
- * Quality benchmark phase entry point.
- *
- * Drives a single Claude Agent SDK session whose system prompt is the
- * full rubric (loaded by `prompts.ts`) and whose tool surface is the
- * in-process MCP server in `mcp-server.ts`. The session walks the
- * rubric's 6-phase pipeline by calling MCP tools; we observe the
- * messages for progress events and capture the final JSON report.
- *
- * Pure compute boundary: this function does NOT touch the database.
- * Callers (the CLI command, the desktop-app IPC bridge) decide how to
- * persist the returned payload — see the CLI command in
- * `src/commands/quality-benchmark/index.ts` for the MCP-mediated
- * persistence path.
- *
- * Side effects that DO happen here:
- *   - Creates a scratch dir under ~/.edsger/quality-runs/<run-id>/
- *   - Writes raw tool outputs to that dir
- *   - Spawns child_process invocations of the catalog tools
- */
-import { spawnSync } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
-import { query } from '@anthropic-ai/claude-agent-sdk';
-import { DEFAULT_MODEL } from '../../constants.js';
-import { logError, logInfo, logSuccess } from '../../utils/logger.js';
-import { createPromptGenerator } from '../pr-shared/agent-utils.js';
-import { createEmptyRunState, createQualityBenchmarkMcpServer, } from './mcp-server.js';
-import { createQualityBenchmarkSystemPrompt, createQualityBenchmarkUserPrompt, extractReportJson, } from './prompts.js';
-import { createRunnerContext } from './tool-runner.js';
-import { RUBRIC_VERSION, } from './types.js';
-// SDK upper bound. Each phase usually takes ~1-3 turns; with up to 30
-// tools probed + run plus per-finding verification, 500 gives headroom while
-// still cutting off runaway loops.
-const MAX_TURNS = 500;
-// ---------------------------------------------------------------------------
-// Orchestration
-// ---------------------------------------------------------------------------
-// eslint-disable-next-line complexity
-export async function runQualityBenchmark(opts) {
-    const { productId, productName, repoRoot, branch, packageManager, installEnabled, onProgress, verbose, scanDirBase, commitShaOverride, } = opts;
-    // 1. Determine commit sha from the local checkout.
-    const commitSha = commitShaOverride ?? readGitHead(repoRoot);
-    if (!commitSha) {
-        return {
-            status: 'error',
-            message: `Could not read git HEAD at ${repoRoot} — is this a git repo?`,
-        };
-    }
-    // 2. Set up runner + MCP server.
-    const runId = randomUUID();
-    const startedAt = new Date().toISOString();
-    const runner = createRunnerContext({
-        repo_root: repoRoot,
-        package_manager: packageManager,
-        install_enabled: installEnabled,
-        run_id: runId,
-        base_dir: scanDirBase,
-    });
-    const state = createEmptyRunState();
-    let lastPhase = 'detection';
-    const emit = (event) => {
-        lastPhase = event.phase;
-        onProgress?.(event);
-        if (verbose) {
-            logInfo(`[${event.phase}] ${event.message}`);
-        }
-    };
-    const mcpServer = createQualityBenchmarkMcpServer({ runner, onProgress: emit, installEnabled }, state);
-    // 3. Run the SDK session.
-    const systemPrompt = createQualityBenchmarkSystemPrompt();
-    const userPrompt = createQualityBenchmarkUserPrompt({
-        productName,
-        productId,
-        repoRoot,
-        branch,
-        commitSha,
-        runId,
-        installEnabled,
-    });
-    let lastAssistantText = '';
-    try {
-        for await (const message of query({
-            prompt: createPromptGenerator(userPrompt),
-            options: {
-                systemPrompt: {
-                    type: 'preset',
-                    preset: 'claude_code',
-                    append: systemPrompt,
-                },
-                model: DEFAULT_MODEL,
-                maxTurns: MAX_TURNS,
-                permissionMode: 'bypassPermissions',
-                cwd: repoRoot,
-                mcpServers: {
-                    'quality-benchmark': mcpServer,
-                },
-            },
-        })) {
-            if (message.type === 'assistant' && message.message?.content) {
-                for (const c of message.message.content) {
-                    if (c.type === 'text') {
-                        lastAssistantText += `${c.text}\n`;
-                        if (verbose) {
-                            logInfo(`🤖 ${c.text}`);
-                        }
-                    }
-                }
-            }
-            if (message.type === 'result') {
-                if (message.subtype !== 'success') {
-                    logError(`SDK session ended with subtype=${message.subtype}`);
-                }
-                if (message.subtype === 'success' &&
-                    'result' in message &&
-                    typeof message.result === 'string') {
-                    lastAssistantText = message.result || lastAssistantText;
-                }
-            }
-        }
-    }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        return {
-            status: 'error',
-            message: `SDK error: ${msg}`,
-            runId,
-            commitSha,
-            lastPhase,
-        };
-    }
-    // 4. Extract + validate JSON report.
-    const raw = extractReportJson(lastAssistantText);
-    if (!raw || typeof raw !== 'object') {
-        return {
-            status: 'error',
-            message: 'SDK session did not emit a parseable JSON report',
-            runId,
-            commitSha,
-            lastPhase: 'synthesis',
-        };
-    }
-    const report = raw;
-    // Merge SDK-emitted fields with the state we accumulated server-side. The
-    // server-side state is authoritative for execution metadata; the LLM owns
-    // the scoring fields.
-    const merged = {
-        ...report,
-        rubric_version: RUBRIC_VERSION,
-        tool_versions: { ...state.tool_versions, ...(report.tool_versions ?? {}) },
-        unavailable_tools: dedupUnavailable([
-            ...state.unavailable_tools,
-            ...(report.unavailable_tools ?? []),
-        ]),
-        tool_outputs: enrichToolOutputsWithMetrics({ ...state.tool_outputs, ...(report.tool_outputs ?? {}) }, state.parsed_summaries),
-        dropped_findings: Math.max(state.dropped_findings, report.dropped_findings ?? 0),
-    };
-    const completedAt = new Date().toISOString();
-    const durationSec = Math.round((new Date(completedAt).getTime() - new Date(startedAt).getTime()) / 1000);
-    logSuccess(`Benchmark complete — grade ${merged.overall_grade ?? '?'} (${merged.overall_score ?? '?'})`);
-    return {
-        status: 'success',
-        report: merged,
-        runId,
-        commitSha,
-        startedAt,
-        completedAt,
-        durationSeconds: durationSec,
-    };
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function readGitHead(repoRoot) {
-    const res = spawnSync('git', ['rev-parse', 'HEAD'], {
-        cwd: repoRoot,
-        encoding: 'utf8',
-    });
-    if (res.status !== 0) {
-        return null;
-    }
-    return res.stdout.trim() || null;
-}
-/**
- * Fold tier-3 (`metrics`) parser output into the persisted tool_outputs so
- * trend charts can read numeric values (duplication %, complexity, …) without
- * re-parsing oneliners. Count/finding tools carry no metrics and are untouched.
- */
-function enrichToolOutputsWithMetrics(toolOutputs, parsedSummaries) {
-    const out = { ...toolOutputs };
-    for (const [id, parsed] of Object.entries(parsedSummaries)) {
-        if (parsed.summary.tier === 'metrics' && out[id]) {
-            out[id] = { ...out[id], metrics: parsed.summary.metrics };
-        }
-    }
-    return out;
-}
-function dedupUnavailable(list) {
-    const seen = new Set();
-    const out = [];
-    for (const entry of list) {
-        if (seen.has(entry.name)) {
-            continue;
-        }
-        seen.add(entry.name);
-        out.push(entry);
-    }
-    return out;
-}

package/dist/phases/quality-benchmark/mcp-server.d.ts

@@ -1,46 +0,0 @@
-/**
- * In-process MCP server exposing the quality-benchmark execution
- * primitives to the Claude Agent SDK session.
- *
- * The single SDK session reasons about the rubric in its system prompt
- * and calls these tools to drive the 6-phase pipeline:
- *
- *   list_applicable_tools(detected_context) — Phase 1 -> 2 handoff
- *   probe_tool(tool_id)                     — Phase 2
- *   install_tool(tool_id)                   — Phase 2.5
- *   run_tool(tool_id)                       — Phase 3 (returns parsed summary)
- *   verify_finding(file, line, snippet?)    — Phase 5
- *   record_progress(phase, message)         — UI streaming side channel
- *
- * Every command and install step is whitelisted by `tool-catalog.ts`
- * and re-checked at runtime by `tool-runner.ts`. The MCP server is a
- * thin adapter between the SDK's tool-call protocol and that runner —
- * it adds no new privilege, no new commands, and no new side effects.
- */
-import { type RunnerContext } from './tool-runner.js';
-import type { ParsedToolOutput, RunPhase, ToolRunOutput, UnavailableTool } from './types.js';
-export interface QualityMcpDeps {
-    /** Runner context shared across all tool calls in this benchmark run. */
-    runner: RunnerContext;
-    /** Optional callback for streaming progress updates to the UI / DB. */
-    onProgress?: (event: ProgressEvent) => void;
-    /** When false, install_tool always returns install_disabled. */
-    installEnabled: boolean;
-}
-export interface ProgressEvent {
-    phase: RunPhase;
-    message: string;
-    data?: Record<string, unknown>;
-}
-/** State accumulated across a single benchmark run. */
-export interface RunState {
-    tool_versions: Record<string, string>;
-    unavailable_tools: UnavailableTool[];
-    tool_outputs: Record<string, ToolRunOutput>;
-    /** Cached parsed summaries for synthesis-time read-back. */
-    parsed_summaries: Record<string, ParsedToolOutput>;
-    /** Dropped finding count from Phase 5 verification. */
-    dropped_findings: number;
-}
-export declare function createEmptyRunState(): RunState;
-export declare function createQualityBenchmarkMcpServer(deps: QualityMcpDeps, state: RunState): import("@anthropic-ai/claude-agent-sdk").McpSdkServerConfigWithInstance;

package/dist/phases/quality-benchmark/mcp-server.js

@@ -1,252 +0,0 @@
-/**
- * In-process MCP server exposing the quality-benchmark execution
- * primitives to the Claude Agent SDK session.
- *
- * The single SDK session reasons about the rubric in its system prompt
- * and calls these tools to drive the 6-phase pipeline:
- *
- *   list_applicable_tools(detected_context) — Phase 1 -> 2 handoff
- *   probe_tool(tool_id)                     — Phase 2
- *   install_tool(tool_id)                   — Phase 2.5
- *   run_tool(tool_id)                       — Phase 3 (returns parsed summary)
- *   verify_finding(file, line, snippet?)    — Phase 5
- *   record_progress(phase, message)         — UI streaming side channel
- *
- * Every command and install step is whitelisted by `tool-catalog.ts`
- * and re-checked at runtime by `tool-runner.ts`. The MCP server is a
- * thin adapter between the SDK's tool-call protocol and that runner —
- * it adds no new privilege, no new commands, and no new side effects.
- */
-import { existsSync, readFileSync, statSync } from 'node:fs';
-import { join } from 'node:path';
-import { createSdkMcpServer, tool } from '@anthropic-ai/claude-agent-sdk';
-import { z } from 'zod';
-import { selectToolsForContext, TOOL_CATALOG_BY_ID } from './tool-catalog.js';
-import { executeTool, installFailureToUnavailable, installTool, probeTool, probeToUnavailable, } from './tool-runner.js';
-export function createEmptyRunState() {
-    return {
-        tool_versions: {},
-        unavailable_tools: [],
-        tool_outputs: {},
-        parsed_summaries: {},
-        dropped_findings: 0,
-    };
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function textResult(data, isError = false) {
-    const text = typeof data === 'string' ? data : JSON.stringify(data);
-    return {
-        content: [{ type: 'text', text }],
-        ...(isError ? { isError: true } : {}),
-    };
-}
-function emit(deps, event) {
-    deps.onProgress?.(event);
-}
-// ---------------------------------------------------------------------------
-// Server factory
-// ---------------------------------------------------------------------------
-export function createQualityBenchmarkMcpServer(deps, state) {
-    const listApplicable = tool('list_applicable_tools', 'List the catalog tools that apply to the detected repo. Pass the languages / package-managers / frameworks you identified in Phase 1; the server returns the subset of tools whose `applies_to` and `requires` gates are satisfied. Use this to decide what to probe in Phase 2.', {
-        languages: z
-            .array(z.string())
-            .describe('Detected language tags (e.g. ["ts","py"]).'),
-        package_managers: z
-            .array(z.string())
-            .optional()
-            .describe('Detected package managers (npm/pnpm/yarn/...).'),
-        frameworks: z
-            .array(z.string())
-            .optional()
-            .describe('Detected frameworks (rails/django/...).'),
-        files_present: z
-            .array(z.string())
-            .optional()
-            .describe('Manifest / config filenames present at the repo root (e.g. "tsconfig.json").'),
-    }, async (args) => {
-        const tools = selectToolsForContext({
-            languages: args.languages,
-            package_managers: args.package_managers,
-            frameworks: args.frameworks,
-            files_present: args.files_present,
-        });
-        return textResult({
-            tools: tools.map((t) => ({
-                id: t.id,
-                label: t.label,
-                category: t.category,
-                install_prereq: t.install_prereq,
-                subscores: t.subscores,
-                requires: t.requires,
-            })),
-        });
-    });
-    const probe = tool('probe_tool', 'Probe whether a tool from the catalog is installed and reachable. Returns availability + version + the install command that would install it. Use this in Phase 2 before deciding to install / skip.', {
-        tool_id: z.string().describe('Catalog tool id (e.g. "semgrep").'),
-    }, async (args) => {
-        if (!TOOL_CATALOG_BY_ID.has(args.tool_id)) {
-            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
-        }
-        const result = await probeTool(args.tool_id, deps.runner);
-        if (result.available && result.version) {
-            state.tool_versions[args.tool_id] = result.version;
-        }
-        emit(deps, {
-            phase: 'probing',
-            message: `${args.tool_id}: ${result.available ? `available (${result.version ?? '?'})` : 'not found'}`,
-            data: { tool_id: args.tool_id, available: result.available },
-        });
-        return textResult(result);
-    });
-    const install = tool('install_tool', 'Install a missing catalog tool to user-space. Only the install command registered in the catalog is allowed — `sudo`, `apt`, `brew`, `yum`, etc. are rejected. Returns {installed, version, error?}. If the user has disabled installation (--no-install), this returns install_disabled.', {
-        tool_id: z.string().describe('Catalog tool id to install.'),
-    }, async (args) => {
-        const entry = TOOL_CATALOG_BY_ID.get(args.tool_id);
-        if (!entry) {
-            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
-        }
-        emit(deps, {
-            phase: 'installation',
-            message: `Installing ${args.tool_id}...`,
-            data: { tool_id: args.tool_id },
-        });
-        const result = await installTool(args.tool_id, deps.runner);
-        if (result.installed && result.version) {
-            state.tool_versions[args.tool_id] = result.version;
-        }
-        else {
-            state.unavailable_tools.push(installFailureToUnavailable(args.tool_id, result));
-        }
-        emit(deps, {
-            phase: 'installation',
-            message: `${args.tool_id}: ${result.installed ? `installed (${result.version ?? '?'})` : `failed (${result.error ?? 'unknown'})`}`,
-        });
-        return textResult(result);
-    });
-    const run = tool('run_tool', 'Execute a catalog tool against the repo. Returns the **parsed summary only** (counts / top-N findings / metrics) — never raw output, which is saved to disk and referenced by tool_outputs.raw_output_path. The summary is one of three shapes: counts (style linters), findings (security/correctness), or metrics (LOC/complexity).', {
-        tool_id: z.string().describe('Catalog tool id to run.'),
-    }, async (args) => {
-        const entry = TOOL_CATALOG_BY_ID.get(args.tool_id);
-        if (!entry) {
-            return textResult({ error: `Unknown tool_id: ${args.tool_id}` }, true);
-        }
-        // If we never confirmed the tool is available, refuse — caller must probe first.
-        if (!state.tool_versions[args.tool_id]) {
-            const probed = await probeTool(args.tool_id, deps.runner);
-            if (!probed.available) {
-                state.unavailable_tools.push(probeToUnavailable(args.tool_id, probed, 'not_found'));
-                emit(deps, {
-                    phase: 'execution',
-                    message: `${args.tool_id}: skipped (not installed)`,
-                });
-                return textResult({
-                    ran: false,
-                    reason: 'not_available',
-                    install_command: probed.install_command,
-                });
-            }
-            if (probed.version) {
-                state.tool_versions[args.tool_id] = probed.version;
-            }
-        }
-        emit(deps, {
-            phase: 'execution',
-            message: `Running ${args.tool_id}...`,
-            data: { tool_id: args.tool_id },
-        });
-        const { parsed, run: runMeta, ok, } = await executeTool(args.tool_id, deps.runner);
-        state.parsed_summaries[args.tool_id] = parsed;
-        state.tool_outputs[args.tool_id] = runMeta;
-        emit(deps, {
-            phase: 'execution',
-            message: `${args.tool_id}: ${parsed.oneliner}`,
-        });
-        return textResult({
-            ran: true,
-            ok,
-            oneliner: parsed.oneliner,
-            summary: parsed.summary,
-            meta: {
-                duration_ms: runMeta.duration_ms,
-                exit_code: runMeta.exit_code,
-                raw_output_path: runMeta.raw_output_path,
-            },
-        });
-    });
-    const verify = tool('verify_finding', 'Validate that a `file:line` claim is real before including it as evidence in the final report. Returns { exists, line_in_range, snippet_matches } so the LLM can drop hallucinated entries. Used during Phase 5.', {
-        file: z.string().describe('Repo-relative path of the file.'),
-        line: z.number().int().min(1).describe('1-based line number.'),
-        snippet: z
-            .string()
-            .optional()
-            .describe('Optional snippet that should appear within +/-3 lines of `line`.'),
-    }, async (args) => {
-        const result = verifyFinding(deps.runner.repo_root, args);
-        if (!result.exists ||
-            !result.line_in_range ||
-            result.snippet_matches === false) {
-            state.dropped_findings += 1;
-        }
-        return textResult(result);
-    });
-    const progress = tool('record_progress', 'Send a progress message to the UI (and the DB). Does not affect scoring. Use it to keep users informed during long phases.', {
-        phase: z
-            .enum([
-            'detection',
-            'probing',
-            'installation',
-            'execution',
-            'external_signals',
-            'verification',
-            'synthesis',
-        ])
-            .describe('Which phase the message belongs to.'),
-        message: z.string().describe('Human-readable status update.'),
-    }, async (args) => {
-        emit(deps, { phase: args.phase, message: args.message });
-        return textResult({ acknowledged: true });
-    });
-    return createSdkMcpServer({
-        name: 'quality-benchmark',
-        version: '1.0.0',
-        tools: [listApplicable, probe, install, run, verify, progress],
-    });
-}
-function verifyFinding(repoRoot, args) {
-    const abs = join(repoRoot, args.file);
-    if (!existsSync(abs)) {
-        return { exists: false, line_in_range: false };
-    }
-    let stat;
-    try {
-        stat = statSync(abs);
-    }
-    catch {
-        return { exists: false, line_in_range: false };
-    }
-    if (!stat.isFile()) {
-        return { exists: false, line_in_range: false };
-    }
-    let content;
-    try {
-        content = readFileSync(abs, 'utf8');
-    }
-    catch {
-        return { exists: true, line_in_range: false };
-    }
-    const lines = content.split(/\r?\n/);
-    const lineInRange = args.line >= 1 && args.line <= lines.length;
-    if (!args.snippet) {
-        return { exists: true, line_in_range: lineInRange };
-    }
-    if (!lineInRange) {
-        return { exists: true, line_in_range: false, snippet_matches: false };
-    }
-    const start = Math.max(0, args.line - 4);
-    const end = Math.min(lines.length, args.line + 3);
-    const window = lines.slice(start, end).join('\n');
-    const snippet_matches = window.includes(args.snippet.trim().slice(0, 80));
-    return { exists: true, line_in_range: true, snippet_matches };
-}

package/dist/phases/quality-benchmark/parsers.d.ts

@@ -1,45 +0,0 @@
-/**
- * Tool output parsers. Each parser extracts the minimum information the
- * LLM needs to score the dimension — either counts (Tier 1), counts plus
- * top findings (Tier 2), or domain-specific metrics (Tier 3).
- *
- * Full raw outputs are saved to disk by the tool-runner and never enter
- * the LLM context — these parsers operate on the stdout/stderr strings
- * captured during execution.
- *
- * Severity mapping:
- *   tool-specific → our 4-tier enum (critical | high | medium | low)
- *
- * All parsers must be:
- *   - Defensive: never throw. Return `parsed: false` (via a counts-zero
- *     summary) if the output is malformed.
- *   - Cheap: simple JSON parse + small map. No regex over megabytes.
- *   - Stable: same input → same output (no randomness, no clocks).
- */
-import type { ParsedToolOutput, ParserContext, ParserFn } from './types.js';
-interface DepGraphNode {
-    id: string;
-    label: string;
-    fan_in: number;
-    fan_out: number;
-    in_cycle: boolean;
-}
-interface DepGraph {
-    nodes: DepGraphNode[];
-    edges: {
-        from: string;
-        to: string;
-    }[];
-    total_modules: number;
-    truncated: boolean;
-}
-/**
- * Build a bounded dependency graph from madge's adjacency map. Cycle nodes are
- * always kept; the remaining slots go to the highest-degree modules. Edges are
- * restricted to kept nodes.
- */
-export declare function buildDependencyGraph(adjObj: Record<string, string[]>): DepGraph;
-export declare const PARSERS: Record<string, ParserFn>;
-/** Run the parser for a tool, defensively swallowing errors. */
-export declare function parseToolOutput(toolId: string, stdout: string, stderr: string, ctx: ParserContext): ParsedToolOutput;
-export {};