edsger 0.55.4 → 0.56.0

package/dist/commands/quality-benchmark/index.d.ts

@@ -0,0 +1,32 @@
+/**
+ * CLI command: `edsger quality-benchmark <productId>`
+ *
+ * Runs the quality-benchmark phase against a local repo checkout and
+ * writes the resulting JSON report to disk. Persistence to the
+ * `quality_reports` table is the caller's responsibility — desktop-app
+ * picks the JSON up via stdout / file and writes via its supabase
+ * client, exactly the same pattern used by other CLI commands.
+ *
+ * Usage:
+ *   edsger quality-benchmark <productId>
+ *     --repo <path>            (required) local repo checkout
+ *     --branch <name>          (optional) branch name (informational)
+ *     --pkg-manager <name>     (optional) npm|pnpm|yarn (auto-detected if absent)
+ *     --no-install             refuse to install missing tools
+ *     --output <path>          where to write the JSON report
+ *                              (default: ./quality-report-<commit>.json)
+ *     --verbose                print every progress event
+ */
+export interface QualityBenchmarkCliOptions {
+    repo: string;
+    branch?: string;
+    pkgManager?: string;
+    install?: boolean;
+    output?: string;
+    verbose?: boolean;
+    /** When false, skip the quality_reports/save MCP call. */
+    save?: boolean;
+    /** Overwrite any existing report row for this (product, commit, rubric). */
+    force?: boolean;
+}
+export declare function runQualityBenchmarkCli(productId: string, options: QualityBenchmarkCliOptions): Promise<void>;

package/dist/commands/quality-benchmark/index.js

@@ -0,0 +1,124 @@
+/**
+ * CLI command: `edsger quality-benchmark <productId>`
+ *
+ * Runs the quality-benchmark phase against a local repo checkout and
+ * writes the resulting JSON report to disk. Persistence to the
+ * `quality_reports` table is the caller's responsibility — desktop-app
+ * picks the JSON up via stdout / file and writes via its supabase
+ * client, exactly the same pattern used by other CLI commands.
+ *
+ * Usage:
+ *   edsger quality-benchmark <productId>
+ *     --repo <path>            (required) local repo checkout
+ *     --branch <name>          (optional) branch name (informational)
+ *     --pkg-manager <name>     (optional) npm|pnpm|yarn (auto-detected if absent)
+ *     --no-install             refuse to install missing tools
+ *     --output <path>          where to write the JSON report
+ *                              (default: ./quality-report-<commit>.json)
+ *     --verbose                print every progress event
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { callMcpEndpoint } from '../../api/mcp-client.js';
+import { fetchProductBasics } from '../../phases/find-shared/mcp.js';
+import { runQualityBenchmark, } from '../../phases/quality-benchmark/index.js';
+import { logError, logInfo, logSuccess, logWarning, } from '../../utils/logger.js';
+export async function runQualityBenchmarkCli(productId, options) {
+    const repoRoot = resolve(options.repo);
+    const installEnabled = options.install !== false;
+    logInfo(`Starting quality benchmark for product ${productId}`);
+    logInfo(`Repo: ${repoRoot}`);
+    if (!installEnabled) {
+        logWarning('Install consent NOT granted (--no-install). Missing tools will be marked unmeasured.');
+    }
+    const basics = await fetchProductBasics(productId).catch(() => ({
+        name: productId,
+    }));
+    const productName = basics.name;
+    const onProgress = (event) => {
+        if (options.verbose) {
+            logInfo(`[${event.phase}] ${event.message}`);
+        }
+        else {
+            // Non-verbose: surface phase changes and tool completions only.
+            if (event.phase === 'execution' ||
+                event.phase === 'synthesis' ||
+                event.phase === 'installation') {
+                logInfo(`[${event.phase}] ${event.message}`);
+            }
+        }
+    };
+    const outcome = await runQualityBenchmark({
+        productId,
+        productName,
+        repoRoot,
+        branch: options.branch,
+        packageManager: options.pkgManager,
+        installEnabled,
+        onProgress,
+        verbose: options.verbose,
+    });
+    if (outcome.status === 'error') {
+        logError(`Quality benchmark failed: ${outcome.message}`);
+        if (outcome.lastPhase) {
+            logError(`Last phase: ${outcome.lastPhase}`);
+        }
+        process.exit(1);
+    }
+    // Write the report to disk (caller persists from there).
+    const outputPath = options.output ??
+        resolve(process.cwd(), `quality-report-${outcome.commitSha.slice(0, 8)}.json`);
+    const reportEnvelope = {
+        run_id: outcome.runId,
+        product_id: productId,
+        commit_sha: outcome.commitSha,
+        branch: options.branch ?? null,
+        started_at: outcome.startedAt,
+        completed_at: outcome.completedAt,
+        duration_seconds: outcome.durationSeconds,
+        payload: outcome.report,
+    };
+    mkdirSync(dirname(outputPath), { recursive: true });
+    writeFileSync(outputPath, JSON.stringify(reportEnvelope, null, 2), 'utf8');
+    logSuccess(`Report written to ${outputPath}`);
+    // Persist via MCP unless the caller opted out.
+    if (options.save !== false) {
+        try {
+            const saved = (await callMcpEndpoint('quality_reports/save', {
+                product_id: productId,
+                commit_sha: outcome.commitSha,
+                rubric_version: outcome.report.rubric_version,
+                branch: options.branch ?? null,
+                repo_root: repoRoot,
+                detected_context: outcome.report.detected_context,
+                tool_versions: outcome.report.tool_versions,
+                unavailable_tools: outcome.report.unavailable_tools,
+                applied_checks: outcome.report.applied_checks,
+                tool_outputs: outcome.report.tool_outputs,
+                external_signals: outcome.report.external_signals,
+                dropped_findings: outcome.report.dropped_findings,
+                dimension_scores: outcome.report.dimension_scores,
+                overall_score: outcome.report.overall_score,
+                overall_grade: outcome.report.overall_grade,
+                executive_summary: outcome.report.executive_summary,
+                low_confidence: outcome.report.low_confidence,
+                status: 'completed',
+                started_at: outcome.startedAt,
+                completed_at: outcome.completedAt,
+                duration_seconds: outcome.durationSeconds,
+                replace_existing: options.force === true,
+            }));
+            const row = JSON.parse(saved.content?.[0]?.text ?? '{}');
+            if (row.id) {
+                logSuccess(`Saved to quality_reports.id = ${row.id}`);
+            }
+        }
+        catch (err) {
+            logWarning(`Failed to persist report via MCP (will keep local file): ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    logInfo(`Overall: grade ${outcome.report.overall_grade ?? '?'} (${outcome.report.overall_score ?? '?'})`);
+    if (outcome.report.executive_summary) {
+        logInfo(outcome.report.executive_summary);
+    }
+}

package/dist/index.js

@@ -24,6 +24,7 @@ import { runIntelligence } from './commands/intelligence/index.js';
 import { runIssueAnalysisCommand } from './commands/issue-analysis/index.js';
 import { runPRResolve } from './commands/pr-resolve/index.js';
 import { runPRReview } from './commands/pr-review/index.js';
+import { runQualityBenchmarkCli } from './commands/quality-benchmark/index.js';
 import { runRefactor } from './commands/refactor/refactor.js';
 import { runReleaseSyncCommand } from './commands/release-sync/index.js';
 import { runRunSheetCommand } from './commands/run-sheet/index.js';
@@ -486,6 +487,29 @@ program
     }
 });
 // ============================================================
+// Subcommand: edsger quality-benchmark <productId>
+// ============================================================
+program
+    .command('quality-benchmark <productId>')
+    .description('Run an industrial-grade code quality benchmark against a local repo')
+    .requiredOption('--repo <path>', 'Path to the local repo checkout')
+    .option('--branch <name>', 'Branch name (informational; does not checkout)')
+    .option('--pkg-manager <name>', 'npm | pnpm | yarn (auto-detected if absent)')
+    .option('--no-install', 'Refuse to install missing tools; mark them unmeasured')
+    .option('--output <path>', 'Where to write the JSON report (default: ./quality-report-<commit>.json)')
+    .option('--no-save', 'Skip the quality_reports/save MCP call (local-only run)')
+    .option('--force', 'Overwrite any existing report row for this (product, commit, rubric)')
+    .option('-v, --verbose', 'Print every progress event')
+    .action(async (productId, opts) => {
+    try {
+        await runQualityBenchmarkCli(productId, opts);
+    }
+    catch (error) {
+        logError(error instanceof Error ? error.message : String(error));
+        process.exit(1);
+    }
+});
+// ============================================================
 // Subcommand: edsger sync-github-issues <productId>
 // ============================================================
 program

package/dist/phases/quality-benchmark/index.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Quality benchmark phase entry point.
+ *
+ * Drives a single Claude Agent SDK session whose system prompt is the
+ * full rubric (loaded by `prompts.ts`) and whose tool surface is the
+ * in-process MCP server in `mcp-server.ts`. The session walks the
+ * rubric's 6-phase pipeline by calling MCP tools; we observe the
+ * messages for progress events and capture the final JSON report.
+ *
+ * Pure compute boundary: this function does NOT touch the database.
+ * Callers (the CLI command, the desktop-app IPC bridge) decide how to
+ * persist the returned payload — see the CLI command in
+ * `src/commands/quality-benchmark/index.ts` for the MCP-mediated
+ * persistence path.
+ *
+ * Side effects that DO happen here:
+ *   - Creates a scratch dir under ~/.edsger/quality-runs/<run-id>/
+ *   - Writes raw tool outputs to that dir
+ *   - Spawns child_process invocations of the catalog tools
+ */
+import { type ProgressEvent } from './mcp-server.js';
+import { type QualityReportPayload, type RunPhase } from './types.js';
+export interface QualityBenchmarkOptions {
+    productId: string;
+    productName: string;
+    /** Absolute path to the local repo checkout. */
+    repoRoot: string;
+    /** Branch name (informational; doesn't checkout). */
+    branch?: string;
+    /**
+     * Detected JS package manager (informational; the SDK will detect on its
+     * own but we pass through for catalog filtering).
+     */
+    packageManager?: string;
+    /** When false, install_tool always refuses (mirrors --no-install CLI flag). */
+    installEnabled: boolean;
+    /**
+     * Optional progress callback for the calling surface (CLI logger,
+     * Electron IPC, etc.). Receives every MCP-server-emitted event.
+     */
+    onProgress?: (event: ProgressEvent) => void;
+    verbose?: boolean;
+    /** Override scratch-dir base (mainly for tests). */
+    scanDirBase?: string;
+    /** Override commit sha resolution (mainly for tests). */
+    commitShaOverride?: string;
+}
+export type QualityBenchmarkOutcome = {
+    status: 'success';
+    report: QualityReportPayload;
+    runId: string;
+    commitSha: string;
+    startedAt: string;
+    completedAt: string;
+    durationSeconds: number;
+} | {
+    status: 'error';
+    message: string;
+    runId?: string;
+    commitSha?: string;
+    lastPhase?: RunPhase;
+};
+export declare function runQualityBenchmark(opts: QualityBenchmarkOptions): Promise<QualityBenchmarkOutcome>;
+export type { ProgressEvent } from './mcp-server.js';
+export type { QualityReportPayload, RunPhase } from './types.js';

package/dist/phases/quality-benchmark/index.js

@@ -0,0 +1,194 @@
+/**
+ * Quality benchmark phase entry point.
+ *
+ * Drives a single Claude Agent SDK session whose system prompt is the
+ * full rubric (loaded by `prompts.ts`) and whose tool surface is the
+ * in-process MCP server in `mcp-server.ts`. The session walks the
+ * rubric's 6-phase pipeline by calling MCP tools; we observe the
+ * messages for progress events and capture the final JSON report.
+ *
+ * Pure compute boundary: this function does NOT touch the database.
+ * Callers (the CLI command, the desktop-app IPC bridge) decide how to
+ * persist the returned payload — see the CLI command in
+ * `src/commands/quality-benchmark/index.ts` for the MCP-mediated
+ * persistence path.
+ *
+ * Side effects that DO happen here:
+ *   - Creates a scratch dir under ~/.edsger/quality-runs/<run-id>/
+ *   - Writes raw tool outputs to that dir
+ *   - Spawns child_process invocations of the catalog tools
+ */
+import { spawnSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { DEFAULT_MODEL } from '../../constants.js';
+import { logError, logInfo, logSuccess } from '../../utils/logger.js';
+import { createPromptGenerator } from '../pr-shared/agent-utils.js';
+import { createEmptyRunState, createQualityBenchmarkMcpServer, } from './mcp-server.js';
+import { createQualityBenchmarkSystemPrompt, createQualityBenchmarkUserPrompt, extractReportJson, } from './prompts.js';
+import { createRunnerContext } from './tool-runner.js';
+import { RUBRIC_VERSION, } from './types.js';
+// SDK upper bound. Each phase usually takes ~1-3 turns; with up to 30
+// tools probed + run plus per-finding verification, 500 gives headroom while
+// still cutting off runaway loops.
+const MAX_TURNS = 500;
+// ---------------------------------------------------------------------------
+// Orchestration
+// ---------------------------------------------------------------------------
+// eslint-disable-next-line complexity
+export async function runQualityBenchmark(opts) {
+    const { productId, productName, repoRoot, branch, packageManager, installEnabled, onProgress, verbose, scanDirBase, commitShaOverride, } = opts;
+    // 1. Determine commit sha from the local checkout.
+    const commitSha = commitShaOverride ?? readGitHead(repoRoot);
+    if (!commitSha) {
+        return {
+            status: 'error',
+            message: `Could not read git HEAD at ${repoRoot} — is this a git repo?`,
+        };
+    }
+    // 2. Set up runner + MCP server.
+    const runId = randomUUID();
+    const startedAt = new Date().toISOString();
+    const runner = createRunnerContext({
+        repo_root: repoRoot,
+        package_manager: packageManager,
+        install_enabled: installEnabled,
+        run_id: runId,
+        base_dir: scanDirBase,
+    });
+    const state = createEmptyRunState();
+    let lastPhase = 'detection';
+    const emit = (event) => {
+        lastPhase = event.phase;
+        onProgress?.(event);
+        if (verbose) {
+            logInfo(`[${event.phase}] ${event.message}`);
+        }
+    };
+    const mcpServer = createQualityBenchmarkMcpServer({ runner, onProgress: emit, installEnabled }, state);
+    // 3. Run the SDK session.
+    const systemPrompt = createQualityBenchmarkSystemPrompt();
+    const userPrompt = createQualityBenchmarkUserPrompt({
+        productName,
+        productId,
+        repoRoot,
+        branch,
+        commitSha,
+        runId,
+        installEnabled,
+    });
+    let lastAssistantText = '';
+    try {
+        for await (const message of query({
+            prompt: createPromptGenerator(userPrompt),
+            options: {
+                systemPrompt: {
+                    type: 'preset',
+                    preset: 'claude_code',
+                    append: systemPrompt,
+                },
+                model: DEFAULT_MODEL,
+                maxTurns: MAX_TURNS,
+                permissionMode: 'bypassPermissions',
+                cwd: repoRoot,
+                mcpServers: {
+                    'quality-benchmark': mcpServer,
+                },
+            },
+        })) {
+            if (message.type === 'assistant' && message.message?.content) {
+                for (const c of message.message.content) {
+                    if (c.type === 'text') {
+                        lastAssistantText += `${c.text}\n`;
+                        if (verbose) {
+                            logInfo(`🤖 ${c.text}`);
+                        }
+                    }
+                }
+            }
+            if (message.type === 'result') {
+                if (message.subtype !== 'success') {
+                    logError(`SDK session ended with subtype=${message.subtype}`);
+                }
+                if (message.subtype === 'success' &&
+                    'result' in message &&
+                    typeof message.result === 'string') {
+                    lastAssistantText = message.result || lastAssistantText;
+                }
+            }
+        }
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        return {
+            status: 'error',
+            message: `SDK error: ${msg}`,
+            runId,
+            commitSha,
+            lastPhase,
+        };
+    }
+    // 4. Extract + validate JSON report.
+    const raw = extractReportJson(lastAssistantText);
+    if (!raw || typeof raw !== 'object') {
+        return {
+            status: 'error',
+            message: 'SDK session did not emit a parseable JSON report',
+            runId,
+            commitSha,
+            lastPhase: 'synthesis',
+        };
+    }
+    const report = raw;
+    // Merge SDK-emitted fields with the state we accumulated server-side. The
+    // server-side state is authoritative for execution metadata; the LLM owns
+    // the scoring fields.
+    const merged = {
+        ...report,
+        rubric_version: RUBRIC_VERSION,
+        tool_versions: { ...state.tool_versions, ...(report.tool_versions ?? {}) },
+        unavailable_tools: dedupUnavailable([
+            ...state.unavailable_tools,
+            ...(report.unavailable_tools ?? []),
+        ]),
+        tool_outputs: { ...state.tool_outputs, ...(report.tool_outputs ?? {}) },
+        dropped_findings: Math.max(state.dropped_findings, report.dropped_findings ?? 0),
+    };
+    const completedAt = new Date().toISOString();
+    const durationSec = Math.round((new Date(completedAt).getTime() - new Date(startedAt).getTime()) / 1000);
+    logSuccess(`Benchmark complete — grade ${merged.overall_grade ?? '?'} (${merged.overall_score ?? '?'})`);
+    return {
+        status: 'success',
+        report: merged,
+        runId,
+        commitSha,
+        startedAt,
+        completedAt,
+        durationSeconds: durationSec,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function readGitHead(repoRoot) {
+    const res = spawnSync('git', ['rev-parse', 'HEAD'], {
+        cwd: repoRoot,
+        encoding: 'utf8',
+    });
+    if (res.status !== 0) {
+        return null;
+    }
+    return res.stdout.trim() || null;
+}
+function dedupUnavailable(list) {
+    const seen = new Set();
+    const out = [];
+    for (const entry of list) {
+        if (seen.has(entry.name)) {
+            continue;
+        }
+        seen.add(entry.name);
+        out.push(entry);
+    }
+    return out;
+}

package/dist/phases/quality-benchmark/mcp-server.d.ts

@@ -0,0 +1,46 @@
+/**
+ * In-process MCP server exposing the quality-benchmark execution
+ * primitives to the Claude Agent SDK session.
+ *
+ * The single SDK session reasons about the rubric in its system prompt
+ * and calls these tools to drive the 6-phase pipeline:
+ *
+ *   list_applicable_tools(detected_context) — Phase 1 -> 2 handoff
+ *   probe_tool(tool_id)                     — Phase 2
+ *   install_tool(tool_id)                   — Phase 2.5
+ *   run_tool(tool_id)                       — Phase 3 (returns parsed summary)
+ *   verify_finding(file, line, snippet?)    — Phase 5
+ *   record_progress(phase, message)         — UI streaming side channel
+ *
+ * Every command and install step is whitelisted by `tool-catalog.ts`
+ * and re-checked at runtime by `tool-runner.ts`. The MCP server is a
+ * thin adapter between the SDK's tool-call protocol and that runner —
+ * it adds no new privilege, no new commands, and no new side effects.
+ */
+import { type RunnerContext } from './tool-runner.js';
+import type { ParsedToolOutput, RunPhase, ToolRunOutput, UnavailableTool } from './types';
+export interface QualityMcpDeps {
+    /** Runner context shared across all tool calls in this benchmark run. */
+    runner: RunnerContext;
+    /** Optional callback for streaming progress updates to the UI / DB. */
+    onProgress?: (event: ProgressEvent) => void;
+    /** When false, install_tool always returns install_disabled. */
+    installEnabled: boolean;
+}
+export interface ProgressEvent {
+    phase: RunPhase;
+    message: string;
+    data?: Record<string, unknown>;
+}
+/** State accumulated across a single benchmark run. */
+export interface RunState {
+    tool_versions: Record<string, string>;
+    unavailable_tools: UnavailableTool[];
+    tool_outputs: Record<string, ToolRunOutput>;
+    /** Cached parsed summaries for synthesis-time read-back. */
+    parsed_summaries: Record<string, ParsedToolOutput>;
+    /** Dropped finding count from Phase 5 verification. */
+    dropped_findings: number;
+}
+export declare function createEmptyRunState(): RunState;
+export declare function createQualityBenchmarkMcpServer(deps: QualityMcpDeps, state: RunState): import("@anthropic-ai/claude-agent-sdk").McpSdkServerConfigWithInstance;