@provos/ironcurtain 0.9.0 → 0.10.0

package/dist/pipeline/pipeline-runner.js

@@ -12,26 +12,33 @@
  */
 import { resolve } from 'node:path';
 import chalk from 'chalk';
+import pLimit from 'p-limit';
 import { extractServerDomainAllowlists } from '../config/index.js';
+import { createLanguageModel } from '../config/model-provider.js';
+import { loadUserConfig } from '../config/user-config.js';
 import { PolicyEngine } from '../trusted-process/policy-engine.js';
 import { buildCompilerSystemPrompt, ConstitutionCompilerSession, formatAnnotationsSummary, validateCompiledRules, } from './constitution-compiler.js';
 import { getHandwrittenScenarios } from './handwritten-scenarios.js';
 import { resolveAllLists } from './list-resolver.js';
-import { computeHash, loadExistingArtifact, loadStoredToolAnnotationsFile, resolveRulePaths, writeArtifact, withSpinner, showCached, createPipelineLlm, } from './pipeline-shared.js';
+import { computeHash, loadExistingArtifact, loadStoredToolAnnotationsFile, resolveRulePaths, writeArtifact, createPipelineLlm, createPerServerModel, createThrottledModel, SpinnerProgressReporter, } from './pipeline-shared.js';
 import { resolveStoredAnnotationsFile } from '../types/argument-roles.js';
 import { applyScenarioCorrections, buildJudgeSystemPrompt, detectAllDefaultRoleFallbacks, extractScenarioCorrections, filterStructuralConflicts, PolicyVerifierSession, verifyPolicy, } from './policy-verifier.js';
 import { filterInvalidSchemaScenarios } from './scenario-schema-validator.js';
 import { buildGeneratorSystemPrompt, generateScenarios, repairScenarios } from './scenario-generator.js';
+import { prefilterServers } from './server-prefilter.js';
 import { connectViaProxy } from './proxy-mcp-connections.js';
 /** Creates PipelineModels from user config. Delegates to shared createPipelineLlm. */
 export async function createPipelineModels(logDir) {
     const effectiveLogDir = logDir ?? resolve(process.cwd(), 'generated');
     const llm = await createPipelineLlm(effectiveLogDir, 'unknown');
+    const userConfig = loadUserConfig();
+    const haikuBaseLlm = await createLanguageModel(userConfig.prefilterModelId, userConfig);
+    const { model: prefilterModel } = createPerServerModel(haikuBaseLlm, llm.logPath, 'prefilter');
     return {
-        compilationModel: llm.model,
+        baseLlm: llm.baseLlm,
         cacheStrategy: llm.cacheStrategy,
-        logContext: llm.logContext,
         logPath: llm.logPath,
+        prefilterModel,
     };
 }
 // ---------------------------------------------------------------------------
@@ -52,7 +59,8 @@ function extractPermittedDirectories(rules) {
 function collectProbeScenarios(result) {
     return result.rounds.flatMap((round) => round.newScenarios);
 }
-function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded scenario (structural conflict)', storedAnnotations) {
+function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded scenario (structural conflict)', storedAnnotations, reporter) {
+    const warn = (msg) => (reporter ? reporter.warn(msg) : console.error(msg));
     const filterResult = filterStructuralConflicts(engine, scenarios);
     let valid = filterResult.valid;
     const discarded = filterResult.discarded;
@@ -60,7 +68,7 @@ function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded s
         const prefix = d.scenario.source === 'handwritten'
             ? chalk.yellow('Warning: handwritten scenario conflicts with structural invariant:')
             : chalk.dim(`${label}:`);
-        console.error(`  ${prefix} "${d.scenario.description}" — ${d.rule} always returns ${d.actual}`);
+        warn(`  ${prefix} "${d.scenario.description}" — ${d.rule} always returns ${d.actual}`);
     }
     // Discard scenarios whose arguments don't match any conditional role spec —
     // they fall back to all default roles and test the wrong thing.
@@ -74,7 +82,7 @@ function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded s
                     const w = fallbackWarnings.find((fw) => fw.scenario.description === s.description);
                     const details = w?.details.join('; ') ?? 'unknown conditional mismatch';
                     const discardLabel = 'Discarded (default role fallback)';
-                    console.error(`  ${chalk.dim(`${discardLabel}:`)} "${s.description}" — ${details}`);
+                    warn(`  ${chalk.dim(`${discardLabel}:`)} "${s.description}" — ${details}`);
                     discarded.push({ scenario: s, rule: 'default-role-fallback', actual: 'deny' });
                 }
                 else {
@@ -88,7 +96,7 @@ function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded s
         const schemaResult = filterInvalidSchemaScenarios(valid, storedAnnotations);
         if (schemaResult.discarded.length > 0) {
             for (const d of schemaResult.discarded) {
-                console.error(`  ${chalk.dim('Discarded (schema mismatch):')} "${d.scenario.description}" — ${d.rule}`);
+                warn(`  ${chalk.dim('Discarded (schema mismatch):')} "${d.scenario.description}" — ${d.rule}`);
             }
             discarded.push(...schemaResult.discarded);
             valid = schemaResult.valid;
@@ -96,6 +104,10 @@ function filterAndLogStructuralConflicts(engine, scenarios, label = 'Discarded s
     }
     return { valid, discarded };
 }
+/** Coerce an unknown thrown value into an Error. */
+function toError(err) {
+    return err instanceof Error ? err : new Error(String(err));
+}
 class RuleValidationError extends Error {
     errors;
     constructor(errors) {
@@ -104,11 +116,12 @@ class RuleValidationError extends Error {
         this.name = 'RuleValidationError';
     }
 }
-function validateRulesOrThrow(rules, listDefinitions = []) {
+function validateRulesOrThrow(rules, listDefinitions = [], reporter) {
+    const warn = (msg) => (reporter ? reporter.warn(msg) : console.error(msg));
     const ruleValidation = validateCompiledRules(rules, listDefinitions);
     if (ruleValidation.warnings.length > 0) {
         for (const w of ruleValidation.warnings) {
-            console.error(`  ${chalk.yellow('Warning:')} ${w}`);
+            warn(`  ${chalk.yellow('Warning:')} ${w}`);
         }
     }
     if (!ruleValidation.valid) {
@@ -262,7 +275,7 @@ export function deduplicateListDefinitions(defs) {
  * Rules are concatenated in alphabetical server order for determinism.
  * List definitions are deduplicated by name.
  */
-export function mergeServerResults(results, constitutionHash) {
+export function mergeServerResults(results, constitutionHash, skippedServers) {
     const sortedResults = [...results].sort((a, b) => a.serverName.localeCompare(b.serverName));
     const allRules = sortedResults.flatMap((r) => r.rules);
     const allListDefs = sortedResults.flatMap((r) => r.listDefinitions);
@@ -277,6 +290,12 @@ export function mergeServerResults(results, constitutionHash) {
     if (uniqueListDefs.length > 0) {
         artifact.listDefinitions = uniqueListDefs;
     }
+    // skippedServers is informational metadata — not read by PolicyEngine.
+    // It's intentionally excluded from inputHash; the constitutionHash already
+    // changes when the constitution changes, which drives different skip decisions.
+    if (skippedServers && skippedServers.length > 0) {
+        artifact.skippedServers = skippedServers;
+    }
     return artifact;
 }
 // ---------------------------------------------------------------------------
@@ -291,13 +310,15 @@ export function mergeServerResults(results, constitutionHash) {
  * (broad principles) and 'task-policy' (strict whitelist) modes.
  */
 export class PipelineRunner {
-    model;
+    baseLlm;
     cacheStrategy;
-    logContext;
+    logPath;
+    prefilterModel;
     constructor(models) {
-        this.model = models.compilationModel;
+        this.baseLlm = models.baseLlm;
         this.cacheStrategy = models.cacheStrategy;
-        this.logContext = models.logContext;
+        this.prefilterModel = models.prefilterModel;
+        this.logPath = models.logPath;
     }
     /**
      * Runs the full pipeline. Returns the compiled policy on success.
@@ -330,10 +351,10 @@ export class PipelineRunner {
         // Resolve conditional role specs to flat annotations for compiler prompts
         const toolAnnotationsFile = resolveStoredAnnotationsFile(storedAnnotationsFile);
         const constitutionHash = computeHash(config.constitutionInput);
-        // Phase 1: Per-server compilation
-        const serverResults = await this.compileAllServers(config, toolAnnotationsFile, constitutionHash, storedAnnotationsFile);
+        // Phase 1: Per-server compilation (with optional Haiku pre-filter)
+        const { results: serverResults, skippedServers } = await this.compileAllServers(config, toolAnnotationsFile, constitutionHash, storedAnnotationsFile);
         // Phase 2: Merge
-        const mergedPolicy = mergeServerResults(serverResults, constitutionHash);
+        const mergedPolicy = mergeServerResults(serverResults, constitutionHash, skippedServers);
         writeArtifact(config.outputDir, 'compiled-policy.json', mergedPolicy);
         // Merge scenarios for the global artifact
         const allScenarios = serverResults.flatMap((r) => r.scenarios);
@@ -391,11 +412,11 @@ export class PipelineRunner {
         return mergedPolicy;
     }
     /**
-     * Compiles rules for all servers sequentially.
-     * Respects serverFilter if provided (for debugging single servers).
+     * Compiles rules for all servers, in parallel when multiple servers are present.
+     * Single-server compilation uses sequential spinner-based output.
+     * Multi-server compilation uses parallel execution with a multi-line progress display.
      */
     async compileAllServers(config, toolAnnotationsFile, constitutionHash, storedAnnotationsFile) {
-        const results = [];
         const serverEntries = Object.entries(toolAnnotationsFile.servers);
         // Apply server filter if provided
         const { serverFilter } = config;
@@ -406,35 +427,126 @@ export class PipelineRunner {
             throw new Error(`No matching servers found for filter: ${config.serverFilter.join(', ')}. ` +
                 `Available: ${serverEntries.map(([n]) => n).join(', ')}`);
         }
+        // Pre-filter with Haiku: skip servers irrelevant to the input text
+        let entriesToCompile = filteredEntries;
+        const skippedServers = [];
+        if (config.prefilterText !== undefined && !config.serverFilter) {
+            const prefilterText = config.prefilterText;
+            if (prefilterText.trim() === '') {
+                console.error(chalk.yellow('Pre-filter input is empty. All servers skipped (default-deny applies).'));
+                const allSkipped = filteredEntries.map(([name]) => ({
+                    serverName: name,
+                    reason: 'No input text provided — all servers skipped by default',
+                }));
+                return { results: [], skippedServers: allSkipped };
+            }
+            const serverToolPairs = filteredEntries.map(([name]) => [name, toolAnnotationsFile.servers[name].tools]);
+            const decisions = await prefilterServers(prefilterText, serverToolPairs, this.prefilterModel, config.constitutionKind);
+            const decisionMap = new Map(decisions.map((d) => [d.serverName, d]));
+            entriesToCompile = filteredEntries.filter(([name]) => {
+                const decision = decisionMap.get(name);
+                if (decision?.skip) {
+                    console.error(`  ${chalk.dim(name)}: ${chalk.yellow('skipped')} — ${decision.reason}`);
+                    skippedServers.push({ serverName: name, reason: decision.reason });
+                    return false;
+                }
+                return true;
+            });
+            const skippedCount = filteredEntries.length - entriesToCompile.length;
+            if (skippedCount > 0) {
+                console.error(`  Pre-filter: ${skippedCount} server(s) skipped, ${entriesToCompile.length} proceeding`);
+            }
+        }
+        if (entriesToCompile.length === 0) {
+            console.error(chalk.yellow('All servers skipped by pre-filter. Policy will use default-deny only.'));
+            return { results: [], skippedServers };
+        }
         // Load handwritten scenarios once, then filter per server (avoids N redundant loads)
         const includeHandwritten = config.includeHandwrittenScenarios ?? config.constitutionKind === 'constitution';
         const allHandwrittenScenarios = includeHandwritten ? getHandwrittenScenarios(config.allowedDirectory) : [];
+        // Create concurrency controls
+        const llmSemaphore = pLimit(8);
+        const serverLimit = pLimit(10);
+        const buildUnit = (serverName) => ({
+            serverName,
+            annotations: toolAnnotationsFile.servers[serverName].tools,
+            storedAnnotations: storedAnnotationsFile.servers[serverName].tools,
+            constitutionText: config.constitutionInput,
+            constitutionKind: config.constitutionKind,
+            allowedDirectory: config.allowedDirectory,
+            protectedPaths: config.protectedPaths,
+            mcpServerConfig: config.mcpServers?.[serverName],
+            handwrittenScenarios: allHandwrittenScenarios.filter((s) => s.request.serverName === serverName),
+        });
+        const useParallel = entriesToCompile.length > 1;
+        let results;
+        if (useParallel) {
+            results = await this.compileServersParallel(entriesToCompile, config, constitutionHash, buildUnit, llmSemaphore, serverLimit);
+        }
+        else {
+            results = await this.compileServersSequential(entriesToCompile, config, constitutionHash, buildUnit, llmSemaphore);
+        }
+        return { results, skippedServers };
+    }
+    /** Creates a per-server model wrapped with LLM call throttling. */
+    createServerModel(serverName, llmSemaphore) {
+        const { model, logContext } = createPerServerModel(this.baseLlm, this.logPath, serverName);
+        return { model: createThrottledModel(model, llmSemaphore), logContext };
+    }
+    async compileServersSequential(filteredEntries, config, constitutionHash, buildUnit, llmSemaphore) {
+        const results = [];
         const totalServers = filteredEntries.length;
         const failedServers = [];
         for (let i = 0; i < filteredEntries.length; i++) {
-            const [serverName, serverData] = filteredEntries[i];
+            const [serverName] = filteredEntries[i];
             console.error('');
             console.error(chalk.bold(`[${i + 1}/${totalServers}] Compiling server: ${serverName}`));
+            const { model, logContext } = this.createServerModel(serverName, llmSemaphore);
+            const reporter = new SpinnerProgressReporter(serverName);
             try {
-                const result = await this.compileServer({
-                    serverName,
-                    annotations: serverData.tools,
-                    storedAnnotations: storedAnnotationsFile.servers[serverName].tools,
-                    constitutionText: config.constitutionInput,
-                    constitutionKind: config.constitutionKind,
-                    allowedDirectory: config.allowedDirectory,
-                    protectedPaths: config.protectedPaths,
-                    mcpServerConfig: config.mcpServers?.[serverName],
-                    handwrittenScenarios: allHandwrittenScenarios.filter((s) => s.request.serverName === serverName),
-                }, config, constitutionHash);
+                const result = await this.compileServer(buildUnit(serverName), config, constitutionHash, model, logContext, reporter);
                 results.push(result);
             }
             catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                console.error(`  ${chalk.red(`Server "${serverName}" failed:`)} ${msg}`);
+                const coerced = toError(err);
+                reporter.fail('compiling', coerced);
+                console.error(`  ${chalk.red(`Server "${serverName}" failed:`)} ${coerced.message}`);
                 failedServers.push(serverName);
             }
         }
+        return this.handleCompilationResults(results, failedServers);
+    }
+    async compileServersParallel(filteredEntries, config, constitutionHash, buildUnit, llmSemaphore, serverLimit) {
+        // Lazy import to avoid loading parallel-progress.ts when not needed
+        const { ParallelProgressDisplay, ParallelProgressReporter } = await import('./parallel-progress.js');
+        const serverNames = filteredEntries.map(([name]) => name);
+        const display = new ParallelProgressDisplay(serverNames);
+        const settled = await Promise.allSettled(filteredEntries.map(([serverName]) => serverLimit(async () => {
+            const { model, logContext } = this.createServerModel(serverName, llmSemaphore);
+            const reporter = new ParallelProgressReporter(display, serverName);
+            try {
+                return await this.compileServer(buildUnit(serverName), config, constitutionHash, model, logContext, reporter);
+            }
+            catch (err) {
+                reporter.fail('compiling', toError(err));
+                throw err;
+            }
+        })));
+        display.finish();
+        const results = [];
+        const failedServers = [];
+        for (const [i, outcome] of settled.entries()) {
+            if (outcome.status === 'fulfilled') {
+                results.push(outcome.value);
+            }
+            else {
+                const serverName = filteredEntries[i][0];
+                failedServers.push(serverName);
+            }
+        }
+        return this.handleCompilationResults(results, failedServers);
+    }
+    handleCompilationResults(results, failedServers) {
         if (failedServers.length > 0) {
             console.error(`\n${chalk.yellow(`Warning: ${failedServers.length} server(s) failed verification: ${failedServers.join(', ')}`)}`);
             console.error(chalk.yellow('Continuing with successfully compiled servers.'));
@@ -448,8 +560,12 @@ export class PipelineRunner {
     /**
      * Compiles a single server: compile -> generate scenarios -> verify -> repair.
      * Self-contained compile-verify-repair loop per server.
+     *
+     * Each server receives its own model (with per-server logging middleware),
+     * logContext (for step labeling), and reporter (for progress/warnings).
+     * This makes the method safe for concurrent execution.
      */
-    async compileServer(unit, config, constitutionHash) {
+    async compileServer(unit, config, constitutionHash, model, logContext, reporter) {
         const serverOutputDir = resolve(config.outputDir, 'servers', unit.serverName);
         // Build per-server system prompt (only this server's annotations)
         const compilerPrompt = unit.constitutionKind === 'task-policy'
@@ -460,13 +576,12 @@ export class PipelineRunner {
         const existingServerPolicy = loadExistingArtifact(serverOutputDir, 'compiled-policy.json');
         const existingServerScenarios = loadExistingArtifact(serverOutputDir, 'test-scenarios.json');
         if (existingServerPolicy && existingServerPolicy.inputHash === inputHash && existingServerScenarios) {
-            // Verify scenario hash to detect stale scenarios (e.g., changed templates or handwritten scenarios)
             const cachedPermittedDirs = extractPermittedDirectories(resolveRulePaths(existingServerPolicy.rules));
             const cachedDynamicLists = loadExistingArtifact(serverOutputDir, 'dynamic-lists.json');
             const cachedScenarioPrompt = buildGeneratorSystemPrompt(unit.constitutionText, unit.annotations, unit.allowedDirectory, cachedPermittedDirs, cachedDynamicLists, unit.storedAnnotations);
             const cachedScenarioHash = computeScenariosHash(cachedScenarioPrompt, unit.handwrittenScenarios);
             if (existingServerScenarios.inputHash === cachedScenarioHash) {
-                showCached(`  ${unit.serverName}: compilation`);
+                reporter.complete('cached', `  ${unit.serverName}: compilation ${chalk.dim('(cached)')}`, 0);
                 return {
                     serverName: unit.serverName,
                     rules: resolveRulePaths(existingServerPolicy.rules),
@@ -480,20 +595,19 @@ export class PipelineRunner {
         }
         const compilerSystem = this.cacheStrategy.wrapSystemPrompt(compilerPrompt);
         // Step 1: Compile rules for this server
-        this.logContext.stepName = `compile-${unit.serverName}`;
-        const compileResult = await this.compileServerPolicyRules(unit, compilerSystem, inputHash);
+        logContext.stepName = `compile-${unit.serverName}`;
+        const compileResult = await this.compileServerPolicyRules(unit, compilerSystem, inputHash, model, reporter);
         let { rules } = compileResult;
         let { listDefinitions } = compileResult;
-        let compilerSession = compileResult.session;
         // Validate server scoping (debug assertion)
         validateServerScoping(unit.serverName, rules);
         // Resolve dynamic lists for this server (before scenario generation so lists are available)
         let dynamicLists;
         if (listDefinitions.length > 0) {
-            dynamicLists = await this.resolveServerLists(listDefinitions, serverOutputDir, config, `  ${unit.serverName}`);
+            dynamicLists = await this.resolveServerLists(listDefinitions, serverOutputDir, config, `  ${unit.serverName}`, model, logContext, reporter);
         }
         // Build per-server policy artifact for engine construction
-        let serverPolicyFile = buildPolicyArtifact(constitutionHash, rules, listDefinitions, inputHash);
+        const serverPolicyFile = buildPolicyArtifact(constitutionHash, rules, listDefinitions, inputHash);
         // Persist early so intermediate results survive scenario/verification failures
         const writeServerPolicy = () => writeArtifact(serverOutputDir, 'compiled-policy.json', {
             generatedAt: new Date().toISOString(),
@@ -514,143 +628,18 @@ export class PipelineRunner {
             ? extractServerDomainAllowlists({ [unit.serverName]: unit.mcpServerConfig })
             : undefined;
         // Step 2: Generate test scenarios for this server
-        this.logContext.stepName = `scenarios-${unit.serverName}`;
+        logContext.stepName = `scenarios-${unit.serverName}`;
         const permittedDirectories = extractPermittedDirectories(rules);
         const scenarioPrompt = buildGeneratorSystemPrompt(unit.constitutionText, unit.annotations, unit.allowedDirectory, permittedDirectories, dynamicLists, unit.storedAnnotations);
         const scenarioHash = computeScenariosHash(scenarioPrompt, unit.handwrittenScenarios);
-        const scenarioResult = await this.generateTestScenarios(unit.constitutionText, unit.annotations, unit.allowedDirectory, unit.handwrittenScenarios, scenarioHash, existingServerScenarios, `  ${unit.serverName}`, permittedDirectories, dynamicLists, unit.storedAnnotations);
-        // Step 3: Verify compiled rules against scenarios
-        this.logContext.stepName = `verify-${unit.serverName}`;
-        const filterEngine = new PolicyEngine(serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, unit.allowedDirectory, undefined, dynamicLists);
-        const { valid: filteredScenarios, discarded: discardedScenarios } = filterAndLogStructuralConflicts(filterEngine, scenarioResult.scenarios, undefined, unit.storedAnnotations);
-        // Generate replacement scenarios for structurally discarded ones (pre-loop, one-time)
-        if (discardedScenarios.length > 0) {
-            const discardedForRepair = discardedScenarios.map((d) => ({
-                scenario: d.scenario,
-                feedback: `${d.rule} always returns ${d.actual}`,
-            }));
-            this.logContext.stepName = `repair-scenarios-${unit.serverName}`;
-            const replacementScenarios = await repairScenarios(discardedForRepair, unit.constitutionText, unit.annotations, unit.allowedDirectory, this.model, permittedDirectories, dynamicLists, (msg) => console.error(`  ${chalk.dim(msg)}`), unit.storedAnnotations);
-            if (replacementScenarios.length > 0) {
-                // Filter replacements through structural invariants too
-                const { valid: validReplacements } = filterAndLogStructuralConflicts(filterEngine, replacementScenarios, 'Discarded replacement (structural conflict)', unit.storedAnnotations);
-                scenarioResult.scenarios.push(...validReplacements);
-                filteredScenarios.push(...validReplacements);
-                console.error(`  ${chalk.dim(`Repaired ${discardedScenarios.length} discarded scenario(s) → ${validReplacements.length} replacement(s)`)}`);
-            }
-        }
-        const serverToolNames = [...new Set(unit.annotations.map((a) => a.toolName))];
-        const serverNames = [unit.serverName];
-        const serverTools = unit.annotations.map((a) => ({ serverName: a.serverName, toolName: a.toolName }));
-        let verifierSystem = this.cacheStrategy.wrapSystemPrompt(buildJudgeSystemPrompt(unit.constitutionText, serverPolicyFile, unit.protectedPaths, serverTools, dynamicLists, unit.allowedDirectory, unit.storedAnnotations));
-        let verifierSession = new PolicyVerifierSession({
-            system: verifierSystem,
-            model: this.model,
-            serverNames,
-            toolNames: serverToolNames,
-            storedAnnotations: unit.storedAnnotations,
-        });
-        const verifyLabel = `  ${unit.serverName}: Verifying`;
-        const { result: verificationResultInitial } = await withSpinner(verifyLabel, async (spinner) => verifyPolicy(unit.constitutionText, serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, filteredScenarios, this.model, 3, unit.allowedDirectory, (msg) => {
-            spinner.text = `${verifyLabel} — ${msg}`;
-        }, serverDomainAllowlists, dynamicLists, verifierSystem, verifierSession, unit.storedAnnotations), (r, elapsed) => r.pass
-            ? `${verifyLabel}: ${r.rounds.length} round(s) (${elapsed.toFixed(1)}s)`
-            : `${verifyLabel}: completed with failures (${elapsed.toFixed(1)}s)`);
-        let verificationResult = verificationResultInitial;
-        if (!verificationResult.pass) {
-            this.logVerboseFailures(verificationResult);
-        }
-        // Collect probe scenarios
-        const { valid: filteredInitialProbes } = filterAndLogStructuralConflicts(filterEngine, collectProbeScenarios(verificationResult), 'Discarded probe (structural conflict)', unit.storedAnnotations);
-        const accumulatedProbes = filteredInitialProbes;
-        // Compile-verify-repair loop (up to 2 repair attempts)
-        const MAX_REPAIRS = 2;
-        let repairAttempts = 0;
-        let currentFilteredScenarios;
-        if (!verificationResult.pass) {
-            const baseInputHash = inputHash;
-            for (let attempt = 1; attempt <= MAX_REPAIRS; attempt++) {
-                console.error('');
-                const lastRound = verificationResult.rounds[verificationResult.rounds.length - 1];
-                const judgeAnalysis = lastRound?.llmAnalysis ?? verificationResult.summary;
-                const attributedFailures = lastRound?.attributedFailures ?? [];
-                const allScenarios = [...scenarioResult.scenarios, ...accumulatedProbes];
-                const { corrections, handwrittenWarnings } = extractScenarioCorrections(attributedFailures, allScenarios);
-                for (const warning of handwrittenWarnings) {
-                    console.error(`  ${chalk.yellow('Warning:')} ${warning}`);
-                }
-                if (corrections.length > 0) {
-                    scenarioResult.scenarios = applyScenarioCorrections(scenarioResult.scenarios, corrections);
-                    const correctedProbes = applyScenarioCorrections(accumulatedProbes, corrections);
-                    accumulatedProbes.splice(0, accumulatedProbes.length, ...correctedProbes);
-                    console.error(`  ${chalk.dim(`Corrected ${corrections.length} scenario expectation(s)`)}`);
-                }
-                ({ valid: currentFilteredScenarios } = filterAndLogStructuralConflicts(filterEngine, scenarioResult.scenarios, undefined, unit.storedAnnotations));
-                const allRuleBlamedFailures = verificationResult.failedScenarios.filter((f) => {
-                    const attr = attributedFailures.find((a) => a.scenarioDescription === f.scenario.description);
-                    if (!attr || attr.blame.kind === 'rule' || attr.blame.kind === 'both')
-                        return true;
-                    return handwrittenWarnings.some((w) => w.includes(f.scenario.description));
-                });
-                if (allRuleBlamedFailures.length > 0) {
-                    const repairContext = {
-                        failedScenarios: allRuleBlamedFailures,
-                        judgeAnalysis,
-                        attemptNumber: attempt,
-                        existingListDefinitions: listDefinitions.length > 0 ? listDefinitions : undefined,
-                        handwrittenScenarios: unit.handwrittenScenarios.length > 0 ? unit.handwrittenScenarios : undefined,
-                    };
-                    this.logContext.stepName = `repair-compile-${unit.serverName}-${attempt}`;
-                    const repairText = `  ${unit.serverName} repair ${attempt}/${MAX_REPAIRS}: Recompiling`;
-                    const { result: repairResult } = await withSpinner(repairText, async (spinner) => this.compilePolicyRulesWithPointFix(rules, unit.annotations, unit.protectedPaths, baseInputHash, repairContext, compilerSystem, compilerSession, listDefinitions, (msg) => {
-                        spinner.text = `${repairText} — ${msg}`;
-                    }), (r, elapsed) => `${repairText}: ${r.rules.length} rules (${elapsed.toFixed(1)}s)`);
-                    rules = repairResult.rules;
-                    listDefinitions = repairResult.listDefinitions;
-                    compilerSession = repairResult.session;
-                    // Re-validate server scoping after repair
-                    validateServerScoping(unit.serverName, rules);
-                    // Re-resolve dynamic lists if repair changed list definitions
-                    if (listDefinitions.length > 0) {
-                        dynamicLists = await this.resolveServerLists(listDefinitions, serverOutputDir, config, `  ${unit.serverName}`);
-                    }
-                    serverPolicyFile = buildPolicyArtifact(constitutionHash, rules, listDefinitions, inputHash);
-                    // Persist repaired rules immediately
-                    writeServerPolicy();
-                    verifierSystem = this.cacheStrategy.wrapSystemPrompt(buildJudgeSystemPrompt(unit.constitutionText, serverPolicyFile, unit.protectedPaths, serverTools, dynamicLists, unit.allowedDirectory, unit.storedAnnotations));
-                    verifierSession = new PolicyVerifierSession({
-                        system: verifierSystem,
-                        model: this.model,
-                        serverNames,
-                        toolNames: serverToolNames,
-                        storedAnnotations: unit.storedAnnotations,
-                    });
-                }
-                else {
-                    console.error(`  ${chalk.dim('No rule-blamed failures — skipping recompilation')}`);
-                }
-                this.logContext.stepName = `repair-verify-${unit.serverName}-${attempt}`;
-                const scenariosForRepairVerify = [...currentFilteredScenarios, ...accumulatedProbes];
-                const repairVerifyText = `  ${unit.serverName} repair ${attempt}/${MAX_REPAIRS}: Verifying`;
-                const { result: repairVerifyResult } = await withSpinner(repairVerifyText, async (spinner) => verifyPolicy(unit.constitutionText, serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, scenariosForRepairVerify, this.model, 1, unit.allowedDirectory, (msg) => {
-                    spinner.text = `${repairVerifyText} — ${msg}`;
-                }, serverDomainAllowlists, dynamicLists, verifierSystem, verifierSession, unit.storedAnnotations), (r, elapsed) => r.pass
-                    ? `${repairVerifyText}: passed (${elapsed.toFixed(1)}s)`
-                    : `${repairVerifyText}: ${r.failedScenarios.length} failure(s) (${elapsed.toFixed(1)}s)`);
-                verificationResult = repairVerifyResult;
-                if (!verificationResult.pass) {
-                    this.logVerboseFailures(verificationResult);
-                }
-                const { valid: validRepairProbes } = filterAndLogStructuralConflicts(filterEngine, collectProbeScenarios(verificationResult), 'Discarded probe (structural conflict)', unit.storedAnnotations);
-                accumulatedProbes.push(...validRepairProbes);
-                repairAttempts = attempt;
-                if (verificationResult.pass) {
-                    break;
-                }
-            }
-        }
+        const scenarioResult = await this.generateTestScenarios(unit.constitutionText, unit.annotations, unit.allowedDirectory, unit.handwrittenScenarios, scenarioHash, existingServerScenarios, `  ${unit.serverName}`, permittedDirectories, dynamicLists, unit.storedAnnotations, model, reporter);
+        // Step 3: Verify and repair
+        const state = await this.verifyAndRepairServer(unit, config, constitutionHash, inputHash, serverPolicyFile, serverAnnotationsFile, serverDomainAllowlists, dynamicLists, rules, listDefinitions, compileResult.session, scenarioResult, permittedDirectories, compilerSystem, model, logContext, reporter, serverOutputDir, writeServerPolicy);
+        // Sync repaired state back so the writeServerPolicy closure sees updated values
+        rules = state.rules;
+        listDefinitions = state.listDefinitions;
         // Write per-server artifacts (before verification check so they can be inspected on failure)
-        const finalScenarios = [...scenarioResult.scenarios, ...accumulatedProbes];
+        const finalScenarios = [...scenarioResult.scenarios, ...state.accumulatedProbes];
         writeServerPolicy();
         writeArtifact(serverOutputDir, 'test-scenarios.json', {
             generatedAt: new Date().toISOString(),
@@ -658,44 +647,205 @@ export class PipelineRunner {
             inputHash: scenarioResult.inputHash,
             scenarios: finalScenarios,
         });
-        if (!verificationResult.pass) {
+        if (!state.verificationResult.pass) {
             throw new Error(`Verification FAILED for server "${unit.serverName}" — artifacts written for inspection but policy may need review.`);
         }
-        console.error(`  ${chalk.green(unit.serverName)}: ${rules.length} rules, ${finalScenarios.length} scenarios` +
-            (repairAttempts > 0 ? `, ${repairAttempts} repair(s)` : ''));
+        const summary = `${state.rules.length} rules, ${finalScenarios.length} scenarios` +
+            (state.repairAttempts > 0 ? `, ${state.repairAttempts} repair(s)` : '');
+        reporter.done(summary);
         return {
             serverName: unit.serverName,
-            rules,
-            listDefinitions,
+            rules: state.rules,
+            listDefinitions: state.listDefinitions,
             scenarios: finalScenarios,
             inputHash,
             constitutionHash,
-            resolvedLists: dynamicLists,
+            resolvedLists: state.dynamicLists,
+        };
+    }
+    /**
+     * Runs initial verification and the compile-verify-repair loop for a server.
+     * Returns the mutable compilation state after all repair attempts.
+     */
+    async verifyAndRepairServer(unit, config, constitutionHash, inputHash, serverPolicyFile, serverAnnotationsFile, serverDomainAllowlists, dynamicLists, rules, listDefinitions, compilerSession, scenarioResult, permittedDirectories, compilerSystem, model, logContext, reporter, serverOutputDir, writeServerPolicy) {
+        logContext.stepName = `verify-${unit.serverName}`;
+        const filterEngine = new PolicyEngine(serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, unit.allowedDirectory, undefined, dynamicLists);
+        const { filteredScenarios } = await this.filterAndRepairDiscardedScenarios(filterEngine, scenarioResult, unit, permittedDirectories, dynamicLists, model, logContext, reporter);
+        const serverToolNames = [...new Set(unit.annotations.map((a) => a.toolName))];
+        const serverNames = [unit.serverName];
+        const serverTools = unit.annotations.map((a) => ({ serverName: a.serverName, toolName: a.toolName }));
+        const verifierSystem = this.cacheStrategy.wrapSystemPrompt(buildJudgeSystemPrompt(unit.constitutionText, serverPolicyFile, unit.protectedPaths, serverTools, dynamicLists, unit.allowedDirectory, unit.storedAnnotations));
+        const verifierSession = new PolicyVerifierSession({
+            system: verifierSystem,
+            model,
+            serverNames,
+            toolNames: serverToolNames,
+            storedAnnotations: unit.storedAnnotations,
+        });
+        reporter.update('verifying');
+        const verifyStart = Date.now();
+        const verificationResultInitial = await verifyPolicy(unit.constitutionText, serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, filteredScenarios, model, 3, unit.allowedDirectory, (msg) => reporter.update('verifying', msg), serverDomainAllowlists, dynamicLists, verifierSystem, verifierSession, unit.storedAnnotations);
+        const verifyElapsed = (Date.now() - verifyStart) / 1000;
+        const verifyLabel = `  ${unit.serverName}: Verifying`;
+        reporter.complete('verifying', verificationResultInitial.pass
+            ? `${verifyLabel}: ${verificationResultInitial.rounds.length} round(s) (${verifyElapsed.toFixed(1)}s)`
+            : `${verifyLabel}: completed with failures (${verifyElapsed.toFixed(1)}s)`, verifyElapsed);
+        if (!verificationResultInitial.pass) {
+            this.logVerboseFailures(verificationResultInitial, reporter);
+        }
+        // Collect probe scenarios
+        const { valid: filteredInitialProbes } = filterAndLogStructuralConflicts(filterEngine, collectProbeScenarios(verificationResultInitial), 'Discarded probe (structural conflict)', unit.storedAnnotations, reporter);
+        const state = {
+            rules,
+            listDefinitions,
+            compilerSession,
+            dynamicLists,
+            serverPolicyFile,
+            verificationResult: verificationResultInitial,
+            accumulatedProbes: filteredInitialProbes,
+            repairAttempts: 0,
         };
+        if (!state.verificationResult.pass) {
+            await this.runRepairLoop(state, unit, config, constitutionHash, inputHash, serverAnnotationsFile, serverDomainAllowlists, scenarioResult, filterEngine, serverToolNames, serverNames, serverTools, compilerSystem, model, logContext, reporter, serverOutputDir, writeServerPolicy);
+        }
+        return state;
+    }
+    /**
+     * Filters structurally conflicting scenarios and generates replacements
+     * for discarded ones via LLM repair (pre-loop, one-time).
+     */
+    async filterAndRepairDiscardedScenarios(filterEngine, scenarioResult, unit, permittedDirectories, dynamicLists, model, logContext, reporter) {
+        const { valid: filteredScenarios, discarded: discardedScenarios } = filterAndLogStructuralConflicts(filterEngine, scenarioResult.scenarios, undefined, unit.storedAnnotations, reporter);
+        if (discardedScenarios.length > 0) {
+            const discardedForRepair = discardedScenarios.map((d) => ({
+                scenario: d.scenario,
+                feedback: `${d.rule} always returns ${d.actual}`,
+            }));
+            logContext.stepName = `repair-scenarios-${unit.serverName}`;
+            reporter.update('repair-scenarios');
+            const repairStart = Date.now();
+            const replacementScenarios = await repairScenarios(discardedForRepair, unit.constitutionText, unit.annotations, unit.allowedDirectory, model, permittedDirectories, dynamicLists, (msg) => reporter.warn(`  ${chalk.dim(msg)}`), unit.storedAnnotations);
+            if (replacementScenarios.length > 0) {
+                const { valid: validReplacements } = filterAndLogStructuralConflicts(filterEngine, replacementScenarios, 'Discarded replacement (structural conflict)', unit.storedAnnotations, reporter);
+                scenarioResult.scenarios.push(...validReplacements);
+                filteredScenarios.push(...validReplacements);
+                reporter.warn(`  ${chalk.dim(`Repaired ${discardedScenarios.length} discarded scenario(s) → ${validReplacements.length} replacement(s)`)}`);
+            }
+            const repairElapsed = (Date.now() - repairStart) / 1000;
+            reporter.complete('repair-scenarios', `  ${unit.serverName}: Repaired scenarios (${repairElapsed.toFixed(1)}s)`, repairElapsed);
+        }
+        return { filteredScenarios };
+    }
+    /**
+     * Compile-verify-repair loop (up to 2 repair attempts).
+     * Mutates `state` in place with updated rules, lists, and verification results.
+     */
+    async runRepairLoop(state, unit, config, constitutionHash, inputHash, serverAnnotationsFile, serverDomainAllowlists, scenarioResult, filterEngine, serverToolNames, serverNames, serverTools, compilerSystem, model, logContext, reporter, serverOutputDir, writeServerPolicy) {
+        const MAX_REPAIRS = 2;
+        for (let attempt = 1; attempt <= MAX_REPAIRS; attempt++) {
+            const lastRound = state.verificationResult.rounds[state.verificationResult.rounds.length - 1];
+            const judgeAnalysis = lastRound?.llmAnalysis ?? state.verificationResult.summary;
+            const attributedFailures = lastRound?.attributedFailures ?? [];
+            const allScenarios = [...scenarioResult.scenarios, ...state.accumulatedProbes];
+            const { corrections, handwrittenWarnings } = extractScenarioCorrections(attributedFailures, allScenarios);
+            for (const warning of handwrittenWarnings) {
+                reporter.warn(`  ${chalk.yellow('Warning:')} ${warning}`);
+            }
+            if (corrections.length > 0) {
+                scenarioResult.scenarios = applyScenarioCorrections(scenarioResult.scenarios, corrections);
+                const correctedProbes = applyScenarioCorrections(state.accumulatedProbes, corrections);
+                state.accumulatedProbes.splice(0, state.accumulatedProbes.length, ...correctedProbes);
+                reporter.warn(`  ${chalk.dim(`Corrected ${corrections.length} scenario expectation(s)`)}`);
+            }
+            const { valid: currentFilteredScenarios } = filterAndLogStructuralConflicts(filterEngine, scenarioResult.scenarios, undefined, unit.storedAnnotations, reporter);
+            const allRuleBlamedFailures = state.verificationResult.failedScenarios.filter((f) => {
+                const attr = attributedFailures.find((a) => a.scenarioDescription === f.scenario.description);
+                if (!attr || attr.blame.kind === 'rule' || attr.blame.kind === 'both')
+                    return true;
+                return handwrittenWarnings.some((w) => w.includes(f.scenario.description));
+            });
+            if (allRuleBlamedFailures.length > 0) {
+                const repairContext = {
+                    failedScenarios: allRuleBlamedFailures,
+                    judgeAnalysis,
+                    attemptNumber: attempt,
+                    existingListDefinitions: state.listDefinitions.length > 0 ? state.listDefinitions : undefined,
+                    handwrittenScenarios: unit.handwrittenScenarios.length > 0 ? unit.handwrittenScenarios : undefined,
+                };
+                logContext.stepName = `repair-compile-${unit.serverName}-${attempt}`;
+                reporter.update('repair-compile', `attempt ${attempt}/${MAX_REPAIRS}`);
+                const repairCompileStart = Date.now();
+                const repairResult = await this.compilePolicyRulesWithPointFix(state.rules, unit.annotations, unit.protectedPaths, inputHash, repairContext, compilerSystem, state.compilerSession, state.listDefinitions, model, (msg) => reporter.update('repair-compile', msg), reporter);
+                const repairCompileElapsed = (Date.now() - repairCompileStart) / 1000;
+                reporter.complete('repair-compile', `  ${unit.serverName} repair ${attempt}/${MAX_REPAIRS}: ${repairResult.rules.length} rules (${repairCompileElapsed.toFixed(1)}s)`, repairCompileElapsed);
+                state.rules = repairResult.rules;
+                state.listDefinitions = repairResult.listDefinitions;
+                state.compilerSession = repairResult.session;
+                // Re-validate server scoping after repair
+                validateServerScoping(unit.serverName, state.rules);
+                // Re-resolve dynamic lists if repair changed list definitions
+                if (state.listDefinitions.length > 0) {
+                    state.dynamicLists = await this.resolveServerLists(state.listDefinitions, serverOutputDir, config, `  ${unit.serverName}`, model, logContext, reporter);
+                }
+                state.serverPolicyFile = buildPolicyArtifact(constitutionHash, state.rules, state.listDefinitions, inputHash);
+                // Persist repaired rules immediately
+                writeServerPolicy();
+            }
+            else {
+                reporter.warn(`  ${chalk.dim('No rule-blamed failures — skipping recompilation')}`);
+            }
+            // Build verifier from current state (may have been updated by repair above)
+            const verifierSystem = this.cacheStrategy.wrapSystemPrompt(buildJudgeSystemPrompt(unit.constitutionText, state.serverPolicyFile, unit.protectedPaths, serverTools, state.dynamicLists, unit.allowedDirectory, unit.storedAnnotations));
+            const verifierSession = new PolicyVerifierSession({
+                system: verifierSystem,
+                model,
+                serverNames,
+                toolNames: serverToolNames,
+                storedAnnotations: unit.storedAnnotations,
+            });
+            logContext.stepName = `repair-verify-${unit.serverName}-${attempt}`;
+            const scenariosForRepairVerify = [...currentFilteredScenarios, ...state.accumulatedProbes];
+            reporter.update('repair-verify', `attempt ${attempt}/${MAX_REPAIRS}`);
+            const repairVerifyStart = Date.now();
+            const repairVerifyResult = await verifyPolicy(unit.constitutionText, state.serverPolicyFile, serverAnnotationsFile, unit.protectedPaths, scenariosForRepairVerify, model, 1, unit.allowedDirectory, (msg) => reporter.update('repair-verify', msg), serverDomainAllowlists, state.dynamicLists, verifierSystem, verifierSession, unit.storedAnnotations);
+            const repairVerifyElapsed = (Date.now() - repairVerifyStart) / 1000;
+            const repairVerifyText = `  ${unit.serverName} repair ${attempt}/${MAX_REPAIRS}: Verifying`;
+            reporter.complete('repair-verify', repairVerifyResult.pass
+                ? `${repairVerifyText}: passed (${repairVerifyElapsed.toFixed(1)}s)`
+                : `${repairVerifyText}: ${repairVerifyResult.failedScenarios.length} failure(s) (${repairVerifyElapsed.toFixed(1)}s)`, repairVerifyElapsed);
+            state.verificationResult = repairVerifyResult;
+            if (!state.verificationResult.pass) {
+                this.logVerboseFailures(state.verificationResult, reporter);
+            }
+            const { valid: validRepairProbes } = filterAndLogStructuralConflicts(filterEngine, collectProbeScenarios(state.verificationResult), 'Discarded probe (structural conflict)', unit.storedAnnotations, reporter);
+            state.accumulatedProbes.push(...validRepairProbes);
+            state.repairAttempts = attempt;
+            if (state.verificationResult.pass) {
+                break;
+            }
+        }
     }
     /**
      * Compiles policy rules for a single server (initial compilation).
      * Uses per-server schema with requireServer: true.
      */
-    async compileServerPolicyRules(unit, system, inputHash) {
-        const stepText = `  ${unit.serverName}: Compiling rules`;
+    async compileServerPolicyRules(unit, system, inputHash, model, reporter) {
+        reporter.update('compiling');
+        const start = Date.now();
         const session = new ConstitutionCompilerSession({
             system,
-            model: this.model,
+            model,
             annotations: unit.annotations,
             schemaOptions: { requireServer: true },
         });
-        const { result: compilationOutput } = await withSpinner(stepText, async (spinner) => {
-            const output = await session.compile((msg) => {
-                spinner.text = `${stepText} — ${msg}`;
-            });
-            const compiledRules = resolveRulePaths(output.rules);
-            validateRulesOrThrow(compiledRules, output.listDefinitions);
-            return { rules: compiledRules, listDefinitions: output.listDefinitions };
-        }, (output, elapsed) => `${stepText}: ${output.rules.length} rules (${elapsed.toFixed(1)}s)`);
+        const output = await session.compile((msg) => reporter.update('compiling', msg));
+        const compiledRules = resolveRulePaths(output.rules);
+        validateRulesOrThrow(compiledRules, output.listDefinitions, reporter);
+        const elapsed = (Date.now() - start) / 1000;
+        reporter.complete('compiling', `  ${unit.serverName}: Compiling rules: ${compiledRules.length} rules (${elapsed.toFixed(1)}s)`, elapsed);
         return {
-            rules: compilationOutput.rules,
-            listDefinitions: compilationOutput.listDefinitions,
+            rules: compiledRules,
+            listDefinitions: output.listDefinitions,
             inputHash,
             session,
         };
@@ -703,7 +853,7 @@ export class PipelineRunner {
     // -----------------------------------------------------------------------
     // Private compilation methods
     // -----------------------------------------------------------------------
-    async compilePolicyRulesWithRepair(annotations, protectedPaths, baseInputHash, repairContext, system, session, onProgress) {
+    async compilePolicyRulesWithRepair(annotations, protectedPaths, baseInputHash, repairContext, system, session, model, onProgress, reporter) {
         let output;
         if (session) {
             output = await session.recompile(repairContext, onProgress);
@@ -711,10 +861,10 @@ export class PipelineRunner {
         else {
             const { compileConstitution } = await import('./constitution-compiler.js');
             output = await compileConstitution('', // Not needed when system prompt is provided
-            annotations, { protectedPaths }, this.model, repairContext, onProgress, system);
+            annotations, { protectedPaths }, model, repairContext, onProgress, system);
         }
         const compiledRules = resolveRulePaths(output.rules);
-        validateRulesOrThrow(compiledRules, output.listDefinitions);
+        validateRulesOrThrow(compiledRules, output.listDefinitions, reporter);
         return {
             rules: compiledRules,
             listDefinitions: output.listDefinitions,
@@ -728,17 +878,17 @@ export class PipelineRunner {
      * patch instead of regenerating the entire rule set. This avoids oscillation
      * where fixing one failure breaks previously-passing rules.
      */
-    async compilePolicyRulesWithPointFix(existingRules, annotations, protectedPaths, baseInputHash, repairContext, system, session, existingListDefinitions, onProgress) {
+    async compilePolicyRulesWithPointFix(existingRules, annotations, protectedPaths, baseInputHash, repairContext, system, session, existingListDefinitions, model, onProgress, reporter) {
         let output;
         if (session) {
             output = await session.repairPointFix(existingRules, repairContext, existingListDefinitions, onProgress);
         }
         else {
             // No session available -- fall back to full recompile
-            return this.compilePolicyRulesWithRepair(annotations, protectedPaths, baseInputHash, repairContext, system, session, onProgress);
+            return this.compilePolicyRulesWithRepair(annotations, protectedPaths, baseInputHash, repairContext, system, session, model, onProgress, reporter);
         }
         const compiledRules = resolveRulePaths(output.rules);
-        validateRulesOrThrow(compiledRules, output.listDefinitions);
+        validateRulesOrThrow(compiledRules, output.listDefinitions, reporter);
         return {
             rules: compiledRules,
             listDefinitions: output.listDefinitions,
@@ -751,25 +901,24 @@ export class PipelineRunner {
      * Loads existing per-server dynamic-lists.json for cache comparison,
      * connects MCP servers if needed, and writes results to serverOutputDir.
      */
-    async resolveServerLists(listDefinitions, serverOutputDir, config, labelPrefix) {
-        this.logContext.stepName = `resolve-lists-${labelPrefix.trim()}`;
+    async resolveServerLists(listDefinitions, serverOutputDir, config, labelPrefix, model, logContext, reporter) {
+        logContext.stepName = `resolve-lists-${labelPrefix.trim()}`;
         const existingLists = loadExistingArtifact(serverOutputDir, 'dynamic-lists.json');
         const needsMcp = listDefinitions.some((d) => d.requiresMcp);
         let proxy;
         if (needsMcp && config.mcpServers) {
             proxy = await connectViaProxy(listDefinitions, config.mcpServers, config.toolAnnotationsDir);
         }
-        const listStepText = `${labelPrefix}: Resolving dynamic lists`;
+        reporter.update('lists');
+        const start = Date.now();
         try {
-            const { result: resolvedLists } = await withSpinner(listStepText, async (spinner) => resolveAllLists(listDefinitions, {
-                model: this.model,
+            const resolvedLists = await resolveAllLists(listDefinitions, {
+                model,
                 proxyConnection: proxy ? { client: proxy.client, tools: proxy.tools } : undefined,
-            }, existingLists, (msg) => {
-                spinner.text = `${listStepText} — ${msg}`;
-            }), (result, elapsed) => {
-                const count = Object.keys(result.lists).length;
-                return `${listStepText}: ${count} list(s) resolved (${elapsed.toFixed(1)}s)`;
-            });
+            }, existingLists, (msg) => reporter.update('lists', msg));
+            const elapsed = (Date.now() - start) / 1000;
+            const count = Object.keys(resolvedLists.lists).length;
+            reporter.complete('lists', `${labelPrefix}: Resolving dynamic lists: ${count} list(s) resolved (${elapsed.toFixed(1)}s)`, elapsed);
             writeArtifact(serverOutputDir, 'dynamic-lists.json', resolvedLists);
             return resolvedLists;
         }
@@ -779,28 +928,28 @@ export class PipelineRunner {
             }
         }
     }
-    async generateTestScenarios(constitutionText, annotations, allowedDirectory, handwrittenScenarios, inputHash, existingScenarios, stepLabel, permittedDirectories, dynamicLists, storedAnnotations) {
+    async generateTestScenarios(constitutionText, annotations, allowedDirectory, handwrittenScenarios, inputHash, existingScenarios, stepLabel, permittedDirectories, dynamicLists, storedAnnotations, model, reporter) {
         const stepText = `${stepLabel} Generating test scenarios`;
         if (existingScenarios && existingScenarios.inputHash === inputHash) {
-            showCached(stepText);
+            reporter.complete('scenarios', `${stepText} ${chalk.dim('(cached)')}`, 0);
             return { scenarios: existingScenarios.scenarios, inputHash };
         }
-        const { result: scenarios } = await withSpinner(stepText, async (spinner) => generateScenarios(constitutionText, annotations, handwrittenScenarios, allowedDirectory, this.model, permittedDirectories, (msg) => {
-            spinner.text = `${stepText} — ${msg}`;
-        }, dynamicLists, (prompt) => this.cacheStrategy.wrapSystemPrompt(prompt), storedAnnotations), (scenarios, elapsed) => {
-            const generatedCount = scenarios.length - handwrittenScenarios.length;
-            return `${stepText}: ${scenarios.length} scenarios (${handwrittenScenarios.length} handwritten + ${generatedCount} generated) (${elapsed.toFixed(1)}s)`;
-        });
+        reporter.update('scenarios');
+        const start = Date.now();
+        const scenarios = await generateScenarios(constitutionText, annotations, handwrittenScenarios, allowedDirectory, model, permittedDirectories, (msg) => reporter.update('scenarios', msg), dynamicLists, (prompt) => this.cacheStrategy.wrapSystemPrompt(prompt), storedAnnotations);
+        const elapsed = (Date.now() - start) / 1000;
+        const generatedCount = scenarios.length - handwrittenScenarios.length;
+        reporter.complete('scenarios', `${stepText}: ${scenarios.length} scenarios (${handwrittenScenarios.length} handwritten + ${generatedCount} generated) (${elapsed.toFixed(1)}s)`, elapsed);
         return { scenarios, inputHash };
     }
-    logVerboseFailures(result) {
-        console.error('');
-        console.error(chalk.red('Verification FAILED:'));
-        console.error(result.summary);
-        console.error('');
+    logVerboseFailures(result, reporter) {
+        reporter.warn('');
+        reporter.warn(chalk.red('Verification FAILED:'));
+        reporter.warn(result.summary);
+        reporter.warn('');
         for (const f of result.failedScenarios) {
-            console.error(`  ${chalk.red('FAIL:')} ${f.scenario.description}`);
-            console.error(`    Expected: ${f.scenario.expectedDecision}, Got: ${f.actualDecision} (rule: ${f.matchingRule})`);
+            reporter.warn(`  ${chalk.red('FAIL:')} ${f.scenario.description}`);
+            reporter.warn(`    Expected: ${f.scenario.expectedDecision}, Got: ${f.actualDecision} (rule: ${f.matchingRule})`);
         }
     }
 }