@navels/neal 0.1.0

package/dist/neal/commands/compat.js

@@ -0,0 +1,807 @@
+import { spawnSync } from 'node:child_process';
+import { cpSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, relative, resolve } from 'node:path';
+import process from 'node:process';
+import { Writable } from 'node:stream';
+import { fileURLToPath } from 'node:url';
+import { DEFAULT_REVIEW_INSTRUCTION } from '../cli.js';
+import { assertWriterProvidersConfigured } from '../config.js';
+import { createRunId } from '../logger.js';
+import { loadOrInitialize } from '../orchestrator.js';
+import { validatePlanDocument } from '../plan-validation.js';
+import { getReviewFindingsArtifactPaths } from '../review-findings/artifacts.js';
+import { createAgentReviewFindingsProviderAdapter } from '../review-findings/provider.js';
+import { runNealReviewCli } from '../review-findings/run.js';
+import { isNealProviderError } from '../providers/types.js';
+import { getRunDir } from '../storage-paths.js';
+import { verifyConfiguredProviders } from './check.js';
+import { executeRun } from './runtime.js';
+// Native adapters drive their providers' own CLIs/SDKs directly; every other
+// provider id is "generic-agentic" and should be qualified via `neal compat`.
+export const NATIVE_PROVIDER_IDS = new Set([
+    'openai-codex',
+    'anthropic-claude',
+]);
+export function isGenericAgenticProvider(provider) {
+    return !NATIVE_PROVIDER_IDS.has(provider);
+}
+// Default model per native reference provider. `neal compat` routes the
+// non-candidate roles onto a native adapter; these are the known-good models
+// each adapter drives (effort omitted → native provider default).
+export const REFERENCE_DEFAULT_MODELS = {
+    'openai-codex': 'gpt-5.5',
+    'anthropic-claude': 'claude-opus-4-8',
+};
+// A compat reference is either a native provider id (run at its built-in default model)
+// or `generic-agentic:<openrouter-slug>` to use a *validated* OpenRouter model as the
+// known-good partner. The latter is the follow-up the routing plan anticipated once a
+// rock-solid OpenRouter reference was identified: it removes the codex reference and so
+// uncaps parallelism (no shared native-provider capacity pool). Use ONLY a model that
+// reliably passes all roles — a flaky partner makes a candidate FAIL unattributable.
+export function resolveReference(reference) {
+    const id = reference ?? 'openai-codex';
+    const colon = id.indexOf(':');
+    if (colon === -1) {
+        return {
+            provider: id,
+            model: REFERENCE_DEFAULT_MODELS[id] ?? null,
+        };
+    }
+    return { provider: id.slice(0, colon), model: id.slice(colon + 1) };
+}
+// ---------------------------------------------------------------------------
+// Manifest loading
+// ---------------------------------------------------------------------------
+export function getCompatExamplesDir() {
+    // src/neal/commands/compat.ts -> ../../../examples/compat (repo/package root).
+    return fileURLToPath(new URL('../../../examples/compat/', import.meta.url));
+}
+function asStringArray(value, context) {
+    if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {
+        throw new Error(`compat manifest: ${context} must be an array of strings`);
+    }
+    return value;
+}
+function validateFixture(raw, index) {
+    if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
+        throw new Error(`compat manifest: fixtures[${index}] must be an object`);
+    }
+    const value = raw;
+    if (typeof value.id !== 'string' || value.id.trim() === '') {
+        throw new Error(`compat manifest: fixtures[${index}].id must be a non-empty string`);
+    }
+    const roles = asStringArray(value.roles, `fixtures[${index}].roles`);
+    for (const role of roles) {
+        if (role !== 'coder' && role !== 'reviewer' && role !== 'planner') {
+            throw new Error(`compat manifest: ${value.id} has unsupported role ${JSON.stringify(role)}`);
+        }
+    }
+    if (typeof value.projectDir !== 'string' || value.projectDir.trim() === '') {
+        throw new Error(`compat manifest: ${value.id} must define a projectDir`);
+    }
+    const referenceFix = value.referenceFix;
+    if (!referenceFix ||
+        typeof referenceFix.file !== 'string' ||
+        typeof referenceFix.from !== 'string' ||
+        typeof referenceFix.to !== 'string') {
+        throw new Error(`compat manifest: ${value.id} must define a referenceFix { file, from, to }`);
+    }
+    const fixture = {
+        id: value.id,
+        roles: roles,
+        projectDir: value.projectDir,
+        referenceFix: { file: referenceFix.file, from: referenceFix.from, to: referenceFix.to },
+    };
+    if (value.planDoc !== undefined) {
+        if (typeof value.planDoc !== 'string') {
+            throw new Error(`compat manifest: ${value.id}.planDoc must be a string`);
+        }
+        fixture.planDoc = value.planDoc;
+    }
+    if (value.issuePrompt !== undefined) {
+        if (typeof value.issuePrompt !== 'string') {
+            throw new Error(`compat manifest: ${value.id}.issuePrompt must be a string`);
+        }
+        fixture.issuePrompt = value.issuePrompt;
+    }
+    if (value.verifyCommand !== undefined) {
+        if (typeof value.verifyCommand !== 'string') {
+            throw new Error(`compat manifest: ${value.id}.verifyCommand must be a string`);
+        }
+        fixture.verifyCommand = value.verifyCommand;
+    }
+    if (value.reviewer !== undefined) {
+        const reviewer = value.reviewer;
+        if (typeof reviewer.goodDiff !== 'string' || typeof reviewer.brokenDiff !== 'string') {
+            throw new Error(`compat manifest: ${value.id}.reviewer must define goodDiff and brokenDiff`);
+        }
+        fixture.reviewer = { goodDiff: reviewer.goodDiff, brokenDiff: reviewer.brokenDiff };
+    }
+    // Contract checks mirrored from the bundled-fixture structural test
+    // (test/compat-fixtures.test.ts).
+    if (fixture.roles.includes('coder') && !fixture.planDoc) {
+        throw new Error(`compat manifest: ${value.id} carries the coder role but defines no planDoc`);
+    }
+    if (fixture.roles.includes('planner') && !fixture.issuePrompt) {
+        throw new Error(`compat manifest: ${value.id} carries the planner role but defines no issuePrompt`);
+    }
+    if (fixture.roles.includes('reviewer')) {
+        if (!fixture.reviewer) {
+            throw new Error(`compat manifest: ${value.id} carries the reviewer role but defines no diffs`);
+        }
+        if (!fixture.verifyCommand) {
+            throw new Error(`compat manifest: ${value.id} carries the reviewer role but defines no verifyCommand`);
+        }
+    }
+    if ((fixture.roles.includes('coder') || fixture.roles.includes('reviewer')) && !fixture.verifyCommand) {
+        throw new Error(`compat manifest: ${value.id} must define a verifyCommand`);
+    }
+    return fixture;
+}
+export function loadCompatManifest(compatDir = getCompatExamplesDir()) {
+    const manifestPath = join(compatDir, 'manifest.json');
+    const parsed = JSON.parse(readFileSync(manifestPath, 'utf8'));
+    if (parsed === null || typeof parsed !== 'object' || !Array.isArray(parsed.fixtures)) {
+        throw new Error('compat manifest: top-level object must define a fixtures array');
+    }
+    const rawFixtures = parsed.fixtures;
+    if (rawFixtures.length === 0) {
+        throw new Error('compat manifest: must define at least one fixture');
+    }
+    return { fixtures: rawFixtures.map((fixture, index) => validateFixture(fixture, index)) };
+}
+// ---------------------------------------------------------------------------
+// Argument parsing
+// ---------------------------------------------------------------------------
+export function parseCompatArgs(args) {
+    if (args[0] !== 'compat') {
+        throw new Error(`Unknown argument: ${args[0] ?? ''}`);
+    }
+    let model = null;
+    let role = 'all';
+    let reference = null;
+    let json = false;
+    let index = 1;
+    function requireValue(flag) {
+        const value = args[index + 1];
+        if (value === undefined || value.startsWith('--') || value.trim() === '') {
+            throw new Error(`neal compat ${flag} requires a non-empty value`);
+        }
+        return value.trim();
+    }
+    while (index < args.length) {
+        const flag = args[index];
+        switch (flag) {
+            case '--model':
+                model = requireValue('--model');
+                index += 2;
+                break;
+            case '--reference': {
+                const value = requireValue('--reference');
+                const colon = value.indexOf(':');
+                if (colon === -1) {
+                    if (!NATIVE_PROVIDER_IDS.has(value)) {
+                        throw new Error('neal compat --reference must be a native provider id (openai-codex or anthropic-claude) or generic-agentic:<openrouter-model>');
+                    }
+                }
+                else if (value.slice(0, colon) !== 'generic-agentic' || value.slice(colon + 1).trim() === '') {
+                    throw new Error('neal compat --reference provider:model form must be generic-agentic:<openrouter-model>');
+                }
+                reference = value;
+                index += 2;
+                break;
+            }
+            case '--role': {
+                const value = requireValue('--role');
+                if (value !== 'coder' && value !== 'reviewer' && value !== 'planner' && value !== 'all') {
+                    throw new Error('neal compat --role must be one of: coder, reviewer, planner, all');
+                }
+                role = value;
+                index += 2;
+                break;
+            }
+            case '--json':
+                json = true;
+                index += 1;
+                break;
+            default:
+                throw new Error(`Unknown argument: ${flag}`);
+        }
+    }
+    return { model, role, reference, json };
+}
+// ---------------------------------------------------------------------------
+// Candidate config derivation
+// ---------------------------------------------------------------------------
+export function deriveCandidateConfig(base, options) {
+    const next = {
+        planner: { ...base.planner },
+        coder: { ...base.coder },
+        reviewer: { ...base.reviewer },
+    };
+    const routingActive = options.model !== null || options.reference !== null;
+    const reference = resolveReference(options.reference);
+    // Tested role: when a candidate slug is given, force the generic-agentic
+    // provider and drop any configured effort so the slug drives a clean
+    // OpenRouter call. Otherwise leave the configured provider/model untouched.
+    if (options.model !== null) {
+        next[options.testedRole] = { provider: 'generic-agentic', model: options.model };
+    }
+    // Non-tested roles: when either flag is given, route them onto the reference
+    // (a native adapter at its default model, or generic-agentic:<slug> for a validated
+    // OpenRouter partner). When neither flag is given, leave every role as configured
+    // (pure pass-through).
+    if (routingActive) {
+        for (const role of ['planner', 'coder', 'reviewer']) {
+            if (role !== options.testedRole) {
+                next[role] = { provider: reference.provider, model: reference.model };
+            }
+        }
+    }
+    return next;
+}
+export function rolesForSelection(role) {
+    if (role === 'all') {
+        return ['coder', 'reviewer', 'planner'];
+    }
+    return [role];
+}
+// ---------------------------------------------------------------------------
+// Throwaway worktree helpers
+// ---------------------------------------------------------------------------
+function gitInThrowaway(cwd, args) {
+    const result = spawnSync('git', args, { cwd, encoding: 'utf8' });
+    if (result.error) {
+        throw result.error;
+    }
+    return { status: result.status ?? 1, stderr: result.stderr ?? '' };
+}
+function initThrowawayRepo(cwd) {
+    const steps = [
+        ['init', '-q'],
+        ['config', 'user.name', 'Neal Compat'],
+        ['config', 'user.email', 'compat@neal.local'],
+        ['config', 'commit.gpgsign', 'false'],
+        ['add', '-A'],
+        ['commit', '-q', '-m', 'base'],
+    ];
+    for (const step of steps) {
+        const { status, stderr } = gitInThrowaway(cwd, step);
+        if (status !== 0) {
+            throw new Error(`compat: throwaway git ${step[0]} failed: ${stderr.trim()}`);
+        }
+    }
+}
+function applyAndCommitDiff(cwd, diffPath, message) {
+    const apply = gitInThrowaway(cwd, ['apply', diffPath]);
+    if (apply.status !== 0) {
+        throw new Error(`compat: diff ${diffPath} did not apply cleanly: ${apply.stderr.trim()}`);
+    }
+    const add = gitInThrowaway(cwd, ['add', '-A']);
+    if (add.status !== 0) {
+        throw new Error(`compat: git add failed after applying ${diffPath}`);
+    }
+    const commit = gitInThrowaway(cwd, ['commit', '-q', '-m', message]);
+    if (commit.status !== 0) {
+        throw new Error(`compat: git commit failed after applying ${diffPath}`);
+    }
+}
+function copyFixtureProject(compatDir, fixture) {
+    const dir = mkdtempSync(join(tmpdir(), `neal-compat-${fixture.id}-`));
+    cpSync(join(compatDir, fixture.projectDir), dir, { recursive: true });
+    return dir;
+}
+function cleanProcessEnv() {
+    // Strip NODE_TEST_* so a verifyCommand spawning `node --test` reports its own
+    // real exit status instead of inheriting this process's test context.
+    const env = { ...process.env };
+    for (const key of Object.keys(env)) {
+        if (key.startsWith('NODE_TEST')) {
+            delete env[key];
+        }
+    }
+    return env;
+}
+function runVerifyCommand(verifyCommand, cwd) {
+    const parts = verifyCommand.split(/\s+/u).filter(Boolean);
+    const [command, ...commandArgs] = parts;
+    if (!command) {
+        throw new Error('compat: empty verifyCommand');
+    }
+    const result = spawnSync(command, commandArgs, { cwd, encoding: 'utf8', env: cleanProcessEnv() });
+    if (result.error) {
+        throw result.error;
+    }
+    return result.status ?? 1;
+}
+async function readRunEvents(runDir) {
+    let content;
+    try {
+        content = await readFile(join(runDir, 'events.ndjson'), 'utf8');
+    }
+    catch {
+        return [];
+    }
+    return content
+        .split('\n')
+        .filter(Boolean)
+        .map((line) => JSON.parse(line));
+}
+function hasEvent(events, type) {
+    return events.some((event) => event.type === type);
+}
+export async function runWriterFixture(options) {
+    const throwawayCwd = copyFixtureProject(options.compatDir, options.fixture);
+    const runDir = getRunDir(throwawayCwd, createRunId());
+    const planDoc = resolve(throwawayCwd, options.documentRelativePath);
+    let finalStatus = null;
+    let unattended = null;
+    let threwDuringRun = false;
+    let errorMessage = null;
+    let verifyExitCode = null;
+    let finalDocument = null;
+    try {
+        initThrowawayRepo(throwawayCwd);
+        options.onPrepared?.({ throwawayCwd, runDir, planDoc });
+        const loaded = await loadOrInitialize(planDoc, throwawayCwd, options.candidateConfig, undefined, options.mode, {
+            runDir,
+            unattended: true,
+            allowedDirtyPaths: [planDoc],
+        });
+        unattended = loaded.state.unattended;
+        // executeRun renders its final run summary to process.stdout (runtime.ts).
+        // Capture and discard that here so it never leaks into the compat
+        // command's own stdout (the PASS/FAIL table / `--json` matrix). Footer and
+        // diagnostics already go to stderr, so only stdout needs redirecting.
+        const realStdoutWrite = process.stdout.write.bind(process.stdout);
+        process.stdout.write = (() => true);
+        try {
+            const result = await executeRun(loaded.state, loaded.statePath, loaded.logger, { unattended: true });
+            finalStatus = result.finalState.status;
+            unattended = result.finalState.unattended;
+        }
+        catch (error) {
+            threwDuringRun = true;
+            errorMessage = error instanceof Error ? error.message : String(error);
+        }
+        finally {
+            process.stdout.write = realStdoutWrite;
+        }
+        const events = await readRunEvents(runDir);
+        if (options.verifyCommand && finalStatus === 'done') {
+            verifyExitCode = runVerifyCommand(options.verifyCommand, throwawayCwd);
+        }
+        if (options.captureDocument) {
+            try {
+                finalDocument = await readFile(planDoc, 'utf8');
+            }
+            catch {
+                finalDocument = null;
+            }
+        }
+        return {
+            finalStatus,
+            unattended,
+            runDir,
+            throwawayCwd,
+            events,
+            threwDuringRun,
+            errorMessage,
+            verifyExitCode,
+            finalDocument,
+        };
+    }
+    finally {
+        rmSync(throwawayCwd, { recursive: true, force: true });
+    }
+}
+// ---------------------------------------------------------------------------
+// Failure-mode classification for writer runs
+// ---------------------------------------------------------------------------
+export function classifyWriterFailure(args) {
+    const { events } = args;
+    // Precedence follows the plan taxonomy (most specific first).
+    const providerError = events.find((event) => event.type === 'provider.provider_error');
+    if (providerError) {
+        // A provider error names its own cause via `data.errorKind`. Structured-output
+        // kinds are genuine schema failures (the model could not produce/honor
+        // schema-enforced JSON) and route to the `structured_output` mode, matching
+        // `classifyReviewerThrownFailure`. Every other kind (and an absent kind) stays
+        // `provider_failed`.
+        const errorKind = providerError.data?.errorKind;
+        if (errorKind === 'structured_output_missing' || errorKind === 'structured_output_invalid') {
+            return 'structured_output';
+        }
+        return 'provider_failed';
+    }
+    if (hasEvent(events, 'unattended.block_unresolved')) {
+        return 'block_unresolved';
+    }
+    if (args.finalStatus === 'blocked') {
+        return 'block_unresolved';
+    }
+    if (hasEvent(events, 'phase.error')) {
+        return 'structured_output';
+    }
+    if (args.threwDuringRun || args.finalStatus === null) {
+        return 'finalization_error';
+    }
+    // status 'failed' with no conclusive structural signal.
+    return 'provider_failed';
+}
+function makeCell(role, fixtureId, pass, failureMode, detail, diffKind = null) {
+    return { role, fixtureId, diffKind, pass, failureMode, detail };
+}
+// ---------------------------------------------------------------------------
+// Coder fixture evaluation
+// ---------------------------------------------------------------------------
+export async function evaluateCoderFixture(args) {
+    const { fixture } = args;
+    if (!fixture.planDoc || !fixture.verifyCommand) {
+        throw new Error(`compat: coder fixture ${fixture.id} is missing planDoc/verifyCommand`);
+    }
+    const documentRelativePath = relative(fixture.projectDir, fixture.planDoc);
+    const run = await runWriterFixture({
+        compatDir: args.compatDir,
+        fixture,
+        candidateConfig: args.candidateConfig,
+        mode: 'execute',
+        documentRelativePath,
+        verifyCommand: fixture.verifyCommand,
+        onPrepared: args.onPrepared,
+    });
+    if (run.finalStatus === 'done') {
+        if (run.verifyExitCode === 0) {
+            return { cell: makeCell('coder', fixture.id, true, null, null), run };
+        }
+        return {
+            cell: makeCell('coder', fixture.id, false, 'wrong_or_empty_output', `run completed but verifyCommand exited ${run.verifyExitCode}`),
+            run,
+        };
+    }
+    const failureMode = classifyWriterFailure({
+        finalStatus: run.finalStatus,
+        events: run.events,
+        threwDuringRun: run.threwDuringRun,
+    });
+    return { cell: makeCell('coder', fixture.id, false, failureMode, run.errorMessage), run };
+}
+// ---------------------------------------------------------------------------
+// Planner fixture evaluation (secondary)
+// ---------------------------------------------------------------------------
+export async function evaluatePlannerFixture(args) {
+    const { fixture } = args;
+    if (!fixture.issuePrompt) {
+        throw new Error(`compat: planner fixture ${fixture.id} is missing issuePrompt`);
+    }
+    const documentRelativePath = relative(fixture.projectDir, fixture.issuePrompt);
+    const run = await runWriterFixture({
+        compatDir: args.compatDir,
+        fixture,
+        candidateConfig: args.candidateConfig,
+        mode: 'plan',
+        documentRelativePath,
+        captureDocument: true,
+        onPrepared: args.onPrepared,
+    });
+    if (run.finalStatus !== 'done') {
+        const failureMode = classifyWriterFailure({
+            finalStatus: run.finalStatus,
+            events: run.events,
+            threwDuringRun: run.threwDuringRun,
+        });
+        return { cell: makeCell('planner', fixture.id, false, failureMode, run.errorMessage), run };
+    }
+    const document = run.finalDocument;
+    if (document === null) {
+        return {
+            cell: makeCell('planner', fixture.id, false, 'finalization_error', 'no plan document was emitted'),
+            run,
+        };
+    }
+    const validation = validatePlanDocument(document);
+    if (validation.ok && validation.executionShape === 'one_shot') {
+        return { cell: makeCell('planner', fixture.id, true, null, null), run };
+    }
+    const detail = validation.ok
+        ? `emitted plan is ${validation.executionShape ?? 'unshaped'}, expected one_shot`
+        : `emitted plan failed validation: ${validation.errors.join('; ')}`;
+    return { cell: makeCell('planner', fixture.id, false, 'wrong_or_empty_output', detail), run };
+}
+// ---------------------------------------------------------------------------
+// Reviewer fixture evaluation
+// ---------------------------------------------------------------------------
+export function hasBlockingFinding(result) {
+    return result.draft.findings.some((finding) => finding.severity === 'blocking');
+}
+export function classifyNonAcceptedReviewOutcome(outcome) {
+    switch (outcome) {
+        case 'blocked':
+            return 'block_unresolved';
+        case 'cap_reached':
+            return 'max_step_loop';
+        case 'failed':
+            return 'structured_output';
+    }
+}
+// Unwrap the underlying NealProviderError from a reviewer-loop throw. The
+// reviewer fixture path surfaces provider failures in three shapes: a raw
+// NealProviderError (the reviewDraft / structured-advisor path), or wrapped in
+// a CoderRoundError (the draftFindings path via runCoderStructuredPrompt) or a
+// ReviewerRoundError. Both wrappers expose the cause as a `providerError`
+// field, so duck-typing that field covers every wrapper without importing the
+// round-error classes.
+function extractReviewProviderError(error) {
+    if (isNealProviderError(error)) {
+        return error;
+    }
+    if (error !== null && typeof error === 'object' && 'providerError' in error) {
+        const wrapped = error.providerError;
+        if (isNealProviderError(wrapped)) {
+            return wrapped;
+        }
+    }
+    return null;
+}
+// Classify a thrown error from the reviewer loop. A provider error names its own
+// cause via `kind`: only the structured-output kinds are genuine schema
+// failures; every other kind (timeout, api_error, permission, session,
+// provider_failed, unknown) is a provider failure. Non-provider errors fall
+// back to the persisted run outcome (`blocked`/`cap_reached`/`failed`), where
+// `failed` denotes a schema/validation failure raised inside the review loop.
+export function classifyReviewerThrownFailure(error, persistedOutcome) {
+    const providerError = extractReviewProviderError(error);
+    if (providerError) {
+        return providerError.kind === 'structured_output_missing' || providerError.kind === 'structured_output_invalid'
+            ? 'structured_output'
+            : 'provider_failed';
+    }
+    if (persistedOutcome && persistedOutcome !== 'accepted') {
+        return classifyNonAcceptedReviewOutcome(persistedOutcome);
+    }
+    return 'provider_failed';
+}
+function buildReviewParsedArgs() {
+    return {
+        instruction: DEFAULT_REVIEW_INSTRUCTION,
+        instructionSource: 'default',
+        selector: { kind: 'last', count: 1 },
+    };
+}
+async function evaluateReviewerDiff(args) {
+    const { fixture, diffKind } = args;
+    if (!fixture.reviewer || !fixture.verifyCommand) {
+        throw new Error(`compat: reviewer fixture ${fixture.id} is missing diffs/verifyCommand`);
+    }
+    const diffRelative = diffKind === 'good' ? fixture.reviewer.goodDiff : fixture.reviewer.brokenDiff;
+    const diffPath = join(args.compatDir, diffRelative);
+    const throwawayCwd = copyFixtureProject(args.compatDir, fixture);
+    const reviewId = createRunId();
+    try {
+        initThrowawayRepo(throwawayCwd);
+        applyAndCommitDiff(throwawayCwd, diffPath, `apply ${diffKind} diff`);
+        const provider = args.createReviewProvider({ cwd: throwawayCwd, agentConfig: args.candidateConfig });
+        const sink = new Writable({ write(_chunk, _encoding, callback) { callback(); } });
+        let result;
+        try {
+            result = await runNealReviewCli({
+                cwd: throwawayCwd,
+                parsed: buildReviewParsedArgs(),
+                stdout: sink,
+                stderr: sink,
+                provider,
+                reviewId,
+            });
+        }
+        catch (error) {
+            // runNealReviewCli collapses every non-accepted run to a persisted
+            // `failed` outcome, so a provider/API/timeout failure and a genuine
+            // schema-validation failure both surface as `failed`. Classify the thrown
+            // error structurally FIRST (a NealProviderError carries the real cause via
+            // its `kind`) so provider failures stay `provider_failed` and only true
+            // schema failures collapse to `structured_output`.
+            const outcome = await readReviewOutcome(throwawayCwd, reviewId);
+            const failureMode = classifyReviewerThrownFailure(error, outcome);
+            return makeCell('reviewer', fixture.id, false, failureMode, error instanceof Error ? error.message : String(error), diffKind);
+        }
+        const blocking = hasBlockingFinding(result);
+        if (diffKind === 'good') {
+            if (!blocking) {
+                return makeCell('reviewer', fixture.id, true, null, null, 'good');
+            }
+            return makeCell('reviewer', fixture.id, false, 'wrong_or_empty_output', 'reviewer flagged the known-good diff with a blocking finding', 'good');
+        }
+        // broken diff: PASS requires ≥1 blocking finding.
+        if (blocking) {
+            return makeCell('reviewer', fixture.id, true, null, null, 'broken');
+        }
+        return makeCell('reviewer', fixture.id, false, 'wrong_or_empty_output', 'reviewer accepted the deliberately broken diff without a blocking finding', 'broken');
+    }
+    finally {
+        rmSync(throwawayCwd, { recursive: true, force: true });
+    }
+}
+async function readReviewOutcome(cwd, reviewId) {
+    const paths = getReviewFindingsArtifactPaths(cwd, reviewId);
+    try {
+        const meta = JSON.parse(await readFile(paths.meta, 'utf8'));
+        const outcome = meta.outcome;
+        if (outcome === 'accepted' ||
+            outcome === 'blocked' ||
+            outcome === 'failed' ||
+            outcome === 'cap_reached') {
+            return outcome;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+export async function evaluateReviewerFixture(args) {
+    const good = await evaluateReviewerDiff({ ...args, diffKind: 'good' });
+    const broken = await evaluateReviewerDiff({ ...args, diffKind: 'broken' });
+    return { good, broken };
+}
+// ---------------------------------------------------------------------------
+// Output formatting
+// ---------------------------------------------------------------------------
+export function buildRoleRollups(cells, roles) {
+    return roles.map((role) => {
+        const roleCells = cells.filter((cell) => cell.role === role);
+        const passCount = roleCells.filter((cell) => cell.pass).length;
+        return {
+            role,
+            cellCount: roleCells.length,
+            passCount,
+            pass: roleCells.length > 0 && passCount === roleCells.length,
+        };
+    });
+}
+export function formatCompatJson(report) {
+    return JSON.stringify(report, null, 2);
+}
+// Describe the native provider the non-candidate roles run on for the human
+// table. `--reference` is a native provider id (not a model); when omitted but
+// `--model` is set, routing is active and the reference defaults to
+// `openai-codex`. With neither flag the non-candidate roles pass through to the
+// configured providers.
+export function describeReferenceProvider(report) {
+    if (report.reference !== null) {
+        return report.reference;
+    }
+    if (report.model !== null) {
+        return 'openai-codex (default)';
+    }
+    return '(configured / native pass-through)';
+}
+export function formatCompatTable(report) {
+    const lines = [];
+    lines.push('neal compat - compatibility smoke test (PASS/FAIL, not a skill score)');
+    lines.push('');
+    lines.push(`Candidate model: ${report.model ?? '(configured model)'}`);
+    lines.push(`Reference provider: ${describeReferenceProvider(report)}`);
+    lines.push('');
+    lines.push('Role      Fixture                Diff     Result  Failure mode');
+    lines.push('--------  ---------------------  -------  ------  -----------------');
+    for (const cell of report.cells) {
+        const role = cell.role.padEnd(8);
+        const fixture = cell.fixtureId.slice(0, 21).padEnd(21);
+        const diff = (cell.diffKind ?? '-').padEnd(7);
+        const result = (cell.pass ? 'PASS' : 'FAIL').padEnd(6);
+        const mode = cell.pass ? '' : cell.failureMode ?? 'unknown';
+        lines.push(`${role}  ${fixture}  ${diff}  ${result}  ${mode}`);
+    }
+    lines.push('');
+    for (const rollup of report.roles) {
+        lines.push(`${rollup.role}: ${rollup.pass ? 'PASS' : 'FAIL'} (${rollup.passCount}/${rollup.cellCount})`);
+    }
+    lines.push('');
+    lines.push(`Overall: ${report.overallPass ? 'PASS' : 'FAIL'}`);
+    return lines.join('\n');
+}
+export async function runCompat(args) {
+    const { cwd, parsed } = args;
+    const deps = args.deps ?? {};
+    const compatDir = deps.compatDir ?? getCompatExamplesDir();
+    const manifest = deps.manifest ?? loadCompatManifest(compatDir);
+    const verifyProviders = deps.verifyProviders ?? verifyConfiguredProviders;
+    const createReviewProvider = deps.createReviewProvider ?? ((adapterArgs) => createAgentReviewFindingsProviderAdapter(adapterArgs));
+    const baseConfig = assertWriterProvidersConfigured(cwd, { context: 'neal compat' });
+    const targetedRoles = rolesForSelection(parsed.role);
+    const cells = [];
+    // Pre-filter: verify the candidate config for each tested role's provider can
+    // emit the structured check payload. Per the plan, a candidate that fails the
+    // pre-filter for ANY targeted role fails the whole candidate: every targeted
+    // role records FAIL(protocol) and all fixtures are skipped. This keeps the
+    // matrix complete (no role left without a cell/failure mode).
+    const protocolErrors = new Map();
+    for (const role of targetedRoles) {
+        const candidateConfig = deriveCandidateConfig(baseConfig, {
+            testedRole: role,
+            model: parsed.model,
+            reference: parsed.reference,
+        });
+        const sink = new Writable({ write(_chunk, _encoding, callback) { callback(); } });
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            await verifyProviders({ cwd, agentConfig: candidateConfig, stdout: sink });
+        }
+        catch (error) {
+            protocolErrors.set(role, error instanceof Error ? error.message : String(error));
+        }
+    }
+    if (protocolErrors.size > 0) {
+        for (const role of targetedRoles) {
+            const detail = protocolErrors.get(role) ??
+                'candidate skipped: provider pre-filter failed for another targeted role';
+            cells.push(makeCell(role, `provider:${role}`, false, 'protocol', detail));
+        }
+    }
+    else {
+        for (const role of targetedRoles) {
+            const candidateConfig = deriveCandidateConfig(baseConfig, {
+                testedRole: role,
+                model: parsed.model,
+                reference: parsed.reference,
+            });
+            if (role === 'coder') {
+                for (const fixture of manifest.fixtures.filter((f) => f.roles.includes('coder'))) {
+                    // eslint-disable-next-line no-await-in-loop
+                    const { cell } = await evaluateCoderFixture({ compatDir, fixture, candidateConfig });
+                    cells.push(cell);
+                }
+            }
+            else if (role === 'planner') {
+                for (const fixture of manifest.fixtures.filter((f) => f.roles.includes('planner'))) {
+                    // eslint-disable-next-line no-await-in-loop
+                    const { cell } = await evaluatePlannerFixture({ compatDir, fixture, candidateConfig });
+                    cells.push(cell);
+                }
+            }
+            else {
+                for (const fixture of manifest.fixtures.filter((f) => f.roles.includes('reviewer'))) {
+                    // eslint-disable-next-line no-await-in-loop
+                    const { good, broken } = await evaluateReviewerFixture({
+                        compatDir,
+                        fixture,
+                        candidateConfig,
+                        createReviewProvider,
+                    });
+                    cells.push(good, broken);
+                }
+            }
+        }
+    }
+    const roles = buildRoleRollups(cells, targetedRoles);
+    const overallPass = roles.length > 0 && roles.every((rollup) => rollup.pass);
+    return {
+        schemaVersion: 1,
+        model: parsed.model,
+        reference: parsed.reference,
+        role: parsed.role,
+        candidateProviders: {
+            coder: parsed.model !== null ? 'generic-agentic' : baseConfig.coder.provider,
+            reviewer: parsed.model !== null ? 'generic-agentic' : baseConfig.reviewer.provider,
+            planner: parsed.model !== null ? 'generic-agentic' : baseConfig.planner.provider,
+        },
+        cells,
+        roles,
+        overallPass,
+    };
+}
+export async function runCompatCli(args) {
+    const stdout = args.stdout ?? process.stdout;
+    const report = await runCompat({ cwd: args.cwd, parsed: args.parsed, deps: args.deps });
+    stdout.write(`${args.parsed.json ? formatCompatJson(report) : formatCompatTable(report)}\n`);
+    if (!report.overallPass) {
+        process.exitCode = 1;
+    }
+    return report;
+}
+export async function runCompatCommand(args) {
+    const parsed = parseCompatArgs(args);
+    await runCompatCli({ cwd: process.cwd(), parsed });
+}