@guilz-dev/belay 0.2.0 → 0.4.0

package/dist/core/verdict/judge-cli.js

@@ -0,0 +1,401 @@
+import { spawn } from 'node:child_process';
+import { buildTier1Prompt, parseTier1Json, prescanInterpreterCode, } from './judge.js';
+import { scrubOutboundForJudge } from './judge-outbound.js';
+import { DEFAULT_JUDGE_RUNTIME_CONFIG } from './judge-runtime-config.js';
+import { transportFallbackToFailClosedReason } from './judge-session-guard.js';
+import { evaluateWithJudgeTransport } from './judge-transport.js';
+export class CliRunError extends Error {
+    kind;
+    exitCode;
+    stderr;
+    stdout;
+    command;
+    constructor(kind, message, details = {}) {
+        super(message);
+        this.name = 'CliRunError';
+        this.kind = kind;
+        this.exitCode = details.exitCode;
+        this.stderr = details.stderr;
+        this.stdout = details.stdout;
+        this.command = details.command;
+    }
+}
+const CLI_BINARIES = {
+    codex: 'codex',
+    cursor: 'cursor-agent',
+    claude: 'claude',
+};
+const FORCE_KILL_GRACE_MS = 250;
+function failClosedVerdict(reason) {
+    return {
+        local_recoverable: false,
+        destroys_outside_repo: true,
+        destroys_history_or_secrets: true,
+        external_change: true,
+        reason,
+    };
+}
+function transportForProvider(providerId) {
+    return `${providerId}-cli`;
+}
+function traceProvider(providerId) {
+    if (providerId === 'claude') {
+        return 'anthropic';
+    }
+    return 'openai-compatible';
+}
+function initialTrace(options) {
+    return {
+        provider: traceProvider(options.providerId),
+        modelRequested: options.modelRequested,
+        modelResolved: options.modelResolved,
+        latencyMs: 0,
+        transport: transportForProvider(options.providerId),
+    };
+}
+export function buildCliInvocation(providerId, prompt, model) {
+    if (providerId === 'codex') {
+        return {
+            binary: CLI_BINARIES[providerId],
+            args: [
+                'exec',
+                '--json',
+                '--skip-git-repo-check',
+                '--sandbox',
+                'read-only',
+                '--model',
+                model,
+                '-',
+            ],
+            stdin: prompt,
+        };
+    }
+    if (providerId === 'cursor') {
+        return {
+            binary: CLI_BINARIES[providerId],
+            args: [
+                '--print',
+                '--output-format',
+                'json',
+                '--mode',
+                'ask',
+                '--sandbox',
+                'enabled',
+                '--trust',
+                '--model',
+                model,
+                prompt,
+            ],
+        };
+    }
+    return {
+        binary: CLI_BINARIES[providerId],
+        args: [
+            '-p',
+            '--output-format',
+            'json',
+            '--permission-mode',
+            'plan',
+            '--tools',
+            '',
+            '--bare',
+            '--model',
+            model,
+        ],
+        stdin: prompt,
+    };
+}
+function tryParseJson(raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+function extractVerdictFromJsonValue(value) {
+    if (typeof value === 'string') {
+        return parseTier1Json(value.trim());
+    }
+    if (Array.isArray(value)) {
+        for (let index = value.length - 1; index >= 0; index -= 1) {
+            const verdict = extractVerdictFromJsonValue(value[index]);
+            if (verdict) {
+                return verdict;
+            }
+        }
+        return null;
+    }
+    if (!value || typeof value !== 'object') {
+        return null;
+    }
+    const record = value;
+    for (const key of ['result', 'content', 'text', 'message', 'output_text', 'final']) {
+        if (!(key in record)) {
+            continue;
+        }
+        const verdict = extractVerdictFromJsonValue(record[key]);
+        if (verdict) {
+            return verdict;
+        }
+    }
+    for (const candidate of Object.values(record).reverse()) {
+        const verdict = extractVerdictFromJsonValue(candidate);
+        if (verdict) {
+            return verdict;
+        }
+    }
+    return null;
+}
+function parseCliJsonLines(raw) {
+    const lines = raw
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter(Boolean);
+    for (let index = lines.length - 1; index >= 0; index -= 1) {
+        const parsed = tryParseJson(lines[index] ?? '');
+        if (!parsed) {
+            continue;
+        }
+        const verdict = extractVerdictFromJsonValue(parsed);
+        if (verdict) {
+            return verdict;
+        }
+    }
+    return null;
+}
+export function parseCliJudgeOutput(providerId, raw) {
+    const direct = parseTier1Json(raw);
+    if (direct) {
+        return direct;
+    }
+    const parsed = tryParseJson(raw);
+    if (parsed) {
+        const envelopeVerdict = extractVerdictFromJsonValue(parsed);
+        if (envelopeVerdict) {
+            return envelopeVerdict;
+        }
+    }
+    if (providerId === 'codex') {
+        return parseCliJsonLines(raw);
+    }
+    return parseCliJsonLines(raw);
+}
+async function runCliJson(invocation, timeoutMs) {
+    return runCliJsonWithTimeouts(invocation, { evalTimeoutMs: timeoutMs });
+}
+export async function runCliJsonWithTimeouts(invocation, budgets) {
+    const { binary, args, stdin } = invocation;
+    const connectTimeoutMs = budgets.connectTimeoutMs ?? 0;
+    if (connectTimeoutMs > 0) {
+        await new Promise((resolve) => setTimeout(resolve, 0));
+    }
+    return new Promise((resolve, reject) => {
+        const child = spawn(binary, args, {
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        const command = [binary, ...args].join(' ');
+        let stdout = '';
+        let stderr = '';
+        let settled = false;
+        let timer = null;
+        let forceKillTimer = null;
+        const cleanup = () => {
+            if (timer) {
+                clearTimeout(timer);
+            }
+            if (forceKillTimer) {
+                clearTimeout(forceKillTimer);
+            }
+        };
+        const rejectOnce = (error) => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            reject(error);
+        };
+        const resolveOnce = (value) => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            resolve(value);
+        };
+        timer = setTimeout(() => {
+            rejectOnce(new CliRunError('timeout', `${binary} timed out`, {
+                stderr: stderr.trim() || undefined,
+                stdout: stdout.trim() || undefined,
+                command,
+            }));
+            try {
+                child.kill('SIGTERM');
+            }
+            catch {
+                return;
+            }
+            forceKillTimer = setTimeout(() => {
+                try {
+                    child.kill('SIGKILL');
+                }
+                catch {
+                    // process already exited
+                }
+            }, FORCE_KILL_GRACE_MS);
+        }, budgets.evalTimeoutMs);
+        child.stdout.on('data', (chunk) => {
+            stdout += String(chunk);
+        });
+        child.stderr.on('data', (chunk) => {
+            stderr += String(chunk);
+        });
+        if (stdin !== undefined) {
+            child.stdin.write(stdin);
+        }
+        child.stdin.end();
+        child.on('error', (error) => {
+            rejectOnce(new CliRunError('spawn_error', error instanceof Error ? error.message : String(error), {
+                command,
+            }));
+        });
+        child.on('close', (code) => {
+            if (settled)
+                return;
+            if (code === 0 && stdout.trim()) {
+                resolveOnce(stdout.trim());
+                return;
+            }
+            rejectOnce(new CliRunError('exit_nonzero', stderr.trim() || `${binary} exited with code ${code ?? 'unknown'}`, {
+                exitCode: typeof code === 'number' ? code : undefined,
+                stderr: stderr.trim() || undefined,
+                stdout: stdout.trim() || undefined,
+                command,
+            }));
+        });
+    });
+}
+function createCliJudge(options) {
+    const transport = transportForProvider(options.providerId);
+    const timeoutMs = options.timeoutMs ?? 25000;
+    const providerId = options.providerId;
+    const runCommand = options.runCliCommand ?? runCliJson;
+    const runtime = options.runtime ?? DEFAULT_JUDGE_RUNTIME_CONFIG;
+    const repoRoot = options.repoRoot ?? process.cwd();
+    const stateDir = options.stateDir ?? repoRoot;
+    const judgeMode = options.judgeMode ?? 'enforce';
+    const judge = {
+        lastTrace: initialTrace(options),
+        async evaluate(input) {
+            const started = Date.now();
+            const prescan = input.innerCode ? prescanInterpreterCode(input.innerCode) : null;
+            if (prescan?.destroys_history_or_secrets) {
+                judge.lastTrace = {
+                    provider: traceProvider(options.providerId),
+                    modelRequested: options.modelRequested,
+                    modelResolved: options.modelResolved,
+                    latencyMs: Date.now() - started,
+                    transport,
+                };
+                return prescan;
+            }
+            const scrubbed = scrubOutboundForJudge(input.text, {
+                sensitivePaths: options.sensitivePaths,
+                scrubOptions: options.scrubOptions,
+            });
+            if (!scrubbed.ok) {
+                judge.lastTrace = {
+                    provider: 'fallback',
+                    modelRequested: options.modelRequested,
+                    modelResolved: options.modelResolved,
+                    latencyMs: Date.now() - started,
+                    fallbackReason: scrubbed.reason,
+                };
+                return failClosedVerdict('outbound_scrub_failed');
+            }
+            const prompt = buildTier1Prompt(scrubbed.text);
+            try {
+                const transportResult = await evaluateWithJudgeTransport({
+                    prompt,
+                    context: {
+                        providerId,
+                        model: options.modelResolved,
+                        repoRoot: input.context.repoRoot || repoRoot,
+                        stateDir,
+                        judgeMode,
+                        runtime,
+                        judgeTimeoutMs: timeoutMs,
+                    },
+                }, { runCommand });
+                const traceTransport = transportResult.transport === 'session'
+                    ? `${providerId}-cli-session`
+                    : transport;
+                if (!transportResult.verdict) {
+                    const fallbackReason = transportResult.fallbackReason ??
+                        `${providerId}_cli_parse_error`;
+                    judge.lastTrace = {
+                        provider: 'fallback',
+                        modelRequested: options.modelRequested,
+                        modelResolved: options.modelResolved,
+                        latencyMs: Date.now() - started,
+                        fallbackReason,
+                        judgeFallbackReason: fallbackReason,
+                        judgeSessionUsed: transportResult.sessionUsed,
+                        judgeSessionReused: transportResult.sessionReused,
+                        judgeSessionRefHash: transportResult.sessionRefHash,
+                        judgeSessionResetReason: transportResult.sessionResetReason,
+                        judgeConnectMs: transportResult.connectMs,
+                        judgeEvalMs: transportResult.evalMs,
+                        judgeParseMs: transportResult.parseMs,
+                        judgeShadowCompared: transportResult.shadowCompared,
+                        judgeShadowMismatch: transportResult.shadowMismatch,
+                        judgeShadowMismatchRateWindow: transportResult.shadowMismatchRateWindow,
+                        judgeKillSwitchTriggered: transportResult.killSwitchTriggered,
+                    };
+                    return failClosedVerdict(transportFallbackToFailClosedReason(providerId, fallbackReason));
+                }
+                judge.lastTrace = {
+                    provider: traceProvider(options.providerId),
+                    modelRequested: options.modelRequested,
+                    modelResolved: options.modelResolved,
+                    latencyMs: Date.now() - started,
+                    transport: traceTransport,
+                    outboundRedacted: true,
+                    judgeSessionUsed: transportResult.sessionUsed,
+                    judgeSessionReused: transportResult.sessionReused,
+                    judgeSessionRefHash: transportResult.sessionRefHash,
+                    judgeSessionResetReason: transportResult.sessionResetReason,
+                    judgeFallbackReason: transportResult.fallbackReason,
+                    judgeConnectMs: transportResult.connectMs,
+                    judgeEvalMs: transportResult.evalMs,
+                    judgeParseMs: transportResult.parseMs,
+                    judgeShadowCompared: transportResult.shadowCompared,
+                    judgeShadowMismatch: transportResult.shadowMismatch,
+                    judgeShadowMismatchRateWindow: transportResult.shadowMismatchRateWindow,
+                    judgeKillSwitchTriggered: transportResult.killSwitchTriggered,
+                };
+                return transportResult.verdict;
+            }
+            catch {
+                judge.lastTrace = {
+                    provider: 'fallback',
+                    modelRequested: options.modelRequested,
+                    modelResolved: options.modelResolved,
+                    latencyMs: Date.now() - started,
+                    fallbackReason: `${providerId}_cli_unavailable`,
+                    judgeFallbackReason: `${providerId}_cli_unavailable`,
+                };
+                return failClosedVerdict(`${providerId}_cli_unavailable`);
+            }
+        },
+    };
+    return judge;
+}
+export function createCodexCliJudge(options) {
+    return createCliJudge({ ...options, providerId: 'codex' });
+}
+export function createCursorCliJudge(options) {
+    return createCliJudge({ ...options, providerId: 'cursor' });
+}
+export function createClaudeCliJudge(options) {
+    return createCliJudge({ ...options, providerId: 'claude' });
+}

package/dist/core/verdict/judge-factory.d.ts

@@ -9,17 +9,28 @@ export declare function loadPinnedJudgeModels(): Promise<{
         ciPin: string;
     };
 }>;
-export declare function resolveCloudModel(requested: string, pinned: {
+export declare function resolveJudgeModel(judge: BelayJudgeConfig): {
+    requested: string;
+    resolved: string;
+};
+/** @deprecated Use resolveJudgeModel */
+export declare const resolveCloudModel: (requested: string, _pinned: {
+    autoResolved: string;
+}) => {
+    requested: string;
+    resolved: string;
+};
+/** @deprecated Use resolveJudgeModel */
+export declare const resolveCursorModel: (requested: string, _pinned: {
     autoResolved: string;
-}): {
+}) => {
     requested: string;
     resolved: string;
 };
-/** @deprecated Use resolveCloudModel */
-export declare const resolveCursorModel: typeof resolveCloudModel;
 export declare function createJudgeFromConfig(config: BelayConfigV4, options?: {
     pinnedModels?: {
         autoResolved: string;
     };
+    repoRoot?: string;
 }): TracedTier1Judge;
 export declare function judgeConfigSummary(judge: BelayJudgeConfig): string;

package/dist/core/verdict/judge-factory.js

@@ -1,7 +1,14 @@
 import { readFile } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
-import { normalizeJudgeProvider, scrubOptionsFromConfig } from '../config.js';
+import { repoLocalStateDirFor } from '../../config-io.js';
+import { belayStateDir, normalizeJudgeProvider, scrubOptionsFromConfig } from '../config.js';
+import { resolveJudgeCredential } from '../judge-api-key.js';
+import { hasValidCloudConsent, isCloudJudgeConfig } from '../judge-config.js';
+import { rejectDeprecatedJudgeModelAuto } from '../judge-model-policy.js';
+import { detectJudgeRuntimeCapabilities } from '../judge-runtime-detection.js';
 import { createDeterministicJudgeStub, createFailClosedJudge, createOllamaJudge, createOpenAiCompatibleJudge, } from './judge.js';
+import { getJudgeProviderSpec, inferProviderIdFromConfig, isRemovedProviderId, normalizeLegacyProviderId, } from './judge-catalog.js';
+import { createClaudeCliJudge, createCodexCliJudge, createCursorCliJudge } from './judge-cli.js';
 const FIXTURE_MODELS_URL = new URL('../../../fixtures/judge-models.json', import.meta.url);
 let cachedPinnedModels = null;
 export function resetPinnedJudgeModelsCache() {
@@ -24,33 +31,104 @@ export async function loadPinnedJudgeModels() {
         return cachedPinnedModels;
     }
 }
-export function resolveCloudModel(requested, pinned) {
-    if (requested === 'auto') {
-        const envResolved = process.env.BELAY_JUDGE_MODEL_RESOLVED?.trim();
-        return {
-            requested,
-            resolved: envResolved || pinned.autoResolved,
-        };
+export function resolveJudgeModel(judge) {
+    rejectDeprecatedJudgeModelAuto(judge.model);
+    const requested = judge.model;
+    const envOverride = testJudgeModelOverride();
+    return { requested, resolved: envOverride || requested };
+}
+function testJudgeModelOverride() {
+    if (!process.env.VITEST && !process.env.VITEST_WORKER_ID) {
+        return undefined;
     }
-    return { requested, resolved: requested };
+    return process.env.BELAY_JUDGE_MODEL_RESOLVED?.trim() || undefined;
 }
-/** @deprecated Use resolveCloudModel */
+/** @deprecated Use resolveJudgeModel */
+export const resolveCloudModel = (requested, _pinned) => {
+    rejectDeprecatedJudgeModelAuto(requested);
+    const envResolved = testJudgeModelOverride();
+    return {
+        requested,
+        resolved: envResolved || requested,
+    };
+};
+/** @deprecated Use resolveJudgeModel */
 export const resolveCursorModel = resolveCloudModel;
+function providerIdFromJudge(judge) {
+    if (judge.providerId) {
+        const normalized = normalizeLegacyProviderId(judge.providerId);
+        if (normalized) {
+            return normalized;
+        }
+    }
+    return inferProviderIdFromConfig(judge);
+}
+function createCliJudgeForProvider(providerId, judgeConfig, config, repoRoot) {
+    const resolvedRepoRoot = repoRoot ?? process.cwd();
+    const { resolved } = resolveJudgeModel(judgeConfig);
+    const options = {
+        modelRequested: judgeConfig.model,
+        modelResolved: resolved,
+        timeoutMs: judgeConfig.timeoutMs,
+        sensitivePaths: config.classifier.sensitivePaths,
+        scrubOptions: scrubOptionsFromConfig(config),
+        runtime: judgeConfig.runtime,
+        repoRoot: resolvedRepoRoot,
+        stateDir: belayStateDir(config, repoLocalStateDirFor(resolvedRepoRoot, config)),
+        judgeMode: config.mode,
+    };
+    if (providerId === 'codex') {
+        return createCodexCliJudge(options);
+    }
+    if (providerId === 'cursor') {
+        return createCursorCliJudge(options);
+    }
+    return createClaudeCliJudge(options);
+}
 export function createJudgeFromConfig(config, options = {}) {
+    if (process.env.BELAY_DETERMINISTIC_JUDGE === '1') {
+        return createDeterministicJudgeStub();
+    }
     const judgeConfig = config.judge;
+    if (judgeConfig.providerId && isRemovedProviderId(String(judgeConfig.providerId))) {
+        const { resolved } = resolveJudgeModel(judgeConfig);
+        return createFailClosedJudge({
+            reason: 'judge_provider_removed',
+            fallbackReason: 'provider_migration_required',
+            modelRequested: judgeConfig.model,
+            modelResolved: resolved,
+        });
+    }
     const provider = normalizeJudgeProvider(judgeConfig.provider);
+    const providerId = providerIdFromJudge(judgeConfig);
+    const catalogSpec = getJudgeProviderSpec(providerId);
+    const { resolved } = resolveJudgeModel(judgeConfig);
+    const runtime = detectJudgeRuntimeCapabilities(providerId);
+    const repoRoot = options.repoRoot ?? process.cwd();
     if (provider === 'openai-compatible') {
         const endpoint = judgeConfig.endpoint?.trim();
+        if (!endpoint && runtime.cliTransport) {
+            if (providerId === 'codex' || providerId === 'cursor') {
+                return createCliJudgeForProvider(providerId, judgeConfig, config, repoRoot);
+            }
+        }
         if (!endpoint) {
             return createFailClosedJudge({
                 reason: 'openai_compatible_endpoint_missing',
                 fallbackReason: 'missing_endpoint',
                 modelRequested: judgeConfig.model,
-                modelResolved: judgeConfig.model,
+                modelResolved: resolved,
+            });
+        }
+        if (isCloudJudgeConfig(judgeConfig) && !hasValidCloudConsent(judgeConfig)) {
+            return createFailClosedJudge({
+                reason: 'openai_compatible_consent_missing',
+                fallbackReason: 'cloud_consent_missing',
+                modelRequested: judgeConfig.model,
+                modelResolved: resolved,
             });
         }
-        const pinned = options.pinnedModels ?? { autoResolved: 'composer-2.5' };
-        const { resolved } = resolveCloudModel(judgeConfig.model, pinned);
+        const repoLocalStateDir = repoLocalStateDirFor(repoRoot, config);
         return createOpenAiCompatibleJudge({
             endpoint,
             modelRequested: judgeConfig.model,
@@ -58,6 +136,13 @@ export function createJudgeFromConfig(config, options = {}) {
             timeoutMs: judgeConfig.timeoutMs,
             sensitivePaths: config.classifier.sensitivePaths,
             scrubOptions: scrubOptionsFromConfig(config),
+            resolveApiKey: async () => resolveJudgeCredential({
+                judge: judgeConfig,
+                catalogSpec: catalogSpec ?? undefined,
+                repoRoot,
+                repoLocalStateDir,
+                config,
+            }),
         });
     }
     if (provider === 'ollama') {
@@ -68,8 +153,35 @@ export function createJudgeFromConfig(config, options = {}) {
             keepAlive: judgeConfig.keepAlive,
         });
     }
-    return createDeterministicJudgeStub();
+    if (provider === 'anthropic') {
+        if (runtime.cliTransport) {
+            return createClaudeCliJudge({
+                modelRequested: judgeConfig.model,
+                modelResolved: resolved,
+                timeoutMs: judgeConfig.timeoutMs,
+                sensitivePaths: config.classifier.sensitivePaths,
+                scrubOptions: scrubOptionsFromConfig(config),
+                runtime: judgeConfig.runtime,
+                repoRoot,
+                stateDir: belayStateDir(config, repoLocalStateDirFor(repoRoot, config)),
+                judgeMode: config.mode,
+            });
+        }
+        return createFailClosedJudge({
+            reason: 'anthropic_not_implemented',
+            fallbackReason: 'anthropic_runtime_unavailable',
+            modelRequested: judgeConfig.model,
+            modelResolved: resolved,
+        });
+    }
+    return createFailClosedJudge({
+        reason: 'unsupported_judge_provider',
+        fallbackReason: 'unsupported_provider',
+        modelRequested: judgeConfig.model,
+        modelResolved: resolved,
+    });
 }
 export function judgeConfigSummary(judge) {
-    return `${judge.provider}/${judge.model}`;
+    const id = judge.providerId ?? judge.provider;
+    return `${id}/${judge.model}`;
 }

package/dist/core/verdict/judge-provider-matrix.d.ts

@@ -0,0 +1,20 @@
+import type { JudgeProviderId } from './judge-catalog.js';
+export type JudgeSessionFailureMode = 'parse_error' | 'non_json_response' | 'timeout' | 'cli_nonzero_exit' | 'io_error' | 'version_mismatch' | 'unsupported_resume' | 'unsafe_option_rejected';
+export interface JudgeProviderSessionCapability {
+    providerId: Exclude<JudgeProviderId, 'ollama'>;
+    supportsSession: boolean;
+    resumeFlag: string | null;
+    createChatFlag: string | null;
+    readOnlyArgs: string[];
+    rejectedUnsafeOptions: string[];
+    failureModes: JudgeSessionFailureMode[];
+    notes: string;
+}
+export declare const JUDGE_PROVIDER_SESSION_MATRIX: Record<Exclude<JudgeProviderId, 'ollama'>, JudgeProviderSessionCapability>;
+export declare function providerSupportsSession(providerId: Exclude<JudgeProviderId, 'ollama'>, allowlist: JudgeProviderId[]): boolean;
+export declare function assertReadOnlyInvocationArgs(providerId: Exclude<JudgeProviderId, 'ollama'>, args: string[]): {
+    ok: true;
+} | {
+    ok: false;
+    reason: JudgeSessionFailureMode;
+};