@guilz-dev/belay 0.3.0 → 0.4.0

package/dist/core/verdict/judge-baseline.js

@@ -0,0 +1,48 @@
+import { JUDGE_LATENCY_SLO } from './judge-runtime-config.js';
+const samples = [];
+const MAX_SAMPLES = 2_000;
+export function resetJudgeLatencySamples() {
+    samples.length = 0;
+}
+export function recordJudgeLatency(phase, latencyMs, at = Date.now()) {
+    if (!Number.isFinite(latencyMs) || latencyMs < 0) {
+        return;
+    }
+    samples.push({ phase, latencyMs, at });
+    if (samples.length > MAX_SAMPLES) {
+        samples.splice(0, samples.length - MAX_SAMPLES);
+    }
+}
+function percentile(values, p) {
+    if (values.length === 0) {
+        return 0;
+    }
+    const sorted = [...values].sort((a, b) => a - b);
+    const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
+    return sorted[Math.max(0, index)] ?? 0;
+}
+export function judgeLatencyPercentiles(phase) {
+    const values = samples
+        .filter((sample) => sample.phase === phase)
+        .map((sample) => sample.latencyMs);
+    return {
+        p50: percentile(values, 50),
+        p95: percentile(values, 95),
+        count: values.length,
+    };
+}
+export function judgeLatencySloReport() {
+    const measured = {
+        tier0: judgeLatencyPercentiles('tier0'),
+        tier1_spawn: judgeLatencyPercentiles('tier1_spawn'),
+        tier1_session: judgeLatencyPercentiles('tier1_session'),
+        tier1_connect: judgeLatencyPercentiles('tier1_connect'),
+        tier1_eval: judgeLatencyPercentiles('tier1_eval'),
+        tier1_parse: judgeLatencyPercentiles('tier1_parse'),
+    };
+    const spawnP95 = measured.tier1_spawn.count > 0 ? measured.tier1_spawn.p95 : JUDGE_LATENCY_SLO.spawnBaselineP95Ms;
+    const sessionP95 = measured.tier1_session.count > 0 ? measured.tier1_session.p95 : spawnP95;
+    const targetP95 = spawnP95 * (1 - JUDGE_LATENCY_SLO.tier1P95ReductionTarget);
+    const sessionMeetsTarget = sessionP95 <= targetP95 && sessionP95 <= JUDGE_LATENCY_SLO.sessionMaxP95Ms;
+    return { slo: JUDGE_LATENCY_SLO, measured, sessionMeetsTarget };
+}

package/dist/core/verdict/judge-broker-service.d.ts

@@ -0,0 +1,42 @@
+import type { CliJudgeRunCommand } from './judge-cli.js';
+import type { BelayJudgeSessionConfig } from './judge-runtime-config.js';
+import { type BrokerEvaluateRequest, type BrokerEvaluateResult, JudgeSessionBroker, stopRepoJudgeSessionBroker } from './judge-session-broker.js';
+import type { JudgeSessionResetReason } from './judge-session-guard.js';
+export declare const JUDGE_BROKER_SOCKET = "judge-broker.sock";
+export declare const JUDGE_BROKER_STATUS = "judge-broker.json";
+export declare const JUDGE_BROKER_PID = "judge-broker.pid";
+export declare const JUDGE_BROKER_SESSION = "judge-broker-session.json";
+export interface JudgeBrokerStatus {
+    pid: number;
+    repoRoot: string;
+    socketPath: string;
+    startedAt: string;
+}
+export interface JudgeBrokerPaths {
+    stateDir: string;
+    socketPath: string;
+    statusPath: string;
+    pidPath: string;
+    sessionConfigPath: string;
+}
+export declare function judgeBrokerPaths(stateDir: string): JudgeBrokerPaths;
+export declare function ensureJudgeBrokerDaemon(stateDir: string, repoRoot: string, sessionConfig: BelayJudgeSessionConfig): Promise<JudgeBrokerPaths>;
+export declare function cleanupJudgeBrokerArtifacts(paths: JudgeBrokerPaths): Promise<void>;
+export declare function stopJudgeBrokerDaemon(stateDir: string): Promise<number>;
+export declare function evaluateViaJudgeBroker(options: {
+    stateDir: string;
+    repoRoot: string;
+    sessionConfig: BelayJudgeSessionConfig;
+    request: BrokerEvaluateRequest;
+    timeoutMs: number;
+    runCommand?: CliJudgeRunCommand;
+}): Promise<BrokerEvaluateResult>;
+export declare function resetJudgeBrokerForTests(): void;
+export declare function invalidateJudgeSession(options: {
+    stateDir: string;
+    repoRoot: string;
+    sessionConfig: BelayJudgeSessionConfig;
+    sessionKey: string;
+    reason: JudgeSessionResetReason;
+}): Promise<void>;
+export { JudgeSessionBroker, stopRepoJudgeSessionBroker };

package/dist/core/verdict/judge-broker-service.js

@@ -0,0 +1,279 @@
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { unlink } from 'node:fs/promises';
+import net from 'node:net';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runCliJsonWithTimeouts } from './judge-cli.js';
+import { normalizeJudgeSessionConfig } from './judge-runtime-config.js';
+import { getRepoJudgeSessionBroker, JudgeSessionBroker, resetJudgeSessionBrokersForTests, stopRepoJudgeSessionBroker, } from './judge-session-broker.js';
+export const JUDGE_BROKER_SOCKET = 'judge-broker.sock';
+export const JUDGE_BROKER_STATUS = 'judge-broker.json';
+export const JUDGE_BROKER_PID = 'judge-broker.pid';
+export const JUDGE_BROKER_SESSION = 'judge-broker-session.json';
+export function judgeBrokerPaths(stateDir) {
+    return {
+        stateDir,
+        socketPath: path.join(stateDir, JUDGE_BROKER_SOCKET),
+        statusPath: path.join(stateDir, JUDGE_BROKER_STATUS),
+        pidPath: path.join(stateDir, JUDGE_BROKER_PID),
+        sessionConfigPath: path.join(stateDir, JUDGE_BROKER_SESSION),
+    };
+}
+function daemonScriptPath() {
+    return fileURLToPath(new URL('../../judge-broker-daemon.js', import.meta.url));
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function useInProcessBroker(env = process.env) {
+    return Boolean(env.VITEST || env.VITEST_WORKER_ID || env.BELAY_JUDGE_BROKER_IN_PROCESS === '1');
+}
+async function readBrokerStatus(statusPath) {
+    if (!existsSync(statusPath)) {
+        return null;
+    }
+    try {
+        const { readFile } = await import('node:fs/promises');
+        const raw = JSON.parse(await readFile(statusPath, 'utf8'));
+        if (typeof raw.pid !== 'number' || typeof raw.socketPath !== 'string') {
+            return null;
+        }
+        return raw;
+    }
+    catch {
+        return null;
+    }
+}
+async function readBrokerSessionConfig(sessionConfigPath) {
+    if (!existsSync(sessionConfigPath)) {
+        return null;
+    }
+    try {
+        const { readFile } = await import('node:fs/promises');
+        const raw = JSON.parse(await readFile(sessionConfigPath, 'utf8'));
+        return normalizeJudgeSessionConfig({ ...raw, enabled: true });
+    }
+    catch {
+        return null;
+    }
+}
+function brokerSessionConfigPayload(sessionConfig) {
+    return JSON.stringify(normalizeJudgeSessionConfig({ ...sessionConfig, enabled: true }));
+}
+async function writeBrokerSessionConfig(paths, sessionConfig) {
+    const { writeFile } = await import('node:fs/promises');
+    await writeFile(paths.sessionConfigPath, brokerSessionConfigPayload(sessionConfig), {
+        encoding: 'utf8',
+        mode: 0o600,
+    });
+}
+async function connectBrokerSocket(socketPath, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        const socket = net.createConnection(socketPath);
+        const timer = setTimeout(() => {
+            socket.destroy();
+            reject(new Error('judge broker connect timed out'));
+        }, timeoutMs);
+        socket.once('connect', () => {
+            clearTimeout(timer);
+            resolve(socket);
+        });
+        socket.once('error', (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+    });
+}
+async function rpcEvaluate(socketPath, request, timeoutMs, connectTimeoutMs) {
+    const socket = await connectBrokerSocket(socketPath, connectTimeoutMs);
+    const id = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+    const payload = JSON.stringify({
+        type: 'evaluate',
+        id,
+        request,
+        timeoutMs,
+    });
+    return new Promise((resolve, reject) => {
+        let buffer = '';
+        const timer = setTimeout(() => {
+            socket.destroy();
+            reject(new Error('judge broker evaluate timed out'));
+        }, timeoutMs + connectTimeoutMs);
+        const onData = (chunk) => {
+            buffer += String(chunk);
+            const lines = buffer.split('\n');
+            buffer = lines.pop() ?? '';
+            for (const line of lines) {
+                if (!line.trim()) {
+                    continue;
+                }
+                try {
+                    const message = JSON.parse(line);
+                    if (message.id !== id) {
+                        continue;
+                    }
+                    clearTimeout(timer);
+                    socket.destroy();
+                    if (message.type === 'error') {
+                        reject(new Error(message.error ?? 'judge broker error'));
+                        return;
+                    }
+                    if (message.result) {
+                        resolve(message.result);
+                    }
+                }
+                catch {
+                    // wait for more data
+                }
+            }
+        };
+        socket.on('data', onData);
+        socket.on('error', (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+        socket.write(`${payload}\n`);
+    });
+}
+export async function ensureJudgeBrokerDaemon(stateDir, repoRoot, sessionConfig) {
+    const paths = judgeBrokerPaths(stateDir);
+    const brokerSessionConfig = normalizeJudgeSessionConfig({ ...sessionConfig, enabled: true });
+    const nextConfigPayload = brokerSessionConfigPayload(brokerSessionConfig);
+    const existingConfig = await readBrokerSessionConfig(paths.sessionConfigPath);
+    const configChanged = existingConfig !== null && brokerSessionConfigPayload(existingConfig) !== nextConfigPayload;
+    const status = await readBrokerStatus(paths.statusPath);
+    if (status && isProcessAlive(status.pid) && existsSync(paths.socketPath) && !configChanged) {
+        await writeBrokerSessionConfig(paths, brokerSessionConfig);
+        return paths;
+    }
+    if (status?.pid && isProcessAlive(status.pid)) {
+        await stopJudgeBrokerDaemon(stateDir);
+    }
+    else {
+        await cleanupJudgeBrokerArtifacts(paths);
+    }
+    await writeBrokerSessionConfig(paths, brokerSessionConfig);
+    const child = spawn(process.execPath, [daemonScriptPath()], {
+        detached: true,
+        stdio: 'ignore',
+        env: {
+            ...process.env,
+            BELAY_JUDGE_BROKER_REPO_ROOT: repoRoot,
+            BELAY_JUDGE_BROKER_STATE_DIR: stateDir,
+        },
+    });
+    child.unref();
+    const deadline = Date.now() + sessionConfig.connectTimeoutMs;
+    while (Date.now() < deadline) {
+        if (existsSync(paths.socketPath)) {
+            const live = await readBrokerStatus(paths.statusPath);
+            if (live && isProcessAlive(live.pid)) {
+                return paths;
+            }
+        }
+        await new Promise((resolve) => setTimeout(resolve, 25));
+    }
+    throw new Error('judge broker daemon failed to start');
+}
+export async function cleanupJudgeBrokerArtifacts(paths) {
+    for (const artifact of [
+        paths.socketPath,
+        paths.statusPath,
+        paths.pidPath,
+        paths.sessionConfigPath,
+    ]) {
+        if (existsSync(artifact)) {
+            await unlink(artifact).catch(() => undefined);
+        }
+    }
+}
+export async function stopJudgeBrokerDaemon(stateDir) {
+    const paths = judgeBrokerPaths(stateDir);
+    const status = await readBrokerStatus(paths.statusPath);
+    if (status?.pid && isProcessAlive(status.pid)) {
+        try {
+            process.kill(status.pid, 'SIGTERM');
+        }
+        catch {
+            // already stopped
+        }
+    }
+    await cleanupJudgeBrokerArtifacts(paths);
+    return status?.pid ? 1 : 0;
+}
+export async function evaluateViaJudgeBroker(options) {
+    const runCommand = options.runCommand ??
+        ((invocation, timeout) => runCliJsonWithTimeouts(invocation, { evalTimeoutMs: timeout }));
+    if (useInProcessBroker()) {
+        const broker = getRepoJudgeSessionBroker(options.repoRoot, {
+            config: options.sessionConfig,
+            runCommand,
+        });
+        return broker.evaluate(options.request, options.timeoutMs);
+    }
+    const paths = await ensureJudgeBrokerDaemon(options.stateDir, options.repoRoot, options.sessionConfig);
+    return rpcEvaluate(paths.socketPath, options.request, options.timeoutMs, options.sessionConfig.connectTimeoutMs);
+}
+export function resetJudgeBrokerForTests() {
+    resetJudgeSessionBrokersForTests();
+}
+async function rpcInvalidate(socketPath, sessionKey, reason, connectTimeoutMs) {
+    const socket = await connectBrokerSocket(socketPath, connectTimeoutMs);
+    const id = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
+    const payload = JSON.stringify({ type: 'invalidate', id, sessionKey, reason });
+    await new Promise((resolve, reject) => {
+        let buffer = '';
+        const timer = setTimeout(() => {
+            socket.destroy();
+            reject(new Error('judge broker invalidate timed out'));
+        }, connectTimeoutMs);
+        const onData = (chunk) => {
+            buffer += String(chunk);
+            const lines = buffer.split('\n');
+            buffer = lines.pop() ?? '';
+            for (const line of lines) {
+                if (!line.trim()) {
+                    continue;
+                }
+                try {
+                    const message = JSON.parse(line);
+                    if (message.id !== id) {
+                        continue;
+                    }
+                    clearTimeout(timer);
+                    socket.destroy();
+                    resolve();
+                }
+                catch {
+                    // wait for more data
+                }
+            }
+        };
+        socket.on('data', onData);
+        socket.on('error', (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+        socket.write(`${payload}\n`);
+    });
+}
+export async function invalidateJudgeSession(options) {
+    if (useInProcessBroker()) {
+        const broker = getRepoJudgeSessionBroker(options.repoRoot, {
+            config: { ...options.sessionConfig, enabled: true },
+            runCommand: (invocation, timeout) => runCliJsonWithTimeouts(invocation, { evalTimeoutMs: timeout }),
+        });
+        broker.invalidateSession(options.sessionKey, options.reason);
+        return;
+    }
+    const paths = await ensureJudgeBrokerDaemon(options.stateDir, options.repoRoot, options.sessionConfig);
+    await rpcInvalidate(paths.socketPath, options.sessionKey, options.reason, options.sessionConfig.connectTimeoutMs);
+}
+export { JudgeSessionBroker, stopRepoJudgeSessionBroker };

package/dist/core/verdict/judge-cli-fingerprint.d.ts

@@ -0,0 +1,5 @@
+import type { JudgeProviderId } from './judge-catalog.js';
+export type CliFingerprintResolver = (providerId: Exclude<JudgeProviderId, 'ollama'>) => Promise<string>;
+export declare function setCliFingerprintResolverForTests(resolver: CliFingerprintResolver | null): void;
+export declare function resolveCliVersionFingerprint(providerId: Exclude<JudgeProviderId, 'ollama'>, timeoutMs?: number): Promise<string>;
+export declare function resetCliFingerprintCache(): void;

package/dist/core/verdict/judge-cli-fingerprint.js

@@ -0,0 +1,61 @@
+import { spawn } from 'node:child_process';
+const CLI_BINARIES = {
+    codex: 'codex',
+    cursor: 'cursor-agent',
+    claude: 'claude',
+};
+let fingerprintResolver = null;
+const fingerprintCache = new Map();
+export function setCliFingerprintResolverForTests(resolver) {
+    fingerprintResolver = resolver;
+    fingerprintCache.clear();
+}
+async function runVersionProbe(binary, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(binary, ['--version'], { stdio: ['ignore', 'pipe', 'pipe'] });
+        let stdout = '';
+        let stderr = '';
+        const timer = setTimeout(() => {
+            child.kill('SIGTERM');
+            reject(new Error(`${binary} --version timed out`));
+        }, timeoutMs);
+        child.stdout.on('data', (chunk) => {
+            stdout += String(chunk);
+        });
+        child.stderr.on('data', (chunk) => {
+            stderr += String(chunk);
+        });
+        child.on('error', (error) => {
+            clearTimeout(timer);
+            reject(error);
+        });
+        child.on('close', (code) => {
+            clearTimeout(timer);
+            const output = (stdout || stderr).trim();
+            if (code === 0 && output) {
+                resolve(output.split(/\r?\n/)[0]?.trim() || output);
+                return;
+            }
+            reject(new Error(stderr.trim() || `${binary} --version exited with code ${code ?? 'unknown'}`));
+        });
+    });
+}
+export async function resolveCliVersionFingerprint(providerId, timeoutMs = 3_000) {
+    const cacheKey = providerId;
+    const cached = fingerprintCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    if (fingerprintResolver) {
+        const resolved = await fingerprintResolver(providerId);
+        fingerprintCache.set(cacheKey, resolved);
+        return resolved;
+    }
+    const binary = CLI_BINARIES[providerId];
+    const version = await runVersionProbe(binary, timeoutMs);
+    fingerprintCache.set(cacheKey, version);
+    return version;
+}
+export function resetCliFingerprintCache() {
+    fingerprintCache.clear();
+}

package/dist/core/verdict/judge-cli.d.ts

@@ -1,6 +1,7 @@
 import type { ScrubOptions } from '../types.js';
 import { parseTier1Json, type TracedTier1Judge } from './judge.js';
 import type { JudgeProviderId } from './judge-catalog.js';
+import type { BelayJudgeRuntimeConfig } from './judge-runtime-config.js';
 export interface CliJudgeOptions {
     providerId: JudgeProviderId;
     modelRequested: string;
@@ -9,6 +10,10 @@ export interface CliJudgeOptions {
     sensitivePaths: string[];
     scrubOptions: ScrubOptions;
     runCliCommand?: CliJudgeRunCommand;
+    runtime?: BelayJudgeRuntimeConfig;
+    repoRoot?: string;
+    stateDir?: string;
+    judgeMode?: string;
 }
 export interface CliInvocation {
     binary: string;
@@ -16,8 +21,26 @@ export interface CliInvocation {
     stdin?: string;
 }
 export type CliJudgeRunCommand = (invocation: CliInvocation, timeoutMs: number) => Promise<string>;
+export type CliRunErrorKind = 'timeout' | 'spawn_error' | 'exit_nonzero';
+export declare class CliRunError extends Error {
+    kind: CliRunErrorKind;
+    exitCode?: number;
+    stderr?: string;
+    stdout?: string;
+    command?: string;
+    constructor(kind: CliRunErrorKind, message: string, details?: {
+        exitCode?: number;
+        stderr?: string;
+        stdout?: string;
+        command?: string;
+    });
+}
 export declare function buildCliInvocation(providerId: Exclude<JudgeProviderId, 'ollama'>, prompt: string, model: string): CliInvocation;
 export declare function parseCliJudgeOutput(providerId: Exclude<JudgeProviderId, 'ollama'>, raw: string): ReturnType<typeof parseTier1Json>;
+export declare function runCliJsonWithTimeouts(invocation: CliInvocation, budgets: {
+    connectTimeoutMs?: number;
+    evalTimeoutMs: number;
+}): Promise<string>;
 export declare function createCodexCliJudge(options: Omit<CliJudgeOptions, 'providerId'>): TracedTier1Judge;
 export declare function createCursorCliJudge(options: Omit<CliJudgeOptions, 'providerId'>): TracedTier1Judge;
 export declare function createClaudeCliJudge(options: Omit<CliJudgeOptions, 'providerId'>): TracedTier1Judge;