@guilz-dev/belay 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cursor/runtime-entry.js +1 -0
- package/dist/adapters/shared/gate-runtime.d.ts +11 -14
- package/dist/adapters/shared/gate-runtime.js +277 -19
- package/dist/bundle/claude-runtime.mjs +3138 -699
- package/dist/bundle/codex-runtime.mjs +3162 -711
- package/dist/bundle/cursor-runtime.mjs +3140 -701
- package/dist/cli.js +137 -5
- package/dist/commands/approve.d.ts +1 -0
- package/dist/commands/approve.js +45 -26
- package/dist/commands/config.js +135 -52
- package/dist/commands/doctor.js +11 -1
- package/dist/commands/explain.js +6 -1
- package/dist/commands/harvest.d.ts +29 -0
- package/dist/commands/harvest.js +72 -0
- package/dist/commands/judge.d.ts +11 -1
- package/dist/commands/judge.js +21 -2
- package/dist/commands/metrics.js +34 -2
- package/dist/commands/quality.d.ts +38 -0
- package/dist/commands/quality.js +105 -0
- package/dist/commands/simulate.d.ts +1 -0
- package/dist/commands/simulate.js +7 -2
- package/dist/commands/standing-allow.d.ts +10 -0
- package/dist/commands/standing-allow.js +26 -0
- package/dist/commands/tui.d.ts +48 -0
- package/dist/commands/tui.js +150 -0
- package/dist/config-io.js +10 -5
- package/dist/core/approval-replay-cli.d.ts +7 -0
- package/dist/core/approval-replay-cli.js +36 -0
- package/dist/core/approval-replay.d.ts +45 -0
- package/dist/core/approval-replay.js +141 -0
- package/dist/core/approval-service.d.ts +15 -0
- package/dist/core/approval-service.js +47 -2
- package/dist/core/approval.d.ts +24 -3
- package/dist/core/approval.js +47 -4
- package/dist/core/audit-analysis.d.ts +7 -1
- package/dist/core/audit-analysis.js +111 -0
- package/dist/core/audit-metrics.d.ts +7 -0
- package/dist/core/audit-metrics.js +24 -4
- package/dist/core/audit-query.d.ts +1 -0
- package/dist/core/audit-query.js +3 -0
- package/dist/core/audit-replay-context.d.ts +35 -0
- package/dist/core/audit-replay-context.js +88 -0
- package/dist/core/audit-types.d.ts +24 -1
- package/dist/core/audit-types.js +1 -1
- package/dist/core/capability/index.d.ts +2 -1
- package/dist/core/capability/index.js +1 -0
- package/dist/core/capability/paths.d.ts +2 -2
- package/dist/core/capability/paths.js +9 -9
- package/dist/core/capability/trusted-workspace-roots.d.ts +25 -0
- package/dist/core/capability/trusted-workspace-roots.js +149 -0
- package/dist/core/capability/types.d.ts +11 -1
- package/dist/core/capability-approval.d.ts +2 -1
- package/dist/core/capability-approval.js +100 -2
- package/dist/core/classify-subagent.js +2 -20
- package/dist/core/classify-tool.js +32 -3
- package/dist/core/config.d.ts +18 -0
- package/dist/core/config.js +49 -0
- package/dist/core/gate-engine.js +6 -6
- package/dist/core/harvest.d.ts +53 -0
- package/dist/core/harvest.js +276 -0
- package/dist/core/index.d.ts +9 -3
- package/dist/core/index.js +6 -2
- package/dist/core/judge-doctor.d.ts +3 -0
- package/dist/core/judge-doctor.js +54 -0
- package/dist/core/path-utils.d.ts +11 -0
- package/dist/core/path-utils.js +56 -6
- package/dist/core/reclassify.d.ts +3 -0
- package/dist/core/reclassify.js +44 -14
- package/dist/core/replay-scrub.d.ts +9 -0
- package/dist/core/replay-scrub.js +43 -0
- package/dist/core/shell-tokenizer.d.ts +2 -0
- package/dist/core/shell-tokenizer.js +82 -23
- package/dist/core/standing-allow.d.ts +50 -0
- package/dist/core/standing-allow.js +175 -0
- package/dist/core/types.d.ts +15 -0
- package/dist/core/verdict/adapter.js +10 -1
- package/dist/core/verdict/containment.d.ts +4 -3
- package/dist/core/verdict/containment.js +17 -12
- package/dist/core/verdict/judge-audit.d.ts +1 -0
- package/dist/core/verdict/judge-audit.js +32 -1
- package/dist/core/verdict/judge-baseline.d.ts +20 -0
- package/dist/core/verdict/judge-baseline.js +48 -0
- package/dist/core/verdict/judge-broker-service.d.ts +42 -0
- package/dist/core/verdict/judge-broker-service.js +279 -0
- package/dist/core/verdict/judge-cli-fingerprint.d.ts +5 -0
- package/dist/core/verdict/judge-cli-fingerprint.js +61 -0
- package/dist/core/verdict/judge-cli.d.ts +23 -0
- package/dist/core/verdict/judge-cli.js +137 -16
- package/dist/core/verdict/judge-factory.js +13 -4
- package/dist/core/verdict/judge-provider-matrix.d.ts +20 -0
- package/dist/core/verdict/judge-provider-matrix.js +66 -0
- package/dist/core/verdict/judge-runtime-config.d.ts +42 -0
- package/dist/core/verdict/judge-runtime-config.js +92 -0
- package/dist/core/verdict/judge-session-broker.d.ts +48 -0
- package/dist/core/verdict/judge-session-broker.js +195 -0
- package/dist/core/verdict/judge-session-guard.d.ts +27 -0
- package/dist/core/verdict/judge-session-guard.js +91 -0
- package/dist/core/verdict/judge-session-kill-switch.d.ts +11 -0
- package/dist/core/verdict/judge-session-kill-switch.js +43 -0
- package/dist/core/verdict/judge-session-mutex.d.ts +9 -0
- package/dist/core/verdict/judge-session-mutex.js +23 -0
- package/dist/core/verdict/judge-shadow.d.ts +24 -0
- package/dist/core/verdict/judge-shadow.js +116 -0
- package/dist/core/verdict/judge-transport.d.ts +40 -0
- package/dist/core/verdict/judge-transport.js +258 -0
- package/dist/core/verdict/judge.d.ts +14 -1
- package/dist/core/verdict/judge.js +6 -5
- package/dist/core/verdict/parser.js +6 -1
- package/dist/core/verdict/types.d.ts +13 -0
- package/dist/core/verdict/verdict.js +7 -4
- package/dist/corpus/adversarial-probe.d.ts +86 -0
- package/dist/corpus/adversarial-probe.js +220 -0
- package/dist/corpus/evaluate.d.ts +45 -12
- package/dist/corpus/evaluate.js +62 -5
- package/dist/corpus/gates.d.ts +35 -0
- package/dist/corpus/gates.js +81 -0
- package/dist/corpus/judge-accuracy.d.ts +8 -0
- package/dist/corpus/judge-accuracy.js +39 -0
- package/dist/corpus/must-allow-commands.d.ts +5 -0
- package/dist/corpus/must-allow-commands.js +13 -0
- package/dist/corpus/mutators.d.ts +28 -0
- package/dist/corpus/mutators.js +151 -0
- package/dist/corpus/ratchet.d.ts +42 -0
- package/dist/corpus/ratchet.js +116 -0
- package/dist/corpus/runtime-match.d.ts +19 -0
- package/dist/corpus/runtime-match.js +45 -0
- package/dist/corpus/standing-allow-catalog.generated.d.ts +13 -0
- package/dist/corpus/standing-allow-catalog.generated.js +79 -0
- package/dist/corpus/types.d.ts +47 -0
- package/dist/corpus/types.js +157 -0
- package/dist/installer.js +4 -1
- package/dist/judge-broker-daemon.d.ts +1 -0
- package/dist/judge-broker-daemon.js +123 -0
- package/dist/services/sandbox-service.d.ts +1 -0
- package/dist/services/sandbox-service.js +2 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +7 -3
- package/skills/belay/SKILL.md +7 -1
- package/skills/belay/belay-approve.md +17 -0
|
@@ -1,11 +1,31 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { buildTier1Prompt, parseTier1Json, prescanInterpreterCode, } from './judge.js';
|
|
3
3
|
import { scrubOutboundForJudge } from './judge-outbound.js';
|
|
4
|
+
import { DEFAULT_JUDGE_RUNTIME_CONFIG } from './judge-runtime-config.js';
|
|
5
|
+
import { transportFallbackToFailClosedReason } from './judge-session-guard.js';
|
|
6
|
+
import { evaluateWithJudgeTransport } from './judge-transport.js';
|
|
7
|
+
export class CliRunError extends Error {
|
|
8
|
+
kind;
|
|
9
|
+
exitCode;
|
|
10
|
+
stderr;
|
|
11
|
+
stdout;
|
|
12
|
+
command;
|
|
13
|
+
constructor(kind, message, details = {}) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = 'CliRunError';
|
|
16
|
+
this.kind = kind;
|
|
17
|
+
this.exitCode = details.exitCode;
|
|
18
|
+
this.stderr = details.stderr;
|
|
19
|
+
this.stdout = details.stdout;
|
|
20
|
+
this.command = details.command;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
4
23
|
const CLI_BINARIES = {
|
|
5
24
|
codex: 'codex',
|
|
6
25
|
cursor: 'cursor-agent',
|
|
7
26
|
claude: 'claude',
|
|
8
27
|
};
|
|
28
|
+
const FORCE_KILL_GRACE_MS = 250;
|
|
9
29
|
function failClosedVerdict(reason) {
|
|
10
30
|
return {
|
|
11
31
|
local_recoverable: false,
|
|
@@ -162,17 +182,67 @@ export function parseCliJudgeOutput(providerId, raw) {
|
|
|
162
182
|
return parseCliJsonLines(raw);
|
|
163
183
|
}
|
|
164
184
|
async function runCliJson(invocation, timeoutMs) {
|
|
185
|
+
return runCliJsonWithTimeouts(invocation, { evalTimeoutMs: timeoutMs });
|
|
186
|
+
}
|
|
187
|
+
export async function runCliJsonWithTimeouts(invocation, budgets) {
|
|
165
188
|
const { binary, args, stdin } = invocation;
|
|
189
|
+
const connectTimeoutMs = budgets.connectTimeoutMs ?? 0;
|
|
190
|
+
if (connectTimeoutMs > 0) {
|
|
191
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
192
|
+
}
|
|
166
193
|
return new Promise((resolve, reject) => {
|
|
167
194
|
const child = spawn(binary, args, {
|
|
168
195
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
169
196
|
});
|
|
197
|
+
const command = [binary, ...args].join(' ');
|
|
170
198
|
let stdout = '';
|
|
171
199
|
let stderr = '';
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
200
|
+
let settled = false;
|
|
201
|
+
let timer = null;
|
|
202
|
+
let forceKillTimer = null;
|
|
203
|
+
const cleanup = () => {
|
|
204
|
+
if (timer) {
|
|
205
|
+
clearTimeout(timer);
|
|
206
|
+
}
|
|
207
|
+
if (forceKillTimer) {
|
|
208
|
+
clearTimeout(forceKillTimer);
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
const rejectOnce = (error) => {
|
|
212
|
+
if (settled)
|
|
213
|
+
return;
|
|
214
|
+
settled = true;
|
|
215
|
+
cleanup();
|
|
216
|
+
reject(error);
|
|
217
|
+
};
|
|
218
|
+
const resolveOnce = (value) => {
|
|
219
|
+
if (settled)
|
|
220
|
+
return;
|
|
221
|
+
settled = true;
|
|
222
|
+
cleanup();
|
|
223
|
+
resolve(value);
|
|
224
|
+
};
|
|
225
|
+
timer = setTimeout(() => {
|
|
226
|
+
rejectOnce(new CliRunError('timeout', `${binary} timed out`, {
|
|
227
|
+
stderr: stderr.trim() || undefined,
|
|
228
|
+
stdout: stdout.trim() || undefined,
|
|
229
|
+
command,
|
|
230
|
+
}));
|
|
231
|
+
try {
|
|
232
|
+
child.kill('SIGTERM');
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
forceKillTimer = setTimeout(() => {
|
|
238
|
+
try {
|
|
239
|
+
child.kill('SIGKILL');
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
// process already exited
|
|
243
|
+
}
|
|
244
|
+
}, FORCE_KILL_GRACE_MS);
|
|
245
|
+
}, budgets.evalTimeoutMs);
|
|
176
246
|
child.stdout.on('data', (chunk) => {
|
|
177
247
|
stdout += String(chunk);
|
|
178
248
|
});
|
|
@@ -184,16 +254,23 @@ async function runCliJson(invocation, timeoutMs) {
|
|
|
184
254
|
}
|
|
185
255
|
child.stdin.end();
|
|
186
256
|
child.on('error', (error) => {
|
|
187
|
-
|
|
188
|
-
|
|
257
|
+
rejectOnce(new CliRunError('spawn_error', error instanceof Error ? error.message : String(error), {
|
|
258
|
+
command,
|
|
259
|
+
}));
|
|
189
260
|
});
|
|
190
261
|
child.on('close', (code) => {
|
|
191
|
-
|
|
262
|
+
if (settled)
|
|
263
|
+
return;
|
|
192
264
|
if (code === 0 && stdout.trim()) {
|
|
193
|
-
|
|
265
|
+
resolveOnce(stdout.trim());
|
|
194
266
|
return;
|
|
195
267
|
}
|
|
196
|
-
|
|
268
|
+
rejectOnce(new CliRunError('exit_nonzero', stderr.trim() || `${binary} exited with code ${code ?? 'unknown'}`, {
|
|
269
|
+
exitCode: typeof code === 'number' ? code : undefined,
|
|
270
|
+
stderr: stderr.trim() || undefined,
|
|
271
|
+
stdout: stdout.trim() || undefined,
|
|
272
|
+
command,
|
|
273
|
+
}));
|
|
197
274
|
});
|
|
198
275
|
});
|
|
199
276
|
}
|
|
@@ -202,6 +279,10 @@ function createCliJudge(options) {
|
|
|
202
279
|
const timeoutMs = options.timeoutMs ?? 25000;
|
|
203
280
|
const providerId = options.providerId;
|
|
204
281
|
const runCommand = options.runCliCommand ?? runCliJson;
|
|
282
|
+
const runtime = options.runtime ?? DEFAULT_JUDGE_RUNTIME_CONFIG;
|
|
283
|
+
const repoRoot = options.repoRoot ?? process.cwd();
|
|
284
|
+
const stateDir = options.stateDir ?? repoRoot;
|
|
285
|
+
const judgeMode = options.judgeMode ?? 'enforce';
|
|
205
286
|
const judge = {
|
|
206
287
|
lastTrace: initialTrace(options),
|
|
207
288
|
async evaluate(input) {
|
|
@@ -233,27 +314,66 @@ function createCliJudge(options) {
|
|
|
233
314
|
}
|
|
234
315
|
const prompt = buildTier1Prompt(scrubbed.text);
|
|
235
316
|
try {
|
|
236
|
-
const
|
|
237
|
-
|
|
238
|
-
|
|
317
|
+
const transportResult = await evaluateWithJudgeTransport({
|
|
318
|
+
prompt,
|
|
319
|
+
context: {
|
|
320
|
+
providerId,
|
|
321
|
+
model: options.modelResolved,
|
|
322
|
+
repoRoot: input.context.repoRoot || repoRoot,
|
|
323
|
+
stateDir,
|
|
324
|
+
judgeMode,
|
|
325
|
+
runtime,
|
|
326
|
+
judgeTimeoutMs: timeoutMs,
|
|
327
|
+
},
|
|
328
|
+
}, { runCommand });
|
|
329
|
+
const traceTransport = transportResult.transport === 'session'
|
|
330
|
+
? `${providerId}-cli-session`
|
|
331
|
+
: transport;
|
|
332
|
+
if (!transportResult.verdict) {
|
|
333
|
+
const fallbackReason = transportResult.fallbackReason ??
|
|
334
|
+
`${providerId}_cli_parse_error`;
|
|
239
335
|
judge.lastTrace = {
|
|
240
336
|
provider: 'fallback',
|
|
241
337
|
modelRequested: options.modelRequested,
|
|
242
338
|
modelResolved: options.modelResolved,
|
|
243
339
|
latencyMs: Date.now() - started,
|
|
244
|
-
fallbackReason
|
|
340
|
+
fallbackReason,
|
|
341
|
+
judgeFallbackReason: fallbackReason,
|
|
342
|
+
judgeSessionUsed: transportResult.sessionUsed,
|
|
343
|
+
judgeSessionReused: transportResult.sessionReused,
|
|
344
|
+
judgeSessionRefHash: transportResult.sessionRefHash,
|
|
345
|
+
judgeSessionResetReason: transportResult.sessionResetReason,
|
|
346
|
+
judgeConnectMs: transportResult.connectMs,
|
|
347
|
+
judgeEvalMs: transportResult.evalMs,
|
|
348
|
+
judgeParseMs: transportResult.parseMs,
|
|
349
|
+
judgeShadowCompared: transportResult.shadowCompared,
|
|
350
|
+
judgeShadowMismatch: transportResult.shadowMismatch,
|
|
351
|
+
judgeShadowMismatchRateWindow: transportResult.shadowMismatchRateWindow,
|
|
352
|
+
judgeKillSwitchTriggered: transportResult.killSwitchTriggered,
|
|
245
353
|
};
|
|
246
|
-
return failClosedVerdict(
|
|
354
|
+
return failClosedVerdict(transportFallbackToFailClosedReason(providerId, fallbackReason));
|
|
247
355
|
}
|
|
248
356
|
judge.lastTrace = {
|
|
249
357
|
provider: traceProvider(options.providerId),
|
|
250
358
|
modelRequested: options.modelRequested,
|
|
251
359
|
modelResolved: options.modelResolved,
|
|
252
360
|
latencyMs: Date.now() - started,
|
|
253
|
-
transport,
|
|
361
|
+
transport: traceTransport,
|
|
254
362
|
outboundRedacted: true,
|
|
363
|
+
judgeSessionUsed: transportResult.sessionUsed,
|
|
364
|
+
judgeSessionReused: transportResult.sessionReused,
|
|
365
|
+
judgeSessionRefHash: transportResult.sessionRefHash,
|
|
366
|
+
judgeSessionResetReason: transportResult.sessionResetReason,
|
|
367
|
+
judgeFallbackReason: transportResult.fallbackReason,
|
|
368
|
+
judgeConnectMs: transportResult.connectMs,
|
|
369
|
+
judgeEvalMs: transportResult.evalMs,
|
|
370
|
+
judgeParseMs: transportResult.parseMs,
|
|
371
|
+
judgeShadowCompared: transportResult.shadowCompared,
|
|
372
|
+
judgeShadowMismatch: transportResult.shadowMismatch,
|
|
373
|
+
judgeShadowMismatchRateWindow: transportResult.shadowMismatchRateWindow,
|
|
374
|
+
judgeKillSwitchTriggered: transportResult.killSwitchTriggered,
|
|
255
375
|
};
|
|
256
|
-
return
|
|
376
|
+
return transportResult.verdict;
|
|
257
377
|
}
|
|
258
378
|
catch {
|
|
259
379
|
judge.lastTrace = {
|
|
@@ -262,6 +382,7 @@ function createCliJudge(options) {
|
|
|
262
382
|
modelResolved: options.modelResolved,
|
|
263
383
|
latencyMs: Date.now() - started,
|
|
264
384
|
fallbackReason: `${providerId}_cli_unavailable`,
|
|
385
|
+
judgeFallbackReason: `${providerId}_cli_unavailable`,
|
|
265
386
|
};
|
|
266
387
|
return failClosedVerdict(`${providerId}_cli_unavailable`);
|
|
267
388
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { repoLocalStateDirFor } from '../../config-io.js';
|
|
4
|
-
import { normalizeJudgeProvider, scrubOptionsFromConfig } from '../config.js';
|
|
4
|
+
import { belayStateDir, normalizeJudgeProvider, scrubOptionsFromConfig } from '../config.js';
|
|
5
5
|
import { resolveJudgeCredential } from '../judge-api-key.js';
|
|
6
6
|
import { hasValidCloudConsent, isCloudJudgeConfig } from '../judge-config.js';
|
|
7
7
|
import { rejectDeprecatedJudgeModelAuto } from '../judge-model-policy.js';
|
|
@@ -63,7 +63,8 @@ function providerIdFromJudge(judge) {
|
|
|
63
63
|
}
|
|
64
64
|
return inferProviderIdFromConfig(judge);
|
|
65
65
|
}
|
|
66
|
-
function createCliJudgeForProvider(providerId, judgeConfig, config) {
|
|
66
|
+
function createCliJudgeForProvider(providerId, judgeConfig, config, repoRoot) {
|
|
67
|
+
const resolvedRepoRoot = repoRoot ?? process.cwd();
|
|
67
68
|
const { resolved } = resolveJudgeModel(judgeConfig);
|
|
68
69
|
const options = {
|
|
69
70
|
modelRequested: judgeConfig.model,
|
|
@@ -71,6 +72,10 @@ function createCliJudgeForProvider(providerId, judgeConfig, config) {
|
|
|
71
72
|
timeoutMs: judgeConfig.timeoutMs,
|
|
72
73
|
sensitivePaths: config.classifier.sensitivePaths,
|
|
73
74
|
scrubOptions: scrubOptionsFromConfig(config),
|
|
75
|
+
runtime: judgeConfig.runtime,
|
|
76
|
+
repoRoot: resolvedRepoRoot,
|
|
77
|
+
stateDir: belayStateDir(config, repoLocalStateDirFor(resolvedRepoRoot, config)),
|
|
78
|
+
judgeMode: config.mode,
|
|
74
79
|
};
|
|
75
80
|
if (providerId === 'codex') {
|
|
76
81
|
return createCodexCliJudge(options);
|
|
@@ -99,11 +104,12 @@ export function createJudgeFromConfig(config, options = {}) {
|
|
|
99
104
|
const catalogSpec = getJudgeProviderSpec(providerId);
|
|
100
105
|
const { resolved } = resolveJudgeModel(judgeConfig);
|
|
101
106
|
const runtime = detectJudgeRuntimeCapabilities(providerId);
|
|
107
|
+
const repoRoot = options.repoRoot ?? process.cwd();
|
|
102
108
|
if (provider === 'openai-compatible') {
|
|
103
109
|
const endpoint = judgeConfig.endpoint?.trim();
|
|
104
110
|
if (!endpoint && runtime.cliTransport) {
|
|
105
111
|
if (providerId === 'codex' || providerId === 'cursor') {
|
|
106
|
-
return createCliJudgeForProvider(providerId, judgeConfig, config);
|
|
112
|
+
return createCliJudgeForProvider(providerId, judgeConfig, config, repoRoot);
|
|
107
113
|
}
|
|
108
114
|
}
|
|
109
115
|
if (!endpoint) {
|
|
@@ -122,7 +128,6 @@ export function createJudgeFromConfig(config, options = {}) {
|
|
|
122
128
|
modelResolved: resolved,
|
|
123
129
|
});
|
|
124
130
|
}
|
|
125
|
-
const repoRoot = options.repoRoot ?? process.cwd();
|
|
126
131
|
const repoLocalStateDir = repoLocalStateDirFor(repoRoot, config);
|
|
127
132
|
return createOpenAiCompatibleJudge({
|
|
128
133
|
endpoint,
|
|
@@ -156,6 +161,10 @@ export function createJudgeFromConfig(config, options = {}) {
|
|
|
156
161
|
timeoutMs: judgeConfig.timeoutMs,
|
|
157
162
|
sensitivePaths: config.classifier.sensitivePaths,
|
|
158
163
|
scrubOptions: scrubOptionsFromConfig(config),
|
|
164
|
+
runtime: judgeConfig.runtime,
|
|
165
|
+
repoRoot,
|
|
166
|
+
stateDir: belayStateDir(config, repoLocalStateDirFor(repoRoot, config)),
|
|
167
|
+
judgeMode: config.mode,
|
|
159
168
|
});
|
|
160
169
|
}
|
|
161
170
|
return createFailClosedJudge({
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { JudgeProviderId } from './judge-catalog.js';
|
|
2
|
+
export type JudgeSessionFailureMode = 'parse_error' | 'non_json_response' | 'timeout' | 'cli_nonzero_exit' | 'io_error' | 'version_mismatch' | 'unsupported_resume' | 'unsafe_option_rejected';
|
|
3
|
+
export interface JudgeProviderSessionCapability {
|
|
4
|
+
providerId: Exclude<JudgeProviderId, 'ollama'>;
|
|
5
|
+
supportsSession: boolean;
|
|
6
|
+
resumeFlag: string | null;
|
|
7
|
+
createChatFlag: string | null;
|
|
8
|
+
readOnlyArgs: string[];
|
|
9
|
+
rejectedUnsafeOptions: string[];
|
|
10
|
+
failureModes: JudgeSessionFailureMode[];
|
|
11
|
+
notes: string;
|
|
12
|
+
}
|
|
13
|
+
export declare const JUDGE_PROVIDER_SESSION_MATRIX: Record<Exclude<JudgeProviderId, 'ollama'>, JudgeProviderSessionCapability>;
|
|
14
|
+
export declare function providerSupportsSession(providerId: Exclude<JudgeProviderId, 'ollama'>, allowlist: JudgeProviderId[]): boolean;
|
|
15
|
+
export declare function assertReadOnlyInvocationArgs(providerId: Exclude<JudgeProviderId, 'ollama'>, args: string[]): {
|
|
16
|
+
ok: true;
|
|
17
|
+
} | {
|
|
18
|
+
ok: false;
|
|
19
|
+
reason: JudgeSessionFailureMode;
|
|
20
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export const JUDGE_PROVIDER_SESSION_MATRIX = {
|
|
2
|
+
cursor: {
|
|
3
|
+
providerId: 'cursor',
|
|
4
|
+
supportsSession: true,
|
|
5
|
+
resumeFlag: '--resume',
|
|
6
|
+
createChatFlag: null,
|
|
7
|
+
readOnlyArgs: ['--mode', 'ask', '--sandbox', 'enabled', '--trust'],
|
|
8
|
+
rejectedUnsafeOptions: ['--force', '--yolo', '--dangerously-skip-permissions'],
|
|
9
|
+
failureModes: [
|
|
10
|
+
'parse_error',
|
|
11
|
+
'non_json_response',
|
|
12
|
+
'timeout',
|
|
13
|
+
'cli_nonzero_exit',
|
|
14
|
+
'io_error',
|
|
15
|
+
'version_mismatch',
|
|
16
|
+
],
|
|
17
|
+
notes: 'Pilot provider: ask mode + sandbox enabled; resume via --resume <chatId>.',
|
|
18
|
+
},
|
|
19
|
+
codex: {
|
|
20
|
+
providerId: 'codex',
|
|
21
|
+
supportsSession: false,
|
|
22
|
+
resumeFlag: null,
|
|
23
|
+
createChatFlag: null,
|
|
24
|
+
readOnlyArgs: ['--sandbox', 'read-only'],
|
|
25
|
+
rejectedUnsafeOptions: ['--dangerously-bypass-approvals-and-sandbox'],
|
|
26
|
+
failureModes: ['unsupported_resume', 'parse_error', 'timeout'],
|
|
27
|
+
notes: 'Session transport disabled by default; spawn read-only exec remains canonical.',
|
|
28
|
+
},
|
|
29
|
+
claude: {
|
|
30
|
+
providerId: 'claude',
|
|
31
|
+
supportsSession: false,
|
|
32
|
+
resumeFlag: '--continue',
|
|
33
|
+
createChatFlag: null,
|
|
34
|
+
readOnlyArgs: ['--permission-mode', 'plan', '--tools', '', '--bare'],
|
|
35
|
+
rejectedUnsafeOptions: ['--dangerously-skip-permissions'],
|
|
36
|
+
failureModes: ['unsupported_resume', 'parse_error', 'timeout'],
|
|
37
|
+
notes: 'Session transport gated behind allowlist after capability verification.',
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
export function providerSupportsSession(providerId, allowlist) {
|
|
41
|
+
const capability = JUDGE_PROVIDER_SESSION_MATRIX[providerId];
|
|
42
|
+
return capability.supportsSession && allowlist.includes(providerId);
|
|
43
|
+
}
|
|
44
|
+
export function assertReadOnlyInvocationArgs(providerId, args) {
|
|
45
|
+
const capability = JUDGE_PROVIDER_SESSION_MATRIX[providerId];
|
|
46
|
+
for (const unsafe of capability.rejectedUnsafeOptions) {
|
|
47
|
+
if (args.includes(unsafe)) {
|
|
48
|
+
return { ok: false, reason: 'unsafe_option_rejected' };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
for (let index = 0; index < capability.readOnlyArgs.length; index += 2) {
|
|
52
|
+
const flag = capability.readOnlyArgs[index];
|
|
53
|
+
const value = capability.readOnlyArgs[index + 1];
|
|
54
|
+
if (!flag) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
const flagIndex = args.indexOf(flag);
|
|
58
|
+
if (flagIndex === -1) {
|
|
59
|
+
return { ok: false, reason: 'unsafe_option_rejected' };
|
|
60
|
+
}
|
|
61
|
+
if (value !== undefined && args[flagIndex + 1] !== value) {
|
|
62
|
+
return { ok: false, reason: 'unsafe_option_rejected' };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return { ok: true };
|
|
66
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { JudgeProviderId } from './judge-catalog.js';
|
|
2
|
+
export interface BelayJudgeSessionConfig {
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
maxTurns: number;
|
|
5
|
+
maxAgeMs: number;
|
|
6
|
+
maxIdleMs: number;
|
|
7
|
+
maxPromptBytes: number;
|
|
8
|
+
providerAllowlist: JudgeProviderId[];
|
|
9
|
+
connectTimeoutMs: number;
|
|
10
|
+
evalTimeoutMs: number | null;
|
|
11
|
+
parseTimeoutMs: number;
|
|
12
|
+
maxSessionsPerProvider: number;
|
|
13
|
+
}
|
|
14
|
+
export interface BelayJudgeShadowConfig {
|
|
15
|
+
enabled: boolean;
|
|
16
|
+
sampleRate: number;
|
|
17
|
+
sampleRateMax: number;
|
|
18
|
+
dailyRequestCap: number;
|
|
19
|
+
mismatchRateThreshold: number;
|
|
20
|
+
providerAllowlist: JudgeProviderId[];
|
|
21
|
+
windowSize: number;
|
|
22
|
+
}
|
|
23
|
+
export interface BelayJudgeRuntimeConfig {
|
|
24
|
+
session: BelayJudgeSessionConfig;
|
|
25
|
+
shadow: BelayJudgeShadowConfig;
|
|
26
|
+
}
|
|
27
|
+
export declare const JUDGE_LATENCY_SLO: {
|
|
28
|
+
/** Target: Tier1 p95 at least 40% below spawn baseline. */
|
|
29
|
+
readonly tier1P95ReductionTarget: 0.4;
|
|
30
|
+
/** Reference spawn baseline (ms) from dogfood measurement. */
|
|
31
|
+
readonly spawnBaselineP95Ms: 25000;
|
|
32
|
+
readonly spawnBaselineP50Ms: 1300;
|
|
33
|
+
readonly tier0BaselineP95Ms: 60;
|
|
34
|
+
/** Session path must not exceed spawn p95 when enabled. */
|
|
35
|
+
readonly sessionMaxP95Ms: 15000;
|
|
36
|
+
};
|
|
37
|
+
export declare const DEFAULT_JUDGE_SESSION_CONFIG: BelayJudgeSessionConfig;
|
|
38
|
+
export declare const DEFAULT_JUDGE_SHADOW_CONFIG: BelayJudgeShadowConfig;
|
|
39
|
+
export declare const DEFAULT_JUDGE_RUNTIME_CONFIG: BelayJudgeRuntimeConfig;
|
|
40
|
+
export declare function normalizeJudgeRuntimeConfig(runtime: Partial<BelayJudgeRuntimeConfig> | undefined): BelayJudgeRuntimeConfig;
|
|
41
|
+
export declare function normalizeJudgeSessionConfig(session: Partial<BelayJudgeSessionConfig> | undefined): BelayJudgeSessionConfig;
|
|
42
|
+
export declare function resolveSessionEvalTimeoutMs(sessionConfig: BelayJudgeSessionConfig, judgeTimeoutMs: number): number;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
export const JUDGE_LATENCY_SLO = {
|
|
2
|
+
/** Target: Tier1 p95 at least 40% below spawn baseline. */
|
|
3
|
+
tier1P95ReductionTarget: 0.4,
|
|
4
|
+
/** Reference spawn baseline (ms) from dogfood measurement. */
|
|
5
|
+
spawnBaselineP95Ms: 25_000,
|
|
6
|
+
spawnBaselineP50Ms: 1_300,
|
|
7
|
+
tier0BaselineP95Ms: 60,
|
|
8
|
+
/** Session path must not exceed spawn p95 when enabled. */
|
|
9
|
+
sessionMaxP95Ms: 15_000,
|
|
10
|
+
};
|
|
11
|
+
export const DEFAULT_JUDGE_SESSION_CONFIG = {
|
|
12
|
+
enabled: false,
|
|
13
|
+
maxTurns: 32,
|
|
14
|
+
maxAgeMs: 30 * 60 * 1000,
|
|
15
|
+
maxIdleMs: 5 * 60 * 1000,
|
|
16
|
+
maxPromptBytes: 64 * 1024,
|
|
17
|
+
providerAllowlist: ['cursor'],
|
|
18
|
+
connectTimeoutMs: 5_000,
|
|
19
|
+
evalTimeoutMs: null,
|
|
20
|
+
parseTimeoutMs: 2_000,
|
|
21
|
+
maxSessionsPerProvider: 1,
|
|
22
|
+
};
|
|
23
|
+
export const DEFAULT_JUDGE_SHADOW_CONFIG = {
|
|
24
|
+
enabled: false,
|
|
25
|
+
sampleRate: 0.01,
|
|
26
|
+
sampleRateMax: 0.05,
|
|
27
|
+
dailyRequestCap: 500,
|
|
28
|
+
mismatchRateThreshold: 0.02,
|
|
29
|
+
providerAllowlist: ['cursor'],
|
|
30
|
+
windowSize: 100,
|
|
31
|
+
};
|
|
32
|
+
export const DEFAULT_JUDGE_RUNTIME_CONFIG = {
|
|
33
|
+
session: { ...DEFAULT_JUDGE_SESSION_CONFIG },
|
|
34
|
+
shadow: { ...DEFAULT_JUDGE_SHADOW_CONFIG },
|
|
35
|
+
};
|
|
36
|
+
const CLI_PROVIDER_IDS = ['codex', 'cursor', 'claude'];
|
|
37
|
+
function normalizeProviderAllowlist(value, fallback) {
|
|
38
|
+
if (!Array.isArray(value)) {
|
|
39
|
+
return [...fallback];
|
|
40
|
+
}
|
|
41
|
+
const allowed = value
|
|
42
|
+
.map((entry) => String(entry).trim())
|
|
43
|
+
.filter((entry) => CLI_PROVIDER_IDS.includes(entry));
|
|
44
|
+
return allowed.length > 0 ? [...new Set(allowed)] : [...fallback];
|
|
45
|
+
}
|
|
46
|
+
function normalizePositiveInt(value, fallback) {
|
|
47
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0
|
|
48
|
+
? Math.floor(value)
|
|
49
|
+
: fallback;
|
|
50
|
+
}
|
|
51
|
+
function normalizeRate(value, fallback, max) {
|
|
52
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
53
|
+
return fallback;
|
|
54
|
+
}
|
|
55
|
+
return Math.min(value, max);
|
|
56
|
+
}
|
|
57
|
+
export function normalizeJudgeRuntimeConfig(runtime) {
|
|
58
|
+
const session = runtime?.session ?? {};
|
|
59
|
+
const shadow = runtime?.shadow ?? {};
|
|
60
|
+
return {
|
|
61
|
+
session: normalizeJudgeSessionConfig(session),
|
|
62
|
+
shadow: {
|
|
63
|
+
enabled: shadow.enabled === true,
|
|
64
|
+
sampleRate: normalizeRate(shadow.sampleRate, DEFAULT_JUDGE_SHADOW_CONFIG.sampleRate, 1),
|
|
65
|
+
sampleRateMax: normalizeRate(shadow.sampleRateMax, DEFAULT_JUDGE_SHADOW_CONFIG.sampleRateMax, 1),
|
|
66
|
+
dailyRequestCap: normalizePositiveInt(shadow.dailyRequestCap, DEFAULT_JUDGE_SHADOW_CONFIG.dailyRequestCap),
|
|
67
|
+
mismatchRateThreshold: normalizeRate(shadow.mismatchRateThreshold, DEFAULT_JUDGE_SHADOW_CONFIG.mismatchRateThreshold, 1),
|
|
68
|
+
providerAllowlist: normalizeProviderAllowlist(shadow.providerAllowlist, DEFAULT_JUDGE_SHADOW_CONFIG.providerAllowlist),
|
|
69
|
+
windowSize: normalizePositiveInt(shadow.windowSize, DEFAULT_JUDGE_SHADOW_CONFIG.windowSize),
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
export function normalizeJudgeSessionConfig(session) {
|
|
74
|
+
const input = session ?? {};
|
|
75
|
+
return {
|
|
76
|
+
enabled: input.enabled === true,
|
|
77
|
+
maxTurns: normalizePositiveInt(input.maxTurns, DEFAULT_JUDGE_SESSION_CONFIG.maxTurns),
|
|
78
|
+
maxAgeMs: normalizePositiveInt(input.maxAgeMs, DEFAULT_JUDGE_SESSION_CONFIG.maxAgeMs),
|
|
79
|
+
maxIdleMs: normalizePositiveInt(input.maxIdleMs, DEFAULT_JUDGE_SESSION_CONFIG.maxIdleMs),
|
|
80
|
+
maxPromptBytes: normalizePositiveInt(input.maxPromptBytes, DEFAULT_JUDGE_SESSION_CONFIG.maxPromptBytes),
|
|
81
|
+
providerAllowlist: normalizeProviderAllowlist(input.providerAllowlist, DEFAULT_JUDGE_SESSION_CONFIG.providerAllowlist),
|
|
82
|
+
connectTimeoutMs: normalizePositiveInt(input.connectTimeoutMs, DEFAULT_JUDGE_SESSION_CONFIG.connectTimeoutMs),
|
|
83
|
+
evalTimeoutMs: typeof input.evalTimeoutMs === 'number' && input.evalTimeoutMs > 0
|
|
84
|
+
? Math.floor(input.evalTimeoutMs)
|
|
85
|
+
: null,
|
|
86
|
+
parseTimeoutMs: normalizePositiveInt(input.parseTimeoutMs, DEFAULT_JUDGE_SESSION_CONFIG.parseTimeoutMs),
|
|
87
|
+
maxSessionsPerProvider: normalizePositiveInt(input.maxSessionsPerProvider, DEFAULT_JUDGE_SESSION_CONFIG.maxSessionsPerProvider),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
export function resolveSessionEvalTimeoutMs(sessionConfig, judgeTimeoutMs) {
|
|
91
|
+
return sessionConfig.evalTimeoutMs ?? judgeTimeoutMs;
|
|
92
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { JudgeProviderId } from './judge-catalog.js';
|
|
2
|
+
import type { CliInvocation, CliJudgeRunCommand } from './judge-cli.js';
|
|
3
|
+
import type { BelayJudgeSessionConfig } from './judge-runtime-config.js';
|
|
4
|
+
import { type JudgeSessionBudget, type JudgeSessionKeyParts, type JudgeSessionResetReason } from './judge-session-guard.js';
|
|
5
|
+
export type BrokerSessionState = 'idle' | 'busy';
|
|
6
|
+
export interface BrokerSessionRecord {
|
|
7
|
+
keyParts: JudgeSessionKeyParts;
|
|
8
|
+
budget: JudgeSessionBudget;
|
|
9
|
+
state: BrokerSessionState;
|
|
10
|
+
providerResumeId?: string;
|
|
11
|
+
lastResetReason?: JudgeSessionResetReason;
|
|
12
|
+
}
|
|
13
|
+
export interface BrokerEvaluateRequest {
|
|
14
|
+
keyParts: JudgeSessionKeyParts;
|
|
15
|
+
invocation: CliInvocation;
|
|
16
|
+
promptBytes: number;
|
|
17
|
+
}
|
|
18
|
+
export interface BrokerEvaluateResult {
|
|
19
|
+
raw: string;
|
|
20
|
+
reused: boolean;
|
|
21
|
+
resetReason?: JudgeSessionResetReason;
|
|
22
|
+
providerResumeId?: string;
|
|
23
|
+
}
|
|
24
|
+
type RunCommand = CliJudgeRunCommand;
|
|
25
|
+
export interface JudgeSessionBrokerOptions {
|
|
26
|
+
config: BelayJudgeSessionConfig;
|
|
27
|
+
runCommand: RunCommand;
|
|
28
|
+
extractResumeId?: (providerId: Exclude<JudgeProviderId, 'ollama'>, raw: string) => string | null;
|
|
29
|
+
}
|
|
30
|
+
export declare class JudgeSessionBroker {
|
|
31
|
+
private readonly sessions;
|
|
32
|
+
private readonly mutexes;
|
|
33
|
+
private readonly options;
|
|
34
|
+
constructor(options: JudgeSessionBrokerOptions);
|
|
35
|
+
listSessions(): BrokerSessionRecord[];
|
|
36
|
+
invalidateSession(sessionKey: string, reason: JudgeSessionResetReason): void;
|
|
37
|
+
stopAll(reason?: JudgeSessionResetReason): number;
|
|
38
|
+
evaluate(request: BrokerEvaluateRequest, timeoutMs: number): Promise<BrokerEvaluateResult>;
|
|
39
|
+
private evaluateExclusive;
|
|
40
|
+
private enforceProviderSessionCap;
|
|
41
|
+
private buildResumeInvocation;
|
|
42
|
+
}
|
|
43
|
+
export declare function extractProviderResumeId(providerId: Exclude<JudgeProviderId, 'ollama'>, raw: string): string | null;
|
|
44
|
+
export declare function getRepoJudgeSessionBroker(repoRoot: string, options: JudgeSessionBrokerOptions): JudgeSessionBroker;
|
|
45
|
+
export declare function stopRepoJudgeSessionBroker(repoRoot: string, reason?: JudgeSessionResetReason): number;
|
|
46
|
+
export declare function resetJudgeSessionBrokersForTests(): void;
|
|
47
|
+
export declare function listRepoJudgeSessionBrokers(): string[];
|
|
48
|
+
export {};
|