@guilz-dev/belay 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/adapters/cursor/runtime-entry.js +1 -0
  2. package/dist/adapters/shared/gate-runtime.d.ts +11 -14
  3. package/dist/adapters/shared/gate-runtime.js +277 -19
  4. package/dist/bundle/claude-runtime.mjs +3138 -699
  5. package/dist/bundle/codex-runtime.mjs +3162 -711
  6. package/dist/bundle/cursor-runtime.mjs +3140 -701
  7. package/dist/cli.js +137 -5
  8. package/dist/commands/approve.d.ts +1 -0
  9. package/dist/commands/approve.js +45 -26
  10. package/dist/commands/config.js +135 -52
  11. package/dist/commands/doctor.js +11 -1
  12. package/dist/commands/explain.js +6 -1
  13. package/dist/commands/harvest.d.ts +29 -0
  14. package/dist/commands/harvest.js +72 -0
  15. package/dist/commands/judge.d.ts +11 -1
  16. package/dist/commands/judge.js +21 -2
  17. package/dist/commands/metrics.js +34 -2
  18. package/dist/commands/quality.d.ts +38 -0
  19. package/dist/commands/quality.js +105 -0
  20. package/dist/commands/simulate.d.ts +1 -0
  21. package/dist/commands/simulate.js +7 -2
  22. package/dist/commands/standing-allow.d.ts +10 -0
  23. package/dist/commands/standing-allow.js +26 -0
  24. package/dist/commands/tui.d.ts +48 -0
  25. package/dist/commands/tui.js +150 -0
  26. package/dist/config-io.js +10 -5
  27. package/dist/core/approval-replay-cli.d.ts +7 -0
  28. package/dist/core/approval-replay-cli.js +36 -0
  29. package/dist/core/approval-replay.d.ts +45 -0
  30. package/dist/core/approval-replay.js +141 -0
  31. package/dist/core/approval-service.d.ts +15 -0
  32. package/dist/core/approval-service.js +47 -2
  33. package/dist/core/approval.d.ts +24 -3
  34. package/dist/core/approval.js +47 -4
  35. package/dist/core/audit-analysis.d.ts +7 -1
  36. package/dist/core/audit-analysis.js +111 -0
  37. package/dist/core/audit-metrics.d.ts +7 -0
  38. package/dist/core/audit-metrics.js +24 -4
  39. package/dist/core/audit-query.d.ts +1 -0
  40. package/dist/core/audit-query.js +3 -0
  41. package/dist/core/audit-replay-context.d.ts +35 -0
  42. package/dist/core/audit-replay-context.js +88 -0
  43. package/dist/core/audit-types.d.ts +24 -1
  44. package/dist/core/audit-types.js +1 -1
  45. package/dist/core/capability/index.d.ts +2 -1
  46. package/dist/core/capability/index.js +1 -0
  47. package/dist/core/capability/paths.d.ts +2 -2
  48. package/dist/core/capability/paths.js +9 -9
  49. package/dist/core/capability/trusted-workspace-roots.d.ts +25 -0
  50. package/dist/core/capability/trusted-workspace-roots.js +149 -0
  51. package/dist/core/capability/types.d.ts +11 -1
  52. package/dist/core/capability-approval.d.ts +2 -1
  53. package/dist/core/capability-approval.js +100 -2
  54. package/dist/core/classify-subagent.js +2 -20
  55. package/dist/core/classify-tool.js +32 -3
  56. package/dist/core/config.d.ts +18 -0
  57. package/dist/core/config.js +49 -0
  58. package/dist/core/gate-engine.js +6 -6
  59. package/dist/core/harvest.d.ts +53 -0
  60. package/dist/core/harvest.js +276 -0
  61. package/dist/core/index.d.ts +9 -3
  62. package/dist/core/index.js +6 -2
  63. package/dist/core/judge-doctor.d.ts +3 -0
  64. package/dist/core/judge-doctor.js +54 -0
  65. package/dist/core/path-utils.d.ts +11 -0
  66. package/dist/core/path-utils.js +56 -6
  67. package/dist/core/reclassify.d.ts +3 -0
  68. package/dist/core/reclassify.js +44 -14
  69. package/dist/core/replay-scrub.d.ts +9 -0
  70. package/dist/core/replay-scrub.js +43 -0
  71. package/dist/core/shell-tokenizer.d.ts +2 -0
  72. package/dist/core/shell-tokenizer.js +82 -23
  73. package/dist/core/standing-allow.d.ts +50 -0
  74. package/dist/core/standing-allow.js +175 -0
  75. package/dist/core/types.d.ts +15 -0
  76. package/dist/core/verdict/adapter.js +10 -1
  77. package/dist/core/verdict/containment.d.ts +4 -3
  78. package/dist/core/verdict/containment.js +17 -12
  79. package/dist/core/verdict/judge-audit.d.ts +1 -0
  80. package/dist/core/verdict/judge-audit.js +32 -1
  81. package/dist/core/verdict/judge-baseline.d.ts +20 -0
  82. package/dist/core/verdict/judge-baseline.js +48 -0
  83. package/dist/core/verdict/judge-broker-service.d.ts +42 -0
  84. package/dist/core/verdict/judge-broker-service.js +279 -0
  85. package/dist/core/verdict/judge-cli-fingerprint.d.ts +5 -0
  86. package/dist/core/verdict/judge-cli-fingerprint.js +61 -0
  87. package/dist/core/verdict/judge-cli.d.ts +23 -0
  88. package/dist/core/verdict/judge-cli.js +137 -16
  89. package/dist/core/verdict/judge-factory.js +13 -4
  90. package/dist/core/verdict/judge-provider-matrix.d.ts +20 -0
  91. package/dist/core/verdict/judge-provider-matrix.js +66 -0
  92. package/dist/core/verdict/judge-runtime-config.d.ts +42 -0
  93. package/dist/core/verdict/judge-runtime-config.js +92 -0
  94. package/dist/core/verdict/judge-session-broker.d.ts +48 -0
  95. package/dist/core/verdict/judge-session-broker.js +195 -0
  96. package/dist/core/verdict/judge-session-guard.d.ts +27 -0
  97. package/dist/core/verdict/judge-session-guard.js +91 -0
  98. package/dist/core/verdict/judge-session-kill-switch.d.ts +11 -0
  99. package/dist/core/verdict/judge-session-kill-switch.js +43 -0
  100. package/dist/core/verdict/judge-session-mutex.d.ts +9 -0
  101. package/dist/core/verdict/judge-session-mutex.js +23 -0
  102. package/dist/core/verdict/judge-shadow.d.ts +24 -0
  103. package/dist/core/verdict/judge-shadow.js +116 -0
  104. package/dist/core/verdict/judge-transport.d.ts +40 -0
  105. package/dist/core/verdict/judge-transport.js +258 -0
  106. package/dist/core/verdict/judge.d.ts +14 -1
  107. package/dist/core/verdict/judge.js +6 -5
  108. package/dist/core/verdict/parser.js +6 -1
  109. package/dist/core/verdict/types.d.ts +13 -0
  110. package/dist/core/verdict/verdict.js +7 -4
  111. package/dist/corpus/adversarial-probe.d.ts +86 -0
  112. package/dist/corpus/adversarial-probe.js +220 -0
  113. package/dist/corpus/evaluate.d.ts +45 -12
  114. package/dist/corpus/evaluate.js +62 -5
  115. package/dist/corpus/gates.d.ts +35 -0
  116. package/dist/corpus/gates.js +81 -0
  117. package/dist/corpus/judge-accuracy.d.ts +8 -0
  118. package/dist/corpus/judge-accuracy.js +39 -0
  119. package/dist/corpus/must-allow-commands.d.ts +5 -0
  120. package/dist/corpus/must-allow-commands.js +13 -0
  121. package/dist/corpus/mutators.d.ts +28 -0
  122. package/dist/corpus/mutators.js +151 -0
  123. package/dist/corpus/ratchet.d.ts +42 -0
  124. package/dist/corpus/ratchet.js +116 -0
  125. package/dist/corpus/runtime-match.d.ts +19 -0
  126. package/dist/corpus/runtime-match.js +45 -0
  127. package/dist/corpus/standing-allow-catalog.generated.d.ts +13 -0
  128. package/dist/corpus/standing-allow-catalog.generated.js +79 -0
  129. package/dist/corpus/types.d.ts +47 -0
  130. package/dist/corpus/types.js +157 -0
  131. package/dist/installer.js +4 -1
  132. package/dist/judge-broker-daemon.d.ts +1 -0
  133. package/dist/judge-broker-daemon.js +123 -0
  134. package/dist/services/sandbox-service.d.ts +1 -0
  135. package/dist/services/sandbox-service.js +2 -0
  136. package/dist/version.d.ts +1 -1
  137. package/dist/version.js +1 -1
  138. package/package.json +7 -3
  139. package/skills/belay/SKILL.md +7 -1
  140. package/skills/belay/belay-approve.md +17 -0
@@ -11,11 +11,16 @@ export async function explainCommand(options) {
11
11
  const latest = [...pending.approvals].sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt))[0];
12
12
  const inputKind = latest.inputKind ??
13
13
  (latest.kind === 'egress' || latest.kind === 'capability' ? 'shell' : latest.kind);
14
+ const payload = latest.payloadJson !== undefined
15
+ ? JSON.parse(latest.payloadJson)
16
+ : undefined;
14
17
  const classified = await classifyForReport({
15
18
  targetDir: repoRoot,
16
- cwd: options.cwd,
19
+ cwd: latest.cwd ?? options.cwd,
17
20
  kind: inputKind,
18
21
  command: latest.input ?? latest.summary,
22
+ toolName: latest.toolName,
23
+ payload,
19
24
  });
20
25
  return {
21
26
  repoRoot: classified.repoRoot,
@@ -0,0 +1,29 @@
1
+ import type { AuditRecord } from '../core/audit-types.js';
2
+ import { type HarvestReport, type HarvestReviewOutcome } from '../core/harvest.js';
3
+ export interface HarvestListOptions {
4
+ targetDir?: string;
5
+ since?: string;
6
+ until?: string;
7
+ json?: boolean;
8
+ }
9
+ export interface HarvestApplyOptions {
10
+ targetDir?: string;
11
+ command: string;
12
+ outcome: HarvestReviewOutcome;
13
+ reason?: string;
14
+ corpusPath?: string;
15
+ }
16
+ export declare function harvestListProject(options?: HarvestListOptions): Promise<HarvestReport>;
17
+ export declare function harvestReportFromRecords(records: AuditRecord[], options?: {
18
+ since?: string;
19
+ until?: string;
20
+ allowPatterns?: string[];
21
+ }): HarvestReport;
22
+ export declare function formatHarvestReport(report: HarvestReport): string;
23
+ export declare function harvestApplyProject(options: HarvestApplyOptions): Promise<{
24
+ ok: boolean;
25
+ message: string;
26
+ corpusPath: string;
27
+ }>;
28
+ /** Parse audit ndjson for tests without full project layout. */
29
+ export declare function harvestReportFromNdjson(raw: string): HarvestReport;
@@ -0,0 +1,72 @@
1
+ import { readFile, writeFile } from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { loadConfigFile } from '../config-io.js';
4
+ import { parseAuditNdjson } from '../core/audit-metrics.js';
5
+ import { toAuditRecord } from '../core/audit-query.js';
6
+ import { applyHarvestReview, buildHarvestReport, filterRecordsForHarvest, } from '../core/harvest.js';
7
+ import { parseCorpusCases } from '../corpus/types.js';
8
+ import { loadAuditRecords } from './audit.js';
9
+ export async function harvestListProject(options = {}) {
10
+ const repoRoot = path.resolve(options.targetDir ?? process.cwd());
11
+ const config = await loadConfigFile(repoRoot);
12
+ const records = await loadAuditRecords(repoRoot);
13
+ return harvestReportFromRecords(records, {
14
+ since: options.since,
15
+ until: options.until,
16
+ allowPatterns: config.overrides.allow,
17
+ });
18
+ }
19
+ export function harvestReportFromRecords(records, options = {}) {
20
+ return buildHarvestReport(filterRecordsForHarvest(records, { since: options.since, until: options.until }), { allowPatterns: options.allowPatterns });
21
+ }
22
+ export function formatHarvestReport(report) {
23
+ const lines = [
24
+ `belay harvest (scope: ${report.scope} audit traces only)`,
25
+ `Schema: v${report.schemaVersion}`,
26
+ '',
27
+ `Benign candidates (${report.candidates.length}):`,
28
+ ];
29
+ if (report.candidates.length === 0) {
30
+ lines.push('- (none)');
31
+ }
32
+ else {
33
+ for (const candidate of report.candidates) {
34
+ lines.push(`- ${JSON.stringify(candidate.command)} [${candidate.sources.join(', ')}] asks=${candidate.askCount} approved=${candidate.approvedAfterDeny ? 'yes' : 'no'} fp=${candidate.fingerprint.slice(0, 12)}…`);
35
+ }
36
+ }
37
+ lines.push('', `Availability queue (${report.availabilityQueue.length}):`);
38
+ if (report.availabilityQueue.length === 0) {
39
+ lines.push('- (none)');
40
+ }
41
+ else {
42
+ for (const entry of report.availabilityQueue) {
43
+ lines.push(`- ${JSON.stringify(entry.command)} signal=${entry.availabilitySignal} asks=${entry.askCount} (${entry.reason})`);
44
+ }
45
+ }
46
+ lines.push('', 'Candidates are review-only signals — approve in audit does not auto-promote to corpus.', 'Time filters (--since/--until) keep paired deny/approval rows for round-trip detection.', 'Use: belay harvest apply --command "<text>" --outcome provably-benign|accepted-benign|reject');
47
+ return lines.join('\n');
48
+ }
49
+ export async function harvestApplyProject(options) {
50
+ const repoRoot = path.resolve(options.targetDir ?? process.cwd());
51
+ const corpusPath = path.resolve(repoRoot, options.corpusPath ?? path.join('corpus', 'shell-commands.json'));
52
+ const raw = await readFile(corpusPath, 'utf8');
53
+ const cases = parseCorpusCases(JSON.parse(raw));
54
+ const result = applyHarvestReview(cases, {
55
+ command: options.command,
56
+ outcome: options.outcome,
57
+ reason: options.reason,
58
+ });
59
+ if (result.applied) {
60
+ await writeFile(corpusPath, `${JSON.stringify(result.cases, null, 2)}\n`, 'utf8');
61
+ }
62
+ return {
63
+ ok: result.ok,
64
+ message: result.message,
65
+ corpusPath: path.relative(repoRoot, corpusPath) || corpusPath,
66
+ };
67
+ }
68
+ /** Parse audit ndjson for tests without full project layout. */
69
+ export function harvestReportFromNdjson(raw) {
70
+ const records = parseAuditNdjson(raw).map((entry) => toAuditRecord(entry));
71
+ return harvestReportFromRecords(records);
72
+ }
@@ -2,7 +2,8 @@ import type { BelayJudgeConfig } from '../core/config.js';
2
2
  export interface JudgeCommandOptions {
3
3
  targetDir?: string;
4
4
  json?: boolean;
5
- subcommand?: 'status' | 'list' | 'use' | 'test' | 'consent';
5
+ subcommand?: 'status' | 'list' | 'use' | 'test' | 'bench' | 'consent';
6
+ liveProbe?: boolean;
6
7
  providerId?: string;
7
8
  model?: string;
8
9
  endpoint?: string;
@@ -53,7 +54,16 @@ export declare function judgeTest(options?: JudgeCommandOptions): Promise<string
53
54
  warnings: string[];
54
55
  notes: string[];
55
56
  }>;
57
+ export declare function judgeBench(options?: JudgeCommandOptions): Promise<string | {
58
+ slo: typeof import("../core/verdict/judge-runtime-config.js").JUDGE_LATENCY_SLO;
59
+ measured: Record<import("../core/verdict/judge-baseline.js").JudgeLatencyPhase, import("../core/verdict/judge-baseline.js").JudgeLatencyPercentiles>;
60
+ sessionMeetsTarget: boolean;
61
+ }>;
56
62
  export declare function runJudgeCommand(options: JudgeCommandOptions): Promise<string | {
63
+ slo: typeof import("../core/verdict/judge-runtime-config.js").JUDGE_LATENCY_SLO;
64
+ measured: Record<import("../core/verdict/judge-baseline.js").JudgeLatencyPhase, import("../core/verdict/judge-baseline.js").JudgeLatencyPercentiles>;
65
+ sessionMeetsTarget: boolean;
66
+ } | {
57
67
  providerId: import("../core/config.js").JudgeProviderId | import("../core/config.js").DeprecatedJudgeProviderId | undefined;
58
68
  provider: import("../core/config.js").JudgeProvider;
59
69
  endpoint: string | null;
@@ -254,7 +254,9 @@ export async function judgeRequestCloudConsent(options = {}) {
254
254
  export async function judgeTest(options = {}) {
255
255
  const repoRoot = path.resolve(options.targetDir ?? process.cwd());
256
256
  const config = await loadConfigFile(repoRoot);
257
- const diagnosis = await diagnoseJudge(config, repoRoot);
257
+ const diagnosis = await diagnoseJudge(config, repoRoot, {
258
+ liveProbe: options.liveProbe === true,
259
+ });
258
260
  const modelCheck = diagnosis.modelCheck;
259
261
  if (options.json) {
260
262
  return { ...diagnosis, modelCheck };
@@ -267,6 +269,21 @@ export async function judgeTest(options = {}) {
267
269
  ...diagnosis.issues.map((i) => `Issue: ${i}`),
268
270
  ].join('\n');
269
271
  }
272
+ export async function judgeBench(options = {}) {
273
+ const { judgeLatencySloReport } = await import('../core/verdict/judge-baseline.js');
274
+ const report = judgeLatencySloReport();
275
+ if (options.json) {
276
+ return report;
277
+ }
278
+ const lines = [
279
+ `SLO target: Tier1 session p95 <= ${Math.round(report.slo.spawnBaselineP95Ms * (1 - report.slo.tier1P95ReductionTarget))}ms`,
280
+ `Tier0 p50/p95: ${report.measured.tier0.p50}/${report.measured.tier0.p95}ms (${report.measured.tier0.count} samples)`,
281
+ `Tier1 spawn p50/p95: ${report.measured.tier1_spawn.p50}/${report.measured.tier1_spawn.p95}ms (${report.measured.tier1_spawn.count} samples)`,
282
+ `Tier1 session p50/p95: ${report.measured.tier1_session.p50}/${report.measured.tier1_session.p95}ms (${report.measured.tier1_session.count} samples)`,
283
+ `Session meets target: ${report.sessionMeetsTarget ? 'yes' : 'no'}`,
284
+ ];
285
+ return lines.join('\n');
286
+ }
270
287
  export async function runJudgeCommand(options) {
271
288
  switch (options.subcommand) {
272
289
  case 'status':
@@ -277,9 +294,11 @@ export async function runJudgeCommand(options) {
277
294
  return judgeUse(options);
278
295
  case 'test':
279
296
  return judgeTest(options);
297
+ case 'bench':
298
+ return judgeBench(options);
280
299
  case 'consent':
281
300
  return judgeRequestCloudConsent(options);
282
301
  default:
283
- throw new Error('judge requires subcommand: status, list, use, test, or consent');
302
+ throw new Error('judge requires subcommand: status, list, use, test, bench, or consent');
284
303
  }
285
304
  }
@@ -2,6 +2,12 @@ import { readFile } from 'node:fs/promises';
2
2
  import path from 'node:path';
3
3
  import { loadConfigFile } from '../config-io.js';
4
4
  import { computeAuditMetrics, parseAuditNdjson } from '../core/audit-metrics.js';
5
+ function formatFingerprintPreview(fingerprint) {
6
+ if (fingerprint.length <= 12) {
7
+ return fingerprint;
8
+ }
9
+ return `${fingerprint.slice(0, 12)}…`;
10
+ }
5
11
  export async function metricsProject(options = {}) {
6
12
  const repoRoot = path.resolve(options.targetDir ?? process.cwd());
7
13
  const config = await loadConfigFile(repoRoot);
@@ -26,6 +32,7 @@ export function formatMetricsReport(report) {
26
32
  `Schema: v${report.schemaVersion}`,
27
33
  `Gate events: ${report.gateEvents}`,
28
34
  `Would-block: ${report.wouldBlockCount} (${(report.wouldBlockRate * 100).toFixed(1)}%)`,
35
+ `Classifier-quality would-block: ${report.classifierWouldBlockCount} (${(report.classifierWouldBlockRate * 100).toFixed(1)}%)`,
29
36
  `Approvals recorded during audit: ${report.approvalRecordedCount}`,
30
37
  ];
31
38
  if (report.approvalLatency.count > 0) {
@@ -34,6 +41,31 @@ export function formatMetricsReport(report) {
34
41
  if (report.bypassAttemptCount > 0) {
35
42
  lines.push(`Bypass attempts detected: ${report.bypassAttemptCount}`);
36
43
  }
44
+ if (report.availabilityAsks.total > 0) {
45
+ lines.push('', 'Availability-caused asks (infrastructure, not classifier ground truth):');
46
+ lines.push(`- missing trusted cwd: ${report.availabilityAsks.missingTrustedCwd}`);
47
+ lines.push(`- judge timeout: ${report.availabilityAsks.judgeTimeout}`);
48
+ lines.push(`- other judge fallback: ${report.availabilityAsks.judgeFallback}`);
49
+ lines.push(`- total: ${report.availabilityAsks.total}`);
50
+ }
51
+ if (report.repeatedFingerprintAsks.length > 0) {
52
+ lines.push('', 'Repeated fingerprint asks (repeat friction):');
53
+ for (const entry of report.repeatedFingerprintAsks) {
54
+ lines.push(`- [${entry.reason}] x${entry.askCount} ${formatFingerprintPreview(entry.fingerprint)}: ${entry.summary}`);
55
+ }
56
+ }
57
+ if (Object.keys(report.wouldBlockByReason).length > 0) {
58
+ lines.push('', 'Would-block by reason:');
59
+ for (const [reason, count] of Object.entries(report.wouldBlockByReason).sort((a, b) => b[1] - a[1])) {
60
+ lines.push(`- ${reason}: ${count}`);
61
+ }
62
+ }
63
+ if (report.approvalRatioByReason.length > 0) {
64
+ lines.push('', 'Approval ratio by reason (candidate signal, not ground truth):');
65
+ for (const entry of report.approvalRatioByReason.slice(0, 10)) {
66
+ lines.push(`- ${entry.reason}: ${(entry.approvalRate * 100).toFixed(0)}% approved after deny (${entry.approvedAfterDenyCount}/${entry.wouldBlockCount})`);
67
+ }
68
+ }
37
69
  if (Object.keys(report.byVerdict).length > 0) {
38
70
  lines.push('', 'By verdict:');
39
71
  for (const [verdict, count] of Object.entries(report.byVerdict).sort((a, b) => b[1] - a[1])) {
@@ -61,13 +93,13 @@ export function formatMetricsReport(report) {
61
93
  }
62
94
  }
63
95
  if (report.noisyRuleCandidates.length > 0) {
64
- lines.push('', 'Noisy rule candidates:');
96
+ lines.push('', 'Noisy rule candidates (high-signal subset of approval ratios; ≥50% approved after deny):');
65
97
  for (const rule of report.noisyRuleCandidates) {
66
98
  lines.push(`- ${rule.reason}: ${(rule.approvalRate * 100).toFixed(0)}% approved after deny (${rule.approvedCount}/${rule.denyCount})`);
67
99
  }
68
100
  }
69
101
  if (Object.keys(report.byReason).length > 0) {
70
- lines.push('', 'By reason:');
102
+ lines.push('', 'All gate events by reason:');
71
103
  for (const [reason, count] of Object.entries(report.byReason).sort((a, b) => b[1] - a[1])) {
72
104
  lines.push(`- ${reason}: ${count}`);
73
105
  }
@@ -0,0 +1,38 @@
1
+ import type { CorpusCategory, CorpusProvenanceCounts } from '../corpus/types.js';
2
+ export declare const QUALITY_REPORT_SCHEMA_VERSION = 1;
3
+ export interface QualityReport {
4
+ schemaVersion: typeof QUALITY_REPORT_SCHEMA_VERSION;
5
+ ok: boolean;
6
+ corpus: {
7
+ path: string;
8
+ passesHardGates: boolean;
9
+ totalCases: number;
10
+ categoryCounts: Record<CorpusCategory, number>;
11
+ provenanceCounts: CorpusProvenanceCounts;
12
+ mustAskMisses: number;
13
+ provablyBenignBlocks: number;
14
+ acceptedBenignMismatches: number;
15
+ accuracy: number;
16
+ };
17
+ audit: {
18
+ logPath: string;
19
+ gateEvents: number;
20
+ classifierWouldBlockRate: number;
21
+ availabilityAsks: number;
22
+ readyForEnforce: boolean;
23
+ repeatedFingerprintPatterns: number;
24
+ };
25
+ harvest: {
26
+ scope: 'shell';
27
+ benignCandidates: number;
28
+ availabilityQueue: number;
29
+ };
30
+ notes: string[];
31
+ }
32
+ export interface QualityOptions {
33
+ targetDir?: string;
34
+ corpusDir?: string;
35
+ json?: boolean;
36
+ }
37
+ export declare function qualityCheck(options?: QualityOptions): Promise<QualityReport>;
38
+ export declare function formatQualityReport(report: QualityReport): string;
@@ -0,0 +1,105 @@
1
+ import path from 'node:path';
2
+ import { loadConfigFile } from '../config-io.js';
3
+ import { computeAuditMetrics } from '../core/audit-metrics.js';
4
+ import { runCorpusEvaluation } from '../corpus/evaluate.js';
5
+ import { passesHardGates } from '../corpus/gates.js';
6
+ import { loadAuditRecords } from './audit.js';
7
+ import { harvestReportFromRecords } from './harvest.js';
8
+ export const QUALITY_REPORT_SCHEMA_VERSION = 1;
9
+ export async function qualityCheck(options = {}) {
10
+ const repoRoot = path.resolve(options.targetDir ?? process.cwd());
11
+ const config = await loadConfigFile(repoRoot);
12
+ const corpusDir = path.resolve(repoRoot, options.corpusDir ?? 'corpus');
13
+ const corpusMetrics = await runCorpusEvaluation(corpusDir);
14
+ const hardGatesOk = passesHardGates(corpusMetrics.gates);
15
+ const auditRecords = await loadAuditRecords(repoRoot);
16
+ const metrics = computeAuditMetrics(auditRecords, {
17
+ auditLogPath: config.audit.logPath,
18
+ mode: config.mode,
19
+ unknownLocalEffect: config.policy.unknownLocalEffect,
20
+ });
21
+ const harvest = harvestReportFromRecords(auditRecords, {
22
+ allowPatterns: config.overrides.allow,
23
+ });
24
+ const notes = [
25
+ 'Overall ok reflects corpus hard gates only; audit and harvest signals are advisory.',
26
+ 'Recursive quality loop: corpus hard gates are the FN/FP safety boundary.',
27
+ 'Harvest candidates and audit metrics inform review — approvals are not ground truth.',
28
+ 'Simulate (`belay simulate`) is triage only; it does not replace `pnpm corpus`.',
29
+ ];
30
+ if (!hardGatesOk) {
31
+ notes.push('Corpus hard gates failed — fix must-ask misses and provably-benign blocks before tuning friction.');
32
+ }
33
+ if (metrics.availabilityAsks.total > 0) {
34
+ notes.push(`${metrics.availabilityAsks.total} availability-caused ask(s) — tune judge/cwd infrastructure before corpus promotion.`);
35
+ }
36
+ if (harvest.availabilityQueue.length > 0) {
37
+ notes.push(`${harvest.availabilityQueue.length} shell pattern(s) in the availability queue — do not harvest into corpus.`);
38
+ }
39
+ const ok = hardGatesOk;
40
+ return {
41
+ schemaVersion: QUALITY_REPORT_SCHEMA_VERSION,
42
+ ok,
43
+ corpus: {
44
+ path: path.relative(repoRoot, corpusDir) || corpusDir,
45
+ passesHardGates: hardGatesOk,
46
+ totalCases: corpusMetrics.total,
47
+ categoryCounts: corpusMetrics.categoryCounts,
48
+ provenanceCounts: corpusMetrics.provenanceCounts,
49
+ mustAskMisses: corpusMetrics.gates.mustAsk.mismatches,
50
+ provablyBenignBlocks: corpusMetrics.gates.provablyBenign.mismatches,
51
+ acceptedBenignMismatches: corpusMetrics.gates.acceptedBenign.mismatches,
52
+ accuracy: corpusMetrics.accuracy,
53
+ },
54
+ audit: {
55
+ logPath: config.audit.logPath,
56
+ gateEvents: metrics.gateEvents,
57
+ classifierWouldBlockRate: metrics.classifierWouldBlockRate,
58
+ availabilityAsks: metrics.availabilityAsks.total,
59
+ readyForEnforce: metrics.dogfood.readyForEnforce,
60
+ repeatedFingerprintPatterns: metrics.repeatedFingerprintAsks.length,
61
+ },
62
+ harvest: {
63
+ scope: 'shell',
64
+ benignCandidates: harvest.candidates.length,
65
+ availabilityQueue: harvest.availabilityQueue.length,
66
+ },
67
+ notes,
68
+ };
69
+ }
70
+ export function formatQualityReport(report) {
71
+ const lines = [
72
+ 'belay quality — recursive quality loop status',
73
+ `Schema: v${report.schemaVersion}`,
74
+ `Overall: ${report.ok ? 'OK' : 'ATTENTION NEEDED'}`,
75
+ '',
76
+ 'Corpus hard gates:',
77
+ ` path: ${report.corpus.path}`,
78
+ ` passes: ${report.corpus.passesHardGates ? 'yes' : 'no'}`,
79
+ ` total cases: ${report.corpus.totalCases}`,
80
+ ` categories: must-ask=${report.corpus.categoryCounts['must-ask']} provably-benign=${report.corpus.categoryCounts['provably-benign']} accepted-benign=${report.corpus.categoryCounts['accepted-benign']}`,
81
+ ` provenance: manual=${report.corpus.provenanceCounts.manual} mutation=${report.corpus.provenanceCounts.mutation} harvest=${report.corpus.provenanceCounts.harvest} redteam=${report.corpus.provenanceCounts.redteam} unspecified=${report.corpus.provenanceCounts.unspecified}`,
82
+ ` must-ask misses: ${report.corpus.mustAskMisses}`,
83
+ ` provably-benign blocks: ${report.corpus.provablyBenignBlocks}`,
84
+ ` accepted-benign mismatches (soft): ${report.corpus.acceptedBenignMismatches}`,
85
+ ` accuracy: ${(report.corpus.accuracy * 100).toFixed(1)}%`,
86
+ '',
87
+ 'Audit metrics:',
88
+ ` log: ${report.audit.logPath}`,
89
+ ` gate events: ${report.audit.gateEvents}`,
90
+ ` classifier would-block rate: ${(report.audit.classifierWouldBlockRate * 100).toFixed(1)}%`,
91
+ ` availability asks: ${report.audit.availabilityAsks}`,
92
+ ` ready for enforce: ${report.audit.readyForEnforce ? 'yes' : 'no'}`,
93
+ ` repeated fingerprint patterns: ${report.audit.repeatedFingerprintPatterns}`,
94
+ '',
95
+ 'Harvest (shell only):',
96
+ ` benign candidates: ${report.harvest.benignCandidates}`,
97
+ ` availability queue: ${report.harvest.availabilityQueue}`,
98
+ '',
99
+ 'Notes:',
100
+ ];
101
+ for (const note of report.notes) {
102
+ lines.push(`- ${note}`);
103
+ }
104
+ return lines.join('\n');
105
+ }
@@ -9,6 +9,7 @@ export declare function simulateProject(options: SimulateOptions): Promise<{
9
9
  changedCount: number;
10
10
  allowToDenyCount: number;
11
11
  denyToAllowCount: number;
12
+ missingSnapshotCount: number;
12
13
  diffs: import("../core/reclassify.js").ReclassifyDiff[];
13
14
  }>;
14
15
  export declare function formatSimulateReport(report: Awaited<ReturnType<typeof simulateProject>>): string;
@@ -4,7 +4,7 @@ import path from 'node:path';
4
4
  import { loadConfigFile } from '../config-io.js';
5
5
  import { parseAuditNdjson, toAuditRecord } from '../core/audit-metrics.js';
6
6
  import { mergeConfig } from '../core/config.js';
7
- import { diffReclassification } from '../core/reclassify.js';
7
+ import { countMissingActionSnapshots, diffReclassification } from '../core/reclassify.js';
8
8
  export async function simulateProject(options) {
9
9
  const repoRoot = path.resolve(options.targetDir ?? process.cwd());
10
10
  const currentConfig = await loadConfigFile(repoRoot);
@@ -22,6 +22,7 @@ export async function simulateProject(options) {
22
22
  raw = '';
23
23
  }
24
24
  const records = parseAuditNdjson(raw).map(toAuditRecord);
25
+ const missingSnapshotCount = countMissingActionSnapshots(records);
25
26
  const diffs = (await Promise.all(records.map((record) => diffReclassification(record, candidateConfig, repoRoot)))).filter((diff) => diff !== null);
26
27
  const allowToDeny = diffs.filter((diff) => (diff.previousVerdict === 'allow' || diff.previousVerdict === 'allow_flagged') &&
27
28
  diff.nextVerdict === 'deny_pending_approval');
@@ -33,20 +34,24 @@ export async function simulateProject(options) {
33
34
  changedCount: diffs.length,
34
35
  allowToDenyCount: allowToDeny.length,
35
36
  denyToAllowCount: denyToAllow.length,
37
+ missingSnapshotCount,
36
38
  diffs,
37
39
  };
38
40
  }
39
41
  export function formatSimulateReport(report) {
40
42
  const lines = [
41
43
  `simulate ${report.candidateConfigPath}`,
44
+ 'Triage only — corpus hard gates (`pnpm corpus`), not simulate counts, are the safety boundary.',
42
45
  `Records scanned: ${report.totalRecords}`,
46
+ `Missing action snapshots: ${report.missingSnapshotCount}`,
43
47
  `Verdict changes: ${report.changedCount}`,
44
48
  `allow/flagged → deny: ${report.allowToDenyCount}`,
45
49
  `deny → allow/flagged: ${report.denyToAllowCount}`,
46
50
  '',
47
51
  ];
48
52
  for (const diff of report.diffs.slice(0, 30)) {
49
- lines.push(`- ${diff.summary ?? diff.fingerprint}: ${diff.previousVerdict}/${diff.previousReason} ${diff.nextVerdict}/${diff.nextReason}`);
53
+ const context = diff.replayCwd && diff.replayKind ? ` [${diff.replayKind} cwd=${diff.replayCwd}]` : '';
54
+ lines.push(`- ${diff.summary ?? diff.fingerprint}${context}: ${diff.previousVerdict}/${diff.previousReason} → ${diff.nextVerdict}/${diff.nextReason}`);
50
55
  }
51
56
  if (report.diffs.length > 30) {
52
57
  lines.push(`... ${report.diffs.length - 30} more`);
@@ -0,0 +1,10 @@
1
+ import type { GatedActionKind } from '../core/gate-contract.js';
2
+ export interface RevokeStandingAllowOptions {
3
+ targetDir?: string;
4
+ fingerprint: string;
5
+ kind?: GatedActionKind;
6
+ }
7
+ export declare function revokeStandingAllow(options: RevokeStandingAllowOptions): Promise<{
8
+ ok: boolean;
9
+ message: string;
10
+ }>;
@@ -0,0 +1,26 @@
1
+ import path from 'node:path';
2
+ import { loadConfigFile, repoLocalStateDirFor } from '../config-io.js';
3
+ import { loadStandingAllow, revokeStandingAllowEntry, saveStandingAllow, standingAllowFile, } from '../core/standing-allow.js';
4
+ export async function revokeStandingAllow(options) {
5
+ const repoRoot = path.resolve(options.targetDir ?? process.cwd());
6
+ const config = await loadConfigFile(repoRoot);
7
+ const filePath = standingAllowFile(config, repoLocalStateDirFor(repoRoot, config));
8
+ const state = await loadStandingAllow(filePath);
9
+ const kind = options.kind ?? 'shell';
10
+ const { state: next, removed } = revokeStandingAllowEntry(state, {
11
+ kind,
12
+ fingerprint: options.fingerprint,
13
+ repoRoot,
14
+ });
15
+ if (!removed) {
16
+ return {
17
+ ok: false,
18
+ message: `Standing-allow entry not found for ${kind} fingerprint ${options.fingerprint}.`,
19
+ };
20
+ }
21
+ await saveStandingAllow(filePath, next);
22
+ return {
23
+ ok: true,
24
+ message: `Revoked standing-allow for ${kind} fingerprint ${options.fingerprint}.`,
25
+ };
26
+ }
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Zero-dependency terminal UI helpers for the interactive `belay config` wizard.
3
+ *
4
+ * Everything here is built on Node's built-in `readline`/`tty` only — no runtime
5
+ * dependencies — so it stays aligned with belay's zero-dependency posture.
6
+ *
7
+ * This module is intentionally TTY-only. Callers are responsible for non-TTY
8
+ * fallback behavior.
9
+ */
10
+ export declare const c: {
11
+ bold: (s: string) => string;
12
+ dim: (s: string) => string;
13
+ italic: (s: string) => string;
14
+ underline: (s: string) => string;
15
+ inverse: (s: string) => string;
16
+ cyan: (s: string) => string;
17
+ green: (s: string) => string;
18
+ yellow: (s: string) => string;
19
+ red: (s: string) => string;
20
+ gray: (s: string) => string;
21
+ };
22
+ /** A select wizard only makes sense when both stdin and stdout are real TTYs. */
23
+ export declare function isInteractiveTTY(): boolean;
24
+ /** Top banner shown once at the start of the wizard. */
25
+ export declare function intro(title: string, subtitle?: string): void;
26
+ /** Closing line shown once at the end of the wizard. */
27
+ export declare function outro(message: string): void;
28
+ /** Collapsed one-line summary of an answered question. */
29
+ export declare function summary(label: string, value: string): void;
30
+ export interface SelectChoice<T extends string> {
31
+ value: T;
32
+ /** Visible label; defaults to `value`. */
33
+ label?: string;
34
+ /** Dim hint shown to the right of the focused row. */
35
+ hint?: string;
36
+ }
37
+ export interface SelectOptions<T extends string> {
38
+ message: string;
39
+ choices: SelectChoice<T>[];
40
+ defaultValue: T;
41
+ }
42
+ /**
43
+ * Interactive single-choice prompt driven by arrow keys (or j/k). Resolves with
44
+ * the selected choice value. Must only be called in an interactive TTY.
45
+ */
46
+ export declare function selectPrompt<T extends string>(opts: SelectOptions<T>): Promise<T>;
47
+ /** Interactive yes/no prompt rendered as a two-row arrow-key selection. */
48
+ export declare function confirmPrompt(message: string, defaultValue: boolean): Promise<boolean>;