@guilz-dev/belay 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle/claude-runtime.mjs +1 -1
- package/dist/bundle/codex-runtime.mjs +1 -1
- package/dist/bundle/cursor-runtime.mjs +1 -1
- package/dist/commands/doctor.js +17 -21
- package/dist/commands/metrics.js +33 -4
- package/dist/commands/quality.js +2 -6
- package/dist/commands/status.js +1 -1
- package/dist/core/audit-metrics.d.ts +23 -0
- package/dist/core/audit-metrics.js +73 -15
- package/dist/operational-insights.d.ts +3 -0
- package/dist/operational-insights.js +7 -3
- package/dist/runtime-provenance.d.ts +8 -0
- package/dist/runtime-provenance.js +34 -0
- package/dist/types.d.ts +3 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -16408,7 +16408,7 @@ function resolveStandingAllowMatch(params) {
|
|
|
16408
16408
|
}
|
|
16409
16409
|
|
|
16410
16410
|
// src/version.ts
|
|
16411
|
-
var PACKAGE_VERSION = "0.8.
|
|
16411
|
+
var PACKAGE_VERSION = "0.8.1";
|
|
16412
16412
|
|
|
16413
16413
|
// src/adapters/layouts/protected-paths.ts
|
|
16414
16414
|
import path47 from "node:path";
|
|
@@ -16403,7 +16403,7 @@ function resolveStandingAllowMatch(params) {
|
|
|
16403
16403
|
}
|
|
16404
16404
|
|
|
16405
16405
|
// src/version.ts
|
|
16406
|
-
var PACKAGE_VERSION = "0.8.
|
|
16406
|
+
var PACKAGE_VERSION = "0.8.1";
|
|
16407
16407
|
|
|
16408
16408
|
// src/adapters/layouts/protected-paths.ts
|
|
16409
16409
|
import path47 from "node:path";
|
|
@@ -16403,7 +16403,7 @@ function resolveStandingAllowMatch(params) {
|
|
|
16403
16403
|
}
|
|
16404
16404
|
|
|
16405
16405
|
// src/version.ts
|
|
16406
|
-
var PACKAGE_VERSION = "0.8.
|
|
16406
|
+
var PACKAGE_VERSION = "0.8.1";
|
|
16407
16407
|
|
|
16408
16408
|
// src/adapters/layouts/protected-paths.ts
|
|
16409
16409
|
import path47 from "node:path";
|
package/dist/commands/doctor.js
CHANGED
|
@@ -20,26 +20,13 @@ import { recoveryApprovalSetupNotes, recoveryNotificationConfigured, recoveryNot
|
|
|
20
20
|
import { probeFileCloneStrategy } from '../core/transactional/file-clone.js';
|
|
21
21
|
import { getManagedHookEntries } from '../defaults.js';
|
|
22
22
|
import { resolveNodeBinary } from '../node-resolution.js';
|
|
23
|
+
import { readInstalledRuntimeProvenance } from '../runtime-provenance.js';
|
|
23
24
|
import { egressStatus } from '../services/egress-service.js';
|
|
24
25
|
import { sandboxStatus } from '../services/sandbox-service.js';
|
|
25
26
|
import { PACKAGE_VERSION } from '../version.js';
|
|
26
27
|
import { loadAuditRecords } from './audit.js';
|
|
27
28
|
import { collectHealthSnapshot } from './health-snapshot.js';
|
|
28
29
|
import { metricsProject } from './metrics.js';
|
|
29
|
-
async function readRuntimeVersion(corePath) {
|
|
30
|
-
try {
|
|
31
|
-
const content = await readFile(corePath, 'utf8');
|
|
32
|
-
const stampMatch = content.match(/RUNTIME_BUILD_STAMP\s*=\s*"([^"]+)"/);
|
|
33
|
-
const versionMatch = content.match(/RUNTIME_PACKAGE_VERSION\s*=\s*"([^"]+)"/);
|
|
34
|
-
return {
|
|
35
|
-
stamp: stampMatch?.[1],
|
|
36
|
-
version: versionMatch?.[1],
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
catch {
|
|
40
|
-
return {};
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
30
|
function resolveDoctorAdapter(options, configAdapter) {
|
|
44
31
|
if (options.adapter) {
|
|
45
32
|
return options.adapter;
|
|
@@ -234,7 +221,7 @@ export async function doctorProject(options = {}) {
|
|
|
234
221
|
notes.push(`Node resolved at ${nodeResolution.path}`);
|
|
235
222
|
}
|
|
236
223
|
if (existsSync(corePath)) {
|
|
237
|
-
const runtimeVersions = await
|
|
224
|
+
const runtimeVersions = await readInstalledRuntimeProvenance(corePath);
|
|
238
225
|
if (runtimeVersions.stamp && !runtimeVersions.stamp.startsWith(`${PACKAGE_VERSION}@`)) {
|
|
239
226
|
warnings.push(`Installed runtime stamp (${runtimeVersions.stamp}) differs from package (${PACKAGE_VERSION}). Run belay upgrade.`);
|
|
240
227
|
}
|
|
@@ -269,25 +256,34 @@ export async function doctorProject(options = {}) {
|
|
|
269
256
|
let dogfood = null;
|
|
270
257
|
if (loadedConfig) {
|
|
271
258
|
const auditRecords = await loadAuditRecords(repoRoot);
|
|
272
|
-
const
|
|
259
|
+
const metrics = await metricsProject({ targetDir: repoRoot });
|
|
260
|
+
const cohortIdentity = metrics.currentCohort.identity;
|
|
261
|
+
const cohortAuditRecords = cohortIdentity
|
|
262
|
+
? auditRecords.filter((record) => record.runtimeBuildStamp === cohortIdentity.runtimeBuildStamp &&
|
|
263
|
+
record.configFingerprint === cohortIdentity.configFingerprint)
|
|
264
|
+
: [];
|
|
265
|
+
const auditVisibility = summarizeAuditVisibility(cohortAuditRecords);
|
|
273
266
|
const drift = detectFenceDrift(auditVisibility, {
|
|
274
267
|
threshold: loadedConfig.policy.fenceWarnThreshold,
|
|
275
268
|
});
|
|
276
269
|
warnings.push(...drift.warnings);
|
|
277
270
|
notes.push(...drift.notes);
|
|
278
|
-
const
|
|
271
|
+
const cohort = metrics.currentCohort;
|
|
279
272
|
dogfood = {
|
|
280
273
|
active: loadedConfig.mode === 'audit' && loadedConfig.policy.unknownLocalEffect === 'deny',
|
|
281
274
|
mode: loadedConfig.mode,
|
|
282
275
|
unknownLocalEffect: loadedConfig.policy.unknownLocalEffect,
|
|
283
276
|
readyForEnforce: metrics.dogfood.readyForEnforce,
|
|
284
|
-
gateEvents:
|
|
285
|
-
wouldBlockCount:
|
|
286
|
-
wouldBlockRate:
|
|
277
|
+
gateEvents: cohort.gateEvents,
|
|
278
|
+
wouldBlockCount: cohort.wouldBlockCount,
|
|
279
|
+
wouldBlockRate: cohort.wouldBlockRate,
|
|
280
|
+
excludedGateEvents: cohort.excludedGateEvents,
|
|
281
|
+
runtimeBuildStamp: cohort.identity?.runtimeBuildStamp,
|
|
282
|
+
configFingerprint: cohort.identity?.configFingerprint,
|
|
287
283
|
notes: metrics.dogfood.notes,
|
|
288
284
|
};
|
|
289
285
|
if (dogfood.active) {
|
|
290
|
-
notes.push(`Dogfood active: ${dogfood.gateEvents} gate events, ${dogfood.wouldBlockCount} would-block (${(dogfood.wouldBlockRate * 100).toFixed(1)}%).`);
|
|
286
|
+
notes.push(`Dogfood active cohort: ${dogfood.gateEvents} gate events, ${dogfood.wouldBlockCount} would-block (${(dogfood.wouldBlockRate * 100).toFixed(1)}%); ${dogfood.excludedGateEvents} historical/mismatched event(s) excluded.`);
|
|
291
287
|
if (dogfood.readyForEnforce) {
|
|
292
288
|
notes.push('Dogfood metrics suggest enforce mode is ready (belay dogfood --enforce).');
|
|
293
289
|
}
|
package/dist/commands/metrics.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFile } from 'node:fs/promises';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { loadConfigFile } from '../config-io.js';
|
|
4
4
|
import { computeAuditMetrics, parseAuditNdjson } from '../core/audit-metrics.js';
|
|
5
|
+
import { resolveActiveAuditCohort } from '../runtime-provenance.js';
|
|
5
6
|
function formatFingerprintPreview(fingerprint) {
|
|
6
7
|
if (fingerprint.length <= 12) {
|
|
7
8
|
return fingerprint;
|
|
@@ -20,21 +21,49 @@ export async function metricsProject(options = {}) {
|
|
|
20
21
|
raw = '';
|
|
21
22
|
}
|
|
22
23
|
const records = parseAuditNdjson(raw);
|
|
24
|
+
const activeCohort = await resolveActiveAuditCohort(repoRoot, config);
|
|
23
25
|
return computeAuditMetrics(records, {
|
|
24
26
|
auditLogPath: config.audit.logPath,
|
|
25
27
|
mode: config.mode,
|
|
26
28
|
unknownLocalEffect: config.policy.unknownLocalEffect,
|
|
29
|
+
activeCohort,
|
|
27
30
|
});
|
|
28
31
|
}
|
|
29
32
|
export function formatMetricsReport(report) {
|
|
30
33
|
const lines = [
|
|
31
34
|
`belay metrics for ${report.auditLogPath}`,
|
|
32
35
|
`Schema: v${report.schemaVersion}`,
|
|
33
|
-
`
|
|
34
|
-
`
|
|
35
|
-
`
|
|
36
|
-
`
|
|
36
|
+
`All-time gate events: ${report.gateEvents}`,
|
|
37
|
+
`All-time would-block: ${report.wouldBlockCount} (${(report.wouldBlockRate * 100).toFixed(1)}%)`,
|
|
38
|
+
`All-time classifier-quality would-block: ${report.classifierWouldBlockCount} (${(report.classifierWouldBlockRate * 100).toFixed(1)}%)`,
|
|
39
|
+
`All-time approvals recorded during audit: ${report.approvalRecordedCount}`,
|
|
37
40
|
];
|
|
41
|
+
lines.push('', 'Current readiness cohort:');
|
|
42
|
+
if (report.currentCohort.identity) {
|
|
43
|
+
lines.push(`- runtime build: ${report.currentCohort.identity.runtimeBuildStamp}`);
|
|
44
|
+
lines.push(`- config fingerprint: ${formatFingerprintPreview(report.currentCohort.identity.configFingerprint)}`);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
lines.push('- identity: unavailable');
|
|
48
|
+
}
|
|
49
|
+
lines.push(`- matching gate events: ${report.currentCohort.gateEvents}`);
|
|
50
|
+
lines.push(`- excluded historical/mismatched gate events: ${report.currentCohort.excludedGateEvents}`);
|
|
51
|
+
lines.push(`- would-block: ${report.currentCohort.wouldBlockCount} (${(report.currentCohort.wouldBlockRate * 100).toFixed(1)}%)`);
|
|
52
|
+
lines.push(`- classifier-quality would-block: ${report.currentCohort.classifierWouldBlockCount} (${(report.currentCohort.classifierWouldBlockRate * 100).toFixed(1)}%)`);
|
|
53
|
+
lines.push(`- approvals recorded during audit: ${report.currentCohort.approvalRecordedCount}`);
|
|
54
|
+
lines.push(`- availability-caused asks: ${report.currentCohort.availabilityAsks.total}`);
|
|
55
|
+
if (Object.keys(report.currentCohort.wouldBlockByReason).length > 0) {
|
|
56
|
+
lines.push('', 'Current-cohort would-block by reason:');
|
|
57
|
+
for (const [reason, count] of Object.entries(report.currentCohort.wouldBlockByReason).sort((left, right) => right[1] - left[1])) {
|
|
58
|
+
lines.push(`- ${reason}: ${count}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (report.currentCohort.topWouldBlockSummaries.length > 0) {
|
|
62
|
+
lines.push('', 'Current-cohort top would-block summaries:');
|
|
63
|
+
for (const entry of report.currentCohort.topWouldBlockSummaries) {
|
|
64
|
+
lines.push(`- [${entry.reason}] x${entry.count}: ${entry.summary}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
38
67
|
if (report.approvalLatency.count > 0) {
|
|
39
68
|
lines.push(`Approval latency: median ${report.approvalLatency.medianMs ?? 0}ms, p95 ${report.approvalLatency.p95Ms ?? 0}ms (${report.approvalLatency.count} samples)`);
|
|
40
69
|
}
|
package/dist/commands/quality.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { loadConfigFile } from '../config-io.js';
|
|
3
|
-
import { computeAuditMetrics } from '../core/audit-metrics.js';
|
|
4
3
|
import { runCorpusEvaluation } from '../corpus/evaluate.js';
|
|
5
4
|
import { passesHardGates } from '../corpus/gates.js';
|
|
6
5
|
import { loadAuditRecords } from './audit.js';
|
|
7
6
|
import { harvestReportFromRecords } from './harvest.js';
|
|
7
|
+
import { metricsProject } from './metrics.js';
|
|
8
8
|
export const QUALITY_REPORT_SCHEMA_VERSION = 1;
|
|
9
9
|
export async function qualityCheck(options = {}) {
|
|
10
10
|
const repoRoot = path.resolve(options.targetDir ?? process.cwd());
|
|
@@ -13,11 +13,7 @@ export async function qualityCheck(options = {}) {
|
|
|
13
13
|
const corpusMetrics = await runCorpusEvaluation(corpusDir);
|
|
14
14
|
const hardGatesOk = passesHardGates(corpusMetrics.gates);
|
|
15
15
|
const auditRecords = await loadAuditRecords(repoRoot);
|
|
16
|
-
const metrics =
|
|
17
|
-
auditLogPath: config.audit.logPath,
|
|
18
|
-
mode: config.mode,
|
|
19
|
-
unknownLocalEffect: config.policy.unknownLocalEffect,
|
|
20
|
-
});
|
|
16
|
+
const metrics = await metricsProject({ targetDir: repoRoot });
|
|
21
17
|
const harvest = harvestReportFromRecords(auditRecords);
|
|
22
18
|
const notes = [
|
|
23
19
|
'Overall ok reflects corpus hard gates only; audit and harvest signals are advisory.',
|
package/dist/commands/status.js
CHANGED
|
@@ -54,7 +54,7 @@ export function formatStatusReport(report) {
|
|
|
54
54
|
`Approved (awaiting use): ${report.approved.length}`,
|
|
55
55
|
`Expired pending (not yet compacted): ${report.expiredPendingCount}`,
|
|
56
56
|
`Dogfood: ${report.dogfood.active ? 'active' : 'inactive'} (mode=${report.dogfood.mode}, unknownLocalEffect=${report.dogfood.unknownLocalEffect})`,
|
|
57
|
-
`
|
|
57
|
+
`Current cohort metrics: ${report.dogfood.gateEvents} gate events, ${report.dogfood.wouldBlockCount} would-block (${(report.dogfood.wouldBlockRate * 100).toFixed(1)}%), ${report.dogfood.excludedGateEvents} historical/mismatched excluded`,
|
|
58
58
|
`Ready for enforce: ${report.dogfood.readyForEnforce ? 'yes' : 'not yet'}`,
|
|
59
59
|
`File checkpoint: ${report.fileCheckpoint.enabled ? 'enabled' : 'disabled'} (transactional=${report.fileCheckpoint.transactionalEnabled}, durable=${report.fileCheckpoint.durableCheckpointEnabled}, nonGit=${report.fileCheckpoint.allowNonGit})`,
|
|
60
60
|
`File checkpoint limits: files=${report.fileCheckpoint.maxFiles}, sourceBytes=${report.fileCheckpoint.maxSourceBytes}, workspaceBytes=${report.fileCheckpoint.maxWorkspaceBytes}, prepareTimeoutMs=${report.fileCheckpoint.prepareTimeoutMs}, copyConcurrency=${report.fileCheckpoint.copyConcurrency}`,
|
|
@@ -2,6 +2,27 @@ import { buildApprovalRoundTrips, filterAuditRecords, toAuditRecord } from './au
|
|
|
2
2
|
import type { AvailabilityAskCounts, ReasonApprovalRatio, RepeatedFingerprintAsk } from './audit-types.js';
|
|
3
3
|
/** Minimum gate events before recommending enforce with zero would-block rate. */
|
|
4
4
|
export declare const MIN_GATE_EVENTS_FOR_ENFORCE = 20;
|
|
5
|
+
export interface AuditCohortIdentity {
|
|
6
|
+
runtimeBuildStamp: string;
|
|
7
|
+
configFingerprint: string;
|
|
8
|
+
}
|
|
9
|
+
export interface AuditMetricsCohort {
|
|
10
|
+
identity: AuditCohortIdentity | null;
|
|
11
|
+
gateEvents: number;
|
|
12
|
+
excludedGateEvents: number;
|
|
13
|
+
wouldBlockCount: number;
|
|
14
|
+
wouldBlockRate: number;
|
|
15
|
+
classifierWouldBlockCount: number;
|
|
16
|
+
classifierWouldBlockRate: number;
|
|
17
|
+
approvalRecordedCount: number;
|
|
18
|
+
availabilityAsks: AvailabilityAskCounts;
|
|
19
|
+
wouldBlockByReason: Record<string, number>;
|
|
20
|
+
topWouldBlockSummaries: Array<{
|
|
21
|
+
summary: string;
|
|
22
|
+
reason: string;
|
|
23
|
+
count: number;
|
|
24
|
+
}>;
|
|
25
|
+
}
|
|
5
26
|
export interface AuditMetricsReport {
|
|
6
27
|
schemaVersion: number;
|
|
7
28
|
auditLogPath: string;
|
|
@@ -24,6 +45,7 @@ export interface AuditMetricsReport {
|
|
|
24
45
|
byEffect: Record<string, number>;
|
|
25
46
|
byConfidence: Record<string, number>;
|
|
26
47
|
gateEventsByRuntime: Record<string, number>;
|
|
48
|
+
currentCohort: AuditMetricsCohort;
|
|
27
49
|
approvalRecordedCount: number;
|
|
28
50
|
topWouldBlockSummaries: Array<{
|
|
29
51
|
summary: string;
|
|
@@ -55,5 +77,6 @@ export declare function computeAuditMetrics(records: Record<string, unknown>[],
|
|
|
55
77
|
auditLogPath?: string;
|
|
56
78
|
mode?: string;
|
|
57
79
|
unknownLocalEffect?: string;
|
|
80
|
+
activeCohort?: AuditCohortIdentity | null;
|
|
58
81
|
}): AuditMetricsReport;
|
|
59
82
|
export { buildApprovalRoundTrips, filterAuditRecords, toAuditRecord };
|
|
@@ -95,33 +95,90 @@ export function computeAuditMetrics(records, options = {}) {
|
|
|
95
95
|
const topWouldBlockSummaries = [...summaryCounts.values()]
|
|
96
96
|
.sort((left, right) => right.count - left.count)
|
|
97
97
|
.slice(0, 10);
|
|
98
|
+
const activeCohort = options.activeCohort ?? null;
|
|
99
|
+
const cohortRecords = activeCohort
|
|
100
|
+
? auditRecords.filter((record) => record.runtimeBuildStamp === activeCohort.runtimeBuildStamp &&
|
|
101
|
+
record.configFingerprint === activeCohort.configFingerprint)
|
|
102
|
+
: [];
|
|
103
|
+
const cohortGateRecords = cohortRecords.filter((record) => {
|
|
104
|
+
const event = typeof record.event === 'string' ? record.event : '';
|
|
105
|
+
return GATE_EVENTS.has(event) && !isApprovalRecorded(record);
|
|
106
|
+
});
|
|
107
|
+
const cohortGateEvents = cohortGateRecords.length;
|
|
108
|
+
const cohortWouldBlockCount = cohortGateRecords.filter(inferWouldBlock).length;
|
|
109
|
+
const cohortWouldBlockRate = cohortGateEvents > 0 ? cohortWouldBlockCount / cohortGateEvents : 0;
|
|
110
|
+
const cohortApprovalRecordedCount = cohortRecords.filter(isApprovalRecorded).length;
|
|
111
|
+
const cohortAvailabilityAsks = computeAvailabilityAskCounts(cohortRecords);
|
|
112
|
+
const cohortClassifierWouldBlockCount = Math.max(0, cohortWouldBlockCount - cohortAvailabilityAsks.total);
|
|
113
|
+
const cohortClassifierWouldBlockRate = cohortGateEvents > 0 ? cohortClassifierWouldBlockCount / cohortGateEvents : 0;
|
|
114
|
+
const cohortRoundTrips = buildApprovalRoundTrips(cohortRecords);
|
|
115
|
+
const cohortRepeatedFingerprintAsks = computeRepeatedFingerprintAsks(cohortRecords);
|
|
116
|
+
const cohortNoisyRuleCandidates = detectNoisyRules(cohortRecords, cohortRoundTrips);
|
|
117
|
+
const cohortWouldBlockByReason = computeWouldBlockByReason(cohortRecords);
|
|
118
|
+
const cohortSummaryCounts = new Map();
|
|
119
|
+
for (const record of cohortGateRecords) {
|
|
120
|
+
if (!inferWouldBlock(record)) {
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const reason = typeof record.reason === 'string' ? record.reason : 'unknown';
|
|
124
|
+
const summary = typeof record.summary === 'string' ? record.summary : '';
|
|
125
|
+
const key = `${reason}::${summary}`;
|
|
126
|
+
const existing = cohortSummaryCounts.get(key);
|
|
127
|
+
if (existing) {
|
|
128
|
+
existing.count += 1;
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
cohortSummaryCounts.set(key, { summary, reason, count: 1 });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const cohortTopWouldBlockSummaries = [...cohortSummaryCounts.values()]
|
|
135
|
+
.sort((left, right) => right.count - left.count)
|
|
136
|
+
.slice(0, 10);
|
|
137
|
+
const currentCohort = {
|
|
138
|
+
identity: activeCohort,
|
|
139
|
+
gateEvents: cohortGateEvents,
|
|
140
|
+
excludedGateEvents: gateEvents - cohortGateEvents,
|
|
141
|
+
wouldBlockCount: cohortWouldBlockCount,
|
|
142
|
+
wouldBlockRate: cohortWouldBlockRate,
|
|
143
|
+
classifierWouldBlockCount: cohortClassifierWouldBlockCount,
|
|
144
|
+
classifierWouldBlockRate: cohortClassifierWouldBlockRate,
|
|
145
|
+
approvalRecordedCount: cohortApprovalRecordedCount,
|
|
146
|
+
availabilityAsks: cohortAvailabilityAsks,
|
|
147
|
+
wouldBlockByReason: cohortWouldBlockByReason,
|
|
148
|
+
topWouldBlockSummaries: cohortTopWouldBlockSummaries,
|
|
149
|
+
};
|
|
98
150
|
const mode = options.mode ?? null;
|
|
99
151
|
const unknownLocalEffect = options.unknownLocalEffect ?? null;
|
|
100
152
|
const notes = [];
|
|
101
153
|
let readyForEnforce = false;
|
|
102
154
|
if (mode === 'audit' && unknownLocalEffect === 'deny') {
|
|
103
155
|
notes.push('Dogfood config detected: audit mode with fail-closed shell policy.');
|
|
104
|
-
if (
|
|
105
|
-
notes.push('
|
|
156
|
+
if (!activeCohort) {
|
|
157
|
+
notes.push('Active runtime provenance is unavailable — readiness cannot use historical audit evidence.');
|
|
158
|
+
}
|
|
159
|
+
else if (cohortGateEvents === 0) {
|
|
160
|
+
notes.push('No gate events for the active runtime/config cohort — run normal agent work, then re-check metrics.');
|
|
106
161
|
}
|
|
107
|
-
else if (
|
|
108
|
-
if (
|
|
162
|
+
else if (cohortWouldBlockRate === 0) {
|
|
163
|
+
if (cohortGateEvents >= MIN_GATE_EVENTS_FOR_ENFORCE) {
|
|
109
164
|
readyForEnforce = true;
|
|
110
165
|
notes.push('No would-block events recorded — safe to try mode: "enforce".');
|
|
111
166
|
}
|
|
112
167
|
else {
|
|
113
|
-
notes.push(`Only ${
|
|
168
|
+
notes.push(`Only ${cohortGateEvents} active-cohort gate event(s) recorded — collect at least ${MIN_GATE_EVENTS_FOR_ENFORCE} before enforce.`);
|
|
114
169
|
}
|
|
115
170
|
}
|
|
116
171
|
else {
|
|
117
|
-
notes.push(`${
|
|
118
|
-
if (
|
|
119
|
-
notes.push(`${
|
|
172
|
+
notes.push(`${cohortWouldBlockCount} active-cohort would-block event(s) (${(cohortWouldBlockRate * 100).toFixed(1)}% of gate traffic; classifier-quality ${(cohortClassifierWouldBlockRate * 100).toFixed(1)}%). Review top summaries and correct EffectPlan semantics or resource scope; use exact approval only when the modeled effects are correct.`);
|
|
173
|
+
if (cohortApprovalRecordedCount > 0) {
|
|
174
|
+
notes.push(`${cohortApprovalRecordedCount} active-cohort approval(s) recorded — these likely indicate actions operators wanted.`);
|
|
120
175
|
}
|
|
121
176
|
else {
|
|
122
177
|
notes.push('Review top would-block summaries and correct EffectPlan semantics or resource scope before switching to enforce.');
|
|
123
178
|
}
|
|
124
|
-
if (
|
|
179
|
+
if (cohortClassifierWouldBlockRate < 0.05 &&
|
|
180
|
+
cohortGateEvents >= 20 &&
|
|
181
|
+
cohortAvailabilityAsks.total === 0) {
|
|
125
182
|
readyForEnforce = true;
|
|
126
183
|
notes.push('Classifier-quality would-block rate is below 5% with sufficient sample size — consider enforce mode.');
|
|
127
184
|
}
|
|
@@ -133,16 +190,16 @@ export function computeAuditMetrics(records, options = {}) {
|
|
|
133
190
|
else {
|
|
134
191
|
notes.push('Set policy.unknownLocalEffect to "deny" to dogfood fail-closed defaults.');
|
|
135
192
|
}
|
|
136
|
-
if (
|
|
193
|
+
if (cohortAvailabilityAsks.total > 0) {
|
|
137
194
|
readyForEnforce = false;
|
|
138
|
-
notes.push(`${
|
|
195
|
+
notes.push(`${cohortAvailabilityAsks.total} active-cohort availability-caused ask(s) — tune infrastructure before changing Effect semantics.`);
|
|
139
196
|
notes.push('Ready for enforce withheld while availability-caused asks are present.');
|
|
140
197
|
}
|
|
141
|
-
if (
|
|
142
|
-
notes.push(`${
|
|
198
|
+
if (cohortRepeatedFingerprintAsks.length > 0) {
|
|
199
|
+
notes.push(`${cohortRepeatedFingerprintAsks.length} active-cohort repeated fingerprint ask pattern(s) — review EffectPlan semantics and exact approval history.`);
|
|
143
200
|
}
|
|
144
|
-
if (
|
|
145
|
-
notes.push(`${
|
|
201
|
+
if (cohortNoisyRuleCandidates.length > 0) {
|
|
202
|
+
notes.push(`${cohortNoisyRuleCandidates.length} active-cohort noisy rule candidate(s) — high deny-then-approve rate.`);
|
|
146
203
|
}
|
|
147
204
|
return {
|
|
148
205
|
schemaVersion: AUDIT_METRICS_SCHEMA_VERSION,
|
|
@@ -166,6 +223,7 @@ export function computeAuditMetrics(records, options = {}) {
|
|
|
166
223
|
byEffect,
|
|
167
224
|
byConfidence,
|
|
168
225
|
gateEventsByRuntime,
|
|
226
|
+
currentCohort,
|
|
169
227
|
approvalRecordedCount,
|
|
170
228
|
topWouldBlockSummaries,
|
|
171
229
|
approvalLatency,
|
|
@@ -7,6 +7,9 @@ export interface DogfoodStatus {
|
|
|
7
7
|
gateEvents: number;
|
|
8
8
|
wouldBlockCount: number;
|
|
9
9
|
wouldBlockRate: number;
|
|
10
|
+
excludedGateEvents: number;
|
|
11
|
+
runtimeBuildStamp?: string;
|
|
12
|
+
configFingerprint?: string;
|
|
10
13
|
notes: string[];
|
|
11
14
|
}
|
|
12
15
|
export interface OperationalInsights {
|
|
@@ -8,6 +8,7 @@ export async function loadOperationalInsights(options = {}) {
|
|
|
8
8
|
const repoRoot = path.resolve(options.targetDir ?? process.cwd());
|
|
9
9
|
const config = await loadConfigFile(repoRoot);
|
|
10
10
|
const metrics = await metricsProject({ targetDir: repoRoot });
|
|
11
|
+
const cohort = metrics.currentCohort;
|
|
11
12
|
return {
|
|
12
13
|
repoRoot,
|
|
13
14
|
dogfood: {
|
|
@@ -15,9 +16,12 @@ export async function loadOperationalInsights(options = {}) {
|
|
|
15
16
|
mode: config.mode,
|
|
16
17
|
unknownLocalEffect: config.policy.unknownLocalEffect,
|
|
17
18
|
readyForEnforce: metrics.dogfood.readyForEnforce,
|
|
18
|
-
gateEvents:
|
|
19
|
-
wouldBlockCount:
|
|
20
|
-
wouldBlockRate:
|
|
19
|
+
gateEvents: cohort.gateEvents,
|
|
20
|
+
wouldBlockCount: cohort.wouldBlockCount,
|
|
21
|
+
wouldBlockRate: cohort.wouldBlockRate,
|
|
22
|
+
excludedGateEvents: cohort.excludedGateEvents,
|
|
23
|
+
runtimeBuildStamp: cohort.identity?.runtimeBuildStamp,
|
|
24
|
+
configFingerprint: cohort.identity?.configFingerprint,
|
|
21
25
|
notes: metrics.dogfood.notes,
|
|
22
26
|
},
|
|
23
27
|
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { AuditCohortIdentity } from './core/audit-metrics.js';
|
|
2
|
+
import type { BelayConfigV3 } from './core/config.js';
|
|
3
|
+
export interface InstalledRuntimeProvenance {
|
|
4
|
+
stamp?: string;
|
|
5
|
+
version?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function readInstalledRuntimeProvenance(corePath: string): Promise<InstalledRuntimeProvenance>;
|
|
8
|
+
export declare function resolveActiveAuditCohort(repoRoot: string, config: BelayConfigV3): Promise<AuditCohortIdentity | null>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { getAdapterLayout } from './adapters/layouts/index.js';
|
|
4
|
+
import { resolveScopedPaths } from './adapters/layouts/scope.js';
|
|
5
|
+
import { resolveAdapterName } from './config-io.js';
|
|
6
|
+
import { canonicalStringify, hashValue } from './core/fingerprint.js';
|
|
7
|
+
export async function readInstalledRuntimeProvenance(corePath) {
|
|
8
|
+
try {
|
|
9
|
+
const content = await readFile(corePath, 'utf8');
|
|
10
|
+
const stampMatch = content.match(/RUNTIME_BUILD_STAMP\s*=\s*"([^"]+)"/);
|
|
11
|
+
const versionMatch = content.match(/RUNTIME_PACKAGE_VERSION\s*=\s*"([^"]+)"/);
|
|
12
|
+
return {
|
|
13
|
+
stamp: stampMatch?.[1],
|
|
14
|
+
version: versionMatch?.[1],
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return {};
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export async function resolveActiveAuditCohort(repoRoot, config) {
|
|
22
|
+
const adapter = resolveAdapterName(config);
|
|
23
|
+
const layout = getAdapterLayout(adapter);
|
|
24
|
+
const installScope = config.installScope === 'global' ? 'global' : 'project';
|
|
25
|
+
const scopedPaths = resolveScopedPaths(layout, installScope, repoRoot);
|
|
26
|
+
const runtime = await readInstalledRuntimeProvenance(path.join(scopedPaths.runtimeDir, 'core.mjs'));
|
|
27
|
+
if (!runtime.stamp) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
runtimeBuildStamp: runtime.stamp,
|
|
32
|
+
configFingerprint: hashValue(canonicalStringify(config)),
|
|
33
|
+
};
|
|
34
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -56,6 +56,9 @@ export interface DogfoodStatus {
|
|
|
56
56
|
gateEvents: number;
|
|
57
57
|
wouldBlockCount: number;
|
|
58
58
|
wouldBlockRate: number;
|
|
59
|
+
excludedGateEvents: number;
|
|
60
|
+
runtimeBuildStamp?: string;
|
|
61
|
+
configFingerprint?: string;
|
|
59
62
|
notes: string[];
|
|
60
63
|
}
|
|
61
64
|
export interface ConfigProvenanceNote {
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const PACKAGE_VERSION = "0.8.
|
|
1
|
+
export declare const PACKAGE_VERSION = "0.8.1";
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// Generated by scripts/sync-version.mjs — do not edit.
|
|
2
|
-
export const PACKAGE_VERSION = '0.8.
|
|
2
|
+
export const PACKAGE_VERSION = '0.8.1';
|