@guilz-dev/belay 0.8.1 → 0.9.0

package/dist/corpus/coverage-probe.js

@@ -0,0 +1,341 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { classifierOptionsFromConfig } from '../core/config.js';
+import { classifyShell } from '../core/verdict/adapter.js';
+import { CoverageCompareError, compareBaselineWarnings, compareCoverageReports, formatCoverageCompareReport, parseCoverageProbeReportForCompare, } from './coverage-compare.js';
+import { buildCoverageEvalContexts, hashStableJson, resolvedConfigHash, } from './coverage-contexts.js';
+import { assertKnownContextIds, CoverageMatrixSchemaError, DEFAULT_PROBE_CONTEXT_IDS, defaultCoverageMatrixPath, flattenCoverageCases, loadCoverageMatrix, } from './coverage-matrix.js';
+export class CoverageProbeCliError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'CoverageProbeCliError';
+    }
+}
+export const COVERAGE_PROBE_REPORT_SCHEMA_VERSION = 2;
+function defaultRepoRoot() {
+    return path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+}
+export const defaultClassifyFn = async (command, evalContext) => {
+    const options = {
+        ...classifierOptionsFromConfig(evalContext.config),
+        ...evalContext.options,
+    };
+    return classifyShell(command, evalContext.cwd, evalContext.repoRoot, evalContext.config, options);
+};
+function matchesExpectation(expectation, actual) {
+    if (actual.verdict !== expectation.verdict) {
+        return false;
+    }
+    if (expectation.reason !== undefined && actual.reason !== expectation.reason) {
+        return false;
+    }
+    return true;
+}
+function caseMatchesFilter(testCase, filters) {
+    if (filters.length === 0) {
+        return true;
+    }
+    const filterSet = new Set(filters);
+    if (filterSet.has(testCase.groupId) || filterSet.has(testCase.id)) {
+        return true;
+    }
+    return testCase.tags.some((tag) => filterSet.has(tag));
+}
+async function readPackageVersion(repoRoot) {
+    try {
+        const raw = JSON.parse(await readFile(path.join(repoRoot, 'package.json'), 'utf8'));
+        return raw.version ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+function readGitSha() {
+    return process.env.GITHUB_SHA ?? process.env.GIT_SHA ?? null;
+}
+function initBucket() {
+    return { total: 0, matched: 0, mismatched: 0, observeOnly: 0 };
+}
+function recordBucket(bucket, result) {
+    bucket.total += 1;
+    if (result.observeOnly) {
+        bucket.observeOnly += 1;
+        return;
+    }
+    if (result.match) {
+        bucket.matched += 1;
+    }
+    else {
+        bucket.mismatched += 1;
+    }
+}
+export async function evaluateCoverageMatrix(matrix, options = {}) {
+    const classifyFn = options.classifyFn ?? defaultClassifyFn;
+    const contextIds = options.contextIds ?? [...DEFAULT_PROBE_CONTEXT_IDS];
+    const repoRoot = path.resolve(options.repoRoot ?? defaultRepoRoot());
+    const filters = options.filters ?? [];
+    const contexts = options.evalContexts ?? (await buildCoverageEvalContexts(contextIds, repoRoot));
+    const cases = flattenCoverageCases(matrix).filter((testCase) => caseMatchesFilter(testCase, filters));
+    const results = [];
+    for (const testCase of cases) {
+        for (const evalContext of contexts) {
+            const expectation = testCase.expectations?.[evalContext.id];
+            const observeOnly = expectation === undefined;
+            const actual = await classifyFn(testCase.command, evalContext);
+            const match = observeOnly || expectation === undefined ? null : matchesExpectation(expectation, actual);
+            results.push({
+                caseId: testCase.id,
+                groupId: testCase.groupId,
+                command: testCase.command,
+                commandHash: hashStableJson(testCase.command),
+                context: evalContext.id,
+                tags: testCase.tags,
+                observeOnly,
+                expectation,
+                expectationHash: expectation ? hashStableJson(expectation) : null,
+                actual: {
+                    verdict: actual.verdict,
+                    reason: actual.reason,
+                    fingerprint: actual.fingerprint,
+                },
+                match,
+            });
+        }
+    }
+    return results;
+}
+function summarizeResults(results, filters) {
+    const summary = {
+        total: results.length,
+        observeOnly: 0,
+        matched: 0,
+        mismatched: 0,
+        filteredEmpty: filters.length > 0 && results.length === 0,
+        byGroup: {},
+        byTag: {},
+    };
+    for (const result of results) {
+        if (result.observeOnly) {
+            summary.observeOnly += 1;
+        }
+        else if (result.match) {
+            summary.matched += 1;
+        }
+        else {
+            summary.mismatched += 1;
+        }
+        if (!summary.byGroup[result.groupId]) {
+            summary.byGroup[result.groupId] = initBucket();
+        }
+        recordBucket(summary.byGroup[result.groupId], result);
+        for (const tag of result.tags) {
+            if (!summary.byTag[tag]) {
+                summary.byTag[tag] = initBucket();
+            }
+            recordBucket(summary.byTag[tag], result);
+        }
+    }
+    return summary;
+}
+function resultsSignature(results) {
+    return hashStableJson(results.map((result) => ({
+        caseId: result.caseId,
+        context: result.context,
+        verdict: result.actual.verdict,
+        reason: result.actual.reason,
+        fingerprint: result.actual.fingerprint,
+    })));
+}
+export async function runCoverageProbe(options = {}) {
+    const repoRoot = path.resolve(options.repoRoot ?? defaultRepoRoot());
+    const matrixPath = path.resolve(options.matrixPath ?? defaultCoverageMatrixPath(repoRoot));
+    const matrix = await loadCoverageMatrix(matrixPath);
+    const contextIds = options.contextIds ?? [...DEFAULT_PROBE_CONTEXT_IDS];
+    const contexts = await buildCoverageEvalContexts(contextIds, repoRoot);
+    const repeat = Math.max(1, options.repeat ?? 1);
+    const filters = options.filters ?? [];
+    const probeOptions = { ...options, evalContexts: contexts };
+    let results = await evaluateCoverageMatrix(matrix, probeOptions);
+    let driftRuns = 0;
+    const firstResults = results;
+    const firstSignature = resultsSignature(results);
+    for (let run = 2; run <= repeat; run += 1) {
+        const nextResults = await evaluateCoverageMatrix(matrix, probeOptions);
+        if (resultsSignature(nextResults) !== firstSignature) {
+            driftRuns += 1;
+        }
+    }
+    results = firstResults;
+    const mismatches = results.filter((result) => !result.observeOnly && result.match === false);
+    return {
+        reportSchemaVersion: COVERAGE_PROBE_REPORT_SCHEMA_VERSION,
+        generatedAt: new Date().toISOString(),
+        packageVersion: await readPackageVersion(repoRoot),
+        gitSha: readGitSha(),
+        matrixPath,
+        matrixHash: hashStableJson(matrix),
+        repeat,
+        driftRuns,
+        contexts: contexts.map((evalContext) => ({
+            id: evalContext.id,
+            cwd: evalContext.cwd,
+            repoRoot: evalContext.repoRoot,
+            resolvedConfigHash: resolvedConfigHash(evalContext),
+            configProvenance: evalContext.configProvenance,
+        })),
+        summary: summarizeResults(results, filters),
+        results,
+        mismatches,
+    };
+}
+export function formatCoverageProbeSummary(report) {
+    const lines = [
+        'Coverage probe',
+        `  cases: ${report.summary.total} (observe-only: ${report.summary.observeOnly})`,
+        `  matched: ${report.summary.matched}`,
+        `  mismatched: ${report.summary.mismatched}`,
+        `  repeat: ${report.repeat} (drift runs: ${report.driftRuns})`,
+        '  contexts:',
+    ];
+    for (const evalContext of report.contexts) {
+        lines.push(`    ${evalContext.id}: cwd=${evalContext.cwd} configHash=${evalContext.resolvedConfigHash.slice(0, 12)}…`);
+    }
+    if (report.mismatches.length > 0) {
+        lines.push('  mismatches:');
+        for (const mismatch of report.mismatches.slice(0, 15)) {
+            lines.push(`    - [${mismatch.context}] ${mismatch.caseId}: expected=${JSON.stringify(mismatch.expectation)} actual=${mismatch.actual.verdict}/${mismatch.actual.reason}`);
+        }
+        if (report.mismatches.length > 15) {
+            lines.push(`    … and ${report.mismatches.length - 15} more`);
+        }
+    }
+    if (report.summary.filteredEmpty) {
+        lines.push('  warning: filter matched zero cases');
+    }
+    return lines.join('\n');
+}
+export function parseCoverageProbeCliArgs(argv) {
+    let repoRoot;
+    let matrixPath;
+    let contextIds;
+    let filters = [];
+    let repeat = 1;
+    let strict = false;
+    let outputDir;
+    let comparePath;
+    let json = false;
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        if (arg === '--repo-root' && argv[i + 1]) {
+            repoRoot = argv[++i];
+        }
+        else if (arg === '--matrix' && argv[i + 1]) {
+            matrixPath = argv[++i];
+        }
+        else if (arg === '--context' && i + 1 < argv.length) {
+            const rawContexts = argv[++i]
+                .split(',')
+                .map((value) => value.trim())
+                .filter(Boolean);
+            contextIds = assertKnownContextIds(rawContexts);
+        }
+        else if (arg === '--filter' && i + 1 < argv.length) {
+            filters = argv[++i]
+                .split(',')
+                .map((value) => value.trim())
+                .filter(Boolean);
+            if (filters.length === 0) {
+                throw new CoverageProbeCliError('--filter requires at least one tag, group id, or case id');
+            }
+        }
+        else if (arg === '--repeat' && argv[i + 1]) {
+            repeat = Number.parseInt(argv[++i], 10);
+            if (!Number.isFinite(repeat) || repeat < 1) {
+                throw new CoverageProbeCliError('--repeat must be a positive integer');
+            }
+        }
+        else if (arg === '--output-dir' && argv[i + 1]) {
+            outputDir = argv[++i];
+        }
+        else if (arg === '--compare' && argv[i + 1]) {
+            comparePath = argv[++i];
+        }
+        else if (arg === '--strict') {
+            strict = true;
+        }
+        else if (arg === '--json') {
+            json = true;
+        }
+    }
+    return { repoRoot, matrixPath, contextIds, filters, repeat, strict, outputDir, comparePath, json };
+}
+export async function main(argv = process.argv.slice(2)) {
+    try {
+        const parsed = parseCoverageProbeCliArgs(argv);
+        const report = await runCoverageProbe({
+            repoRoot: parsed.repoRoot,
+            matrixPath: parsed.matrixPath,
+            contextIds: parsed.contextIds,
+            filters: parsed.filters,
+            repeat: parsed.repeat,
+            outputDir: parsed.outputDir,
+            json: parsed.json,
+        });
+        if (parsed.json) {
+            console.log(JSON.stringify(report, null, 2));
+        }
+        else {
+            console.log(formatCoverageProbeSummary(report));
+        }
+        if (parsed.outputDir) {
+            await mkdir(parsed.outputDir, { recursive: true });
+            const stamp = report.generatedAt.replace(/[:.]/g, '-');
+            const outputPath = path.join(parsed.outputDir, `run-${stamp}.json`);
+            await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+            if (!parsed.json) {
+                console.log(`\nWrote ${outputPath}`);
+            }
+        }
+        if (parsed.comparePath) {
+            let baselineRaw;
+            try {
+                baselineRaw = JSON.parse(await readFile(path.resolve(parsed.comparePath), 'utf8'));
+            }
+            catch {
+                throw new CoverageCompareError(`--compare baseline is not valid JSON: ${parsed.comparePath}`);
+            }
+            const baseline = parseCoverageProbeReportForCompare(baselineRaw);
+            const compareReport = compareCoverageReports(baseline, report);
+            for (const warning of compareBaselineWarnings(baseline, report)) {
+                console.warn(`\nWARNING: compare baseline mismatch: ${warning}`);
+            }
+            if (!parsed.json) {
+                console.log(`\n${formatCoverageCompareReport(compareReport)}`);
+            }
+            if (compareReport.configDrift.length > 0) {
+                console.warn(`\nWARNING: config drift detected for ${compareReport.configDrift.length} context(s); compare may include classifier drift from config changes`);
+            }
+        }
+        if (report.driftRuns > 0 && !parsed.json) {
+            console.warn(`\nWARNING: classifier drift detected in ${report.driftRuns}/${report.repeat} repeats; report uses first run results`);
+        }
+        if (report.summary.filteredEmpty) {
+            console.warn('\nWARNING: --filter matched zero cases; check tag/group/case ids');
+            return 2;
+        }
+        if (parsed.strict && report.summary.mismatched > 0) {
+            return 1;
+        }
+        return 0;
+    }
+    catch (error) {
+        if (error instanceof CoverageProbeCliError ||
+            error instanceof CoverageMatrixSchemaError ||
+            error instanceof CoverageCompareError) {
+            console.error(error.message);
+            return 1;
+        }
+        throw error;
+    }
+}

package/dist/corpus/evaluate.d.ts

@@ -49,7 +49,7 @@ export interface FpCommandBatchResult {
     mutatorId: string;
     isFp: boolean;
 }
-/** False-positive probe: must-allow cores that receive deny_pending_approval count as FP. */
+/** False-positive probe: benign probe cores that receive deny_pending_approval count as FP. */
 export declare function evaluateFpCommandBatch(cases: Array<{
     command: string;
     core: string;

package/dist/corpus/evaluate.js

@@ -102,7 +102,7 @@ export async function evaluateCommandBatch(cases, repoRoot = defaultCorpusEvalPa
     }
     return results;
 }
-/** False-positive probe: must-allow cores that receive deny_pending_approval count as FP. */
+/** False-positive probe: benign probe cores that receive deny_pending_approval count as FP. */
 export async function evaluateFpCommandBatch(cases, repoRoot = defaultCorpusEvalPaths().repoRoot) {
     const { cwd } = defaultCorpusEvalPaths(repoRoot);
     const options = classifierOptionsFromConfig(DEFAULT_CONFIG_V3);

package/dist/corpus/structural-fixture-root.d.ts

@@ -0,0 +1,2 @@
+/** Resolve structural suite fixture root for a repository checkout. */
+export declare function structuralFixtureRoot(repoRoot: string): string;

package/dist/corpus/structural-fixture-root.js

@@ -0,0 +1,5 @@
+import path from 'node:path';
+/** Resolve structural suite fixture root for a repository checkout. */
+export function structuralFixtureRoot(repoRoot) {
+    return path.join(repoRoot, 'src', '__tests__', 'verdict', 'fixtures');
+}

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const PACKAGE_VERSION = "0.8.1";
+export declare const PACKAGE_VERSION = "0.9.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 // Generated by scripts/sync-version.mjs — do not edit.
-export const PACKAGE_VERSION = '0.8.1';
+export const PACKAGE_VERSION = '0.9.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@guilz-dev/belay",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "description": "Belay-style approval and audit gating for agent runtimes.",
   "type": "module",
   "license": "MIT",
@@ -60,6 +60,8 @@
     "test:stable": "pnpm build && vitest run && vitest run && vitest run",
     "corpus": "pnpm build && node scripts/corpus.mjs",
     "probe:adversarial": "pnpm build && node scripts/adversarial-probe.mjs",
+    "probe:coverage": "pnpm build && node scripts/coverage-probe.mjs",
+    "probe:coverage:repeat": "pnpm build && node scripts/coverage-probe.mjs --repeat 3",
     "corpus:ratchet": "pnpm build && node scripts/corpus-ratchet.mjs"
   },
   "devDependencies": {

package/skills/belay/SKILL.md

@@ -53,10 +53,28 @@ To restore the legacy two-step UX (approve, then always retry manually), set
 
 For why it was blocked, use `/belay why <command>` or `belay explain --command "<command>"`.
 For the latest pending ask, use `/belay explain` or `belay explain`.
+
+**Do not use command allowlists** (`overrides.allow`) or legacy standing-allow records
+(shell, tool, or subagent) to fix blocks; none of them change runtime authorization.
+Improve EffectPlan semantics, approve once with `/belay-approve`, or use an exact
+resource-scoped grant. See ADR-005 in the belay repository.
 For install health and audit visibility, use `/belay status` or `belay status`.
 For audit-only summary, use `/belay report` or `belay report`.
 For recovery advice after a block, use `/belay recover` or `belay recover`.
 
+## Contained unknown execution
+
+When an operator has explicitly enabled Docker contained execution, an eligible repository-local
+`unknown_local_effect` may run once in a disposable mirror instead of using a command allowlist.
+The route has no network, never mounts the source workspace or control plane, discards all guest
+changes, and always applies mandatory credential scrubbing to its 16 KiB stdout/stderr tails.
+
+If Belay reports that the contained capability, image, or Docker substrate no longer matches, ask
+the operator to run `belay session start`; do not silently run it from this skill. Only typed Docker
+substrate/daemon unavailability returns to the normal one-shot approval flow. Missing or stale
+capability/image, mirror/lease, create/inspect/start, timeout, and cleanup failures deny without
+approval or host replay.
+
 ## Install or repair
 
 - Interactive setup: `belay config`
@@ -75,4 +93,5 @@ Do not run init, config, or doctor implicitly from this skill — only when the
 | Status / dogfood / audit visibility | `belay status` |
 | Audit summary (read-only) | `belay report` |
 | Recovery advice (advisory only) | `belay recover` |
+| Refresh contained Docker attestation | `belay session start` |
 | Approve once | `belay approve <id>` |

package/skills/belay/belay-status.md

@@ -6,4 +6,9 @@ belay status
 
 For audit-only summary: `belay report`. For recovery advice: `belay recover`.
 
+For opt-in contained execution, report the configured and attested Docker capability separately
+from L1-full. If the image, executable, Unix socket, daemon, or limits no longer match, suggest
+`belay session start`. Do not run it implicitly. Contained guest output is always credential-
+scrubbed and capped even when ordinary audit redaction switches are disabled.
+
 If hooks are missing, suggest `npx @guilz-dev/belay init` and `belay doctor`.

package/dist/corpus/must-allow-commands.d.ts

@@ -1,6 +0,0 @@
-/**
- * Structural-suite MUST-ALLOW CI expectations.
- * These commands guard classifier availability in tests and never grant runtime authority.
- * @see src/__tests__/verdict/structural-suite.test.ts
- */
-export declare const MUST_ALLOW_SHELL_COMMANDS: readonly ["npm test", "npm run build", "pnpm test", "pnpm build", "pnpm vitest run src/example.test.ts", "bash -lc 'git status'", "belay approve belay_deadbeef1234"];