@event4u/agent-config 9.6.0 → 9.8.0

package/src/scripts/check_artefact_count_messaging.ts

@@ -53,6 +53,10 @@ export const SURFACES: readonly string[] = [
     'docs/featured-skills.md',
     'docs/featured-commands.md',
     'docs/proof.md',
+    // Public submission text: the block here described version 3.2.0 with a
+    // "4929 tests" figure while the package reached 9.7.0, because nothing
+    // watched this file. Added 2026-07-25 so a hand-typed count here fails CI.
+    'docs/distribution/registries.md',
     'site/src/content/docs/index.mdx',
     'site/src/content/docs/claims.md',
 ];

package/src/scripts/check_backstop_debt.ts

@@ -0,0 +1,200 @@
+#!/usr/bin/env tsx
+/**
+ * Debt ratchet for rule backstops that are red on arrival.
+ *
+ * Wiring the `enforced_by:` validators into CI (`rule-backstops.yml`) surfaced
+ * something the coverage number could not: **five of them were already failing**,
+ * and had been, invisibly, because no workflow ran them. A rule declared a
+ * backstop; the backstop existed; the backstop was red; nothing said so.
+ *
+ * Those failures are pre-existing debt, not regressions from the wiring change,
+ * and fixing eleven framework-leakage spots plus eighteen source-confidentiality
+ * hits is a different piece of work with a different reviewer. Folding them in
+ * would make one PR unreviewable and would mix "make the gate run" with "clean up
+ * what the gate found".
+ *
+ * So the established pattern applies — the one the decision-homing work already
+ * uses and the council endorsed: **baseline the legacy, enforce the boundary.**
+ * Each gate's current finding count is committed. The count may fall freely; it
+ * may not rise. New violations fail, existing debt is visible and counted, and
+ * nothing is silently tolerated.
+ *
+ * What this deliberately is NOT: a `continue-on-error` step. That is a WARN
+ * wearing a gate's clothes, and removing exactly that confusion is why this
+ * workflow exists.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/check_backstop_debt            # ratchet (CI)
+ *   ./scripts-run src/scripts/check_backstop_debt --write-baseline
+ *   ./scripts-run src/scripts/check_backstop_debt --json
+ *
+ * Exit codes: 0 held or improved · 1 a count rose · 2 usage/env error.
+ */
+
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const _HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(_HERE, '..', '..');
+const BASELINE = path.join(REPO_ROOT, 'internal', 'reports', 'rule-backstop-debt.json');
+
+export interface Gate {
+    /** Script stem under `src/scripts/`. */
+    script: string;
+    /** The rule whose `enforced_by:` names it — so a reader knows what is unguarded. */
+    rule: string;
+    /**
+     * Extracts the finding count from the gate's own output.
+     *
+     * Explicit per gate rather than a shared heuristic: these five print their
+     * totals in five different shapes, and a regex that guesses would silently
+     * read 0 the day a gate changes its wording — turning a rising count into a
+     * passing build, which is the failure mode this file exists to prevent.
+     */
+    count: RegExp;
+}
+
+export const GATES: readonly Gate[] = [
+    { script: 'lint_framework_leakage', rule: 'framework-neutrality-in-generic-skills', count: /^(\d+) hits across \d+ files/m },
+    { script: 'check_no_roadmap_refs', rule: 'no-roadmap-references', count: /Found (\d+) roadmap reference\(s\)/ },
+    { script: 'check_council_references', rule: 'no-roadmap-references', count: /(\d+) forbidden council reference\(s\)/ },
+    { script: 'check_no_external_sources', rule: 'source-confidentiality', count: /(\d+) external-source reference\(s\)/ },
+    { script: 'check_token_optimizer_freshness', rule: 'token-optimizer-maintenance', count: /(\d+) drift signal\(s\)/ },
+];
+
+export interface Measured {
+    script: string;
+    rule: string;
+    findings: number;
+    exit_code: number;
+    /** True when the gate ran but its count pattern did not match — never read as 0. */
+    unparsed: boolean;
+}
+
+export function measure(gate: Gate, repo: string = REPO_ROOT): Measured {
+    const r = spawnSync('./scripts-run', [`src/scripts/${gate.script}`], {
+        cwd: repo,
+        encoding: 'utf-8',
+        env: { ...process.env, NO_COLOR: '1' },
+    });
+    const out = `${r.stdout ?? ''}\n${r.stderr ?? ''}`;
+    const m = gate.count.exec(out);
+    // A clean gate legitimately prints no total. Only treat a MISSING count as
+    // unparsed when the gate actually failed — otherwise a green run reads as
+    // broken instrumentation.
+    const failed = (r.status ?? 0) !== 0;
+    return {
+        script: gate.script,
+        rule: gate.rule,
+        findings: m ? Number(m[1]) : 0,
+        exit_code: r.status ?? 0,
+        unparsed: failed && m === null,
+    };
+}
+
+function main(argv: string[]): number {
+    const as_json = argv.includes('--json');
+    const write = argv.includes('--write-baseline');
+
+    const measured = GATES.map((g) => measure(g));
+
+    if (as_json) {
+        process.stdout.write(JSON.stringify({ gates: measured }, null, 2) + '\n');
+        return 0;
+    }
+
+    if (write) {
+        fs.mkdirSync(path.dirname(BASELINE), { recursive: true });
+        fs.writeFileSync(
+            BASELINE,
+            JSON.stringify(
+                {
+                    _doc:
+                        'Pre-existing findings in rule backstops that were red when they were first ' +
+                        'wired into CI — they had been failing invisibly because no workflow ran them. ' +
+                        'Counts may FALL freely; a rise fails the build. This is debt made visible, ' +
+                        'not debt tolerated: every number here is a rule whose declared backstop is ' +
+                        'currently not clean. Regenerate with --write-baseline only when the change ' +
+                        'is deliberate.',
+                    gates: Object.fromEntries(measured.map((m) => [m.script, { rule: m.rule, findings: m.findings }])),
+                    total: measured.reduce((a, m) => a + m.findings, 0),
+                },
+                null,
+                2,
+            ) + '\n',
+        );
+        process.stdout.write(`✅  wrote baseline → ${path.relative(REPO_ROOT, BASELINE)}\n`);
+        return 0;
+    }
+
+    if (!fs.existsSync(BASELINE)) {
+        process.stderr.write(
+            `❌  check_backstop_debt: no baseline at ${path.relative(REPO_ROOT, BASELINE)}; run --write-baseline\n`,
+        );
+        return 2;
+    }
+    const base = JSON.parse(fs.readFileSync(BASELINE, 'utf-8')).gates as Record<string, { findings: number }>;
+
+    const regressions: string[] = [];
+    const improvements: string[] = [];
+    for (const m of measured) {
+        if (m.unparsed) {
+            regressions.push(
+                `${m.script}: failed (exit ${m.exit_code}) but its finding count could not be read — ` +
+                    `the count pattern no longer matches its output. Fix the pattern; do not assume 0.`,
+            );
+            continue;
+        }
+        const was = base[m.script]?.findings;
+        if (was === undefined) {
+            regressions.push(`${m.script}: not in the baseline — add it with --write-baseline`);
+            continue;
+        }
+        if (m.findings > was) {
+            regressions.push(`${m.script} (${m.rule}): findings rose ${was} → ${m.findings}`);
+        } else if (m.findings < was) {
+            improvements.push(`${m.script} (${m.rule}): ${was} → ${m.findings}`);
+        }
+    }
+
+    const total = measured.reduce((a, m) => a + m.findings, 0);
+    process.stdout.write(
+        `rule-backstop debt · ${total} finding(s) across ${measured.filter((m) => m.findings > 0).length} red gate(s)\n`,
+    );
+    for (const m of measured) {
+        if (m.findings > 0) process.stdout.write(`  · ${m.script} (${m.rule}): ${m.findings}\n`);
+    }
+    for (const i of improvements) process.stdout.write(`  ✅ improved — ${i}\n`);
+
+    if (regressions.length > 0) {
+        process.stderr.write('❌  rule-backstop debt ratchet:\n');
+        for (const r of regressions) process.stderr.write(`    · ${r}\n`);
+        process.stderr.write(
+            '    These gates are already failing on pre-existing debt; the ratchet only forbids ADDING to it.\n',
+        );
+        return 1;
+    }
+    if (improvements.length > 0) {
+        process.stdout.write('    Baseline is now loose — regenerate with --write-baseline to lock the improvement.\n');
+    }
+    process.stdout.write('✅  rule-backstop debt ratchet holds\n');
+    return 0;
+}
+
+// Main-guard (realpath-compared, mirrors the repo convention).
+if (process.argv[1] !== undefined) {
+    try {
+        const here = fs.realpathSync(fileURLToPath(import.meta.url));
+        const argv1 = fs.realpathSync(path.resolve(process.argv[1]));
+        if (here === argv1) {
+            process.exit(main(process.argv.slice(2)));
+        }
+    } catch {
+        const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+        if (import.meta.url === argvUrl) {
+            process.exit(main(process.argv.slice(2)));
+        }
+    }
+}

package/src/scripts/check_claims.ts

@@ -15,10 +15,17 @@
  * Ledger entries with `status: unbacked` are inventory (documented debt) and do
  * NOT fail the build — but marking such an entry's claim in prose does.
  *
- * Evidence-pointer grammar (v1):
+ * Evidence-pointer grammar (v2):
  *   <repo-path>[:line]        → the repo file exists (line advisory).
  *   <repo-path>#<substring>   → the file exists AND contains <substring>.
  *   https://… (YYYY-MM-DD)    → external cite with a dated stamp (not fetched).
+ *   exec:<command> -> <code>  → the command RE-RUNS and its exit code matches.
+ *
+ * The first three are existence checks and cannot tell a live claim from a
+ * stale one: a pointer at a report nobody regenerated resolves forever. The
+ * fourth re-derives the claim. It runs in CI only — locally it reports
+ * UNVERIFIED rather than executing anything, because a consumer's checkout has
+ * no business re-running this package's evidence commands.
  *
  * Exit codes: 0 = clean · 2 = unbacked/dangling/missing-entry finding OR usage
  * error. Success prints to stdout; findings print to stderr.
@@ -26,10 +33,17 @@
  * Usage:
  *     ./scripts-run src/scripts/check_claims
  *     ./scripts-run src/scripts/check_claims --quiet
+ *     CI=true ./scripts-run src/scripts/check_claims     # re-runs exec: claims
  */
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import {
+    exec_allowed_here,
+    exec_static_error,
+    parse_exec_pointer,
+    run_exec_evidence,
+} from './_lib/exec_evidence.js';
 
 const _FILE = fileURLToPath(import.meta.url);
 const _HERE = path.dirname(_FILE);
@@ -46,8 +60,47 @@ const URL_DATED = /^https?:\/\/\S+\(\d{4}-\d{2}-\d{2}\)\s*$/;
  *  not appear as a capability claim. docs/proof.md and docs/comparison.yaml
  *  are structurally witnessed (pointer columns + their own linters). */
 const WITNESS_SURFACES = ['README.md', 'CAPABILITIES.yaml'];
-/** A percentage or multiplier reading as a measured capability figure. */
-const QUANTIFIED_CLAIM = /\b\d+(?:\.\d+)?\s*(?:%|[x×](?![A-Za-z0-9]))/;
+
+/**
+ * Shapes that read as a measured capability figure.
+ *
+ * The ratio pattern was written against the "84.8%-style headline" anti-lesson
+ * and catches exactly that. It missed the shape that actually bit us: the README
+ * shipped "compiled into 7+ host agents" while the real, test-pinned number was
+ * 23 detected / 20 emitted — understating coverage by 3x, for months, on the one
+ * surface this sweep already watched. A bare integer carries no `%` and no `x`,
+ * so no ratio pattern could see it and no unit allowlist would either.
+ *
+ * Three narrow classes:
+ *   - RATIO      — percentages and multipliers (the original).
+ *   - MAGNITUDE  — a number carrying a measurement unit. Unit-gated, not
+ *                  digit-gated: a thousands-separator heuristic would catch
+ *                  `22,077` and also every year and large ordinal.
+ *   - SELF_COUNT — a count-shaped assertion about the package's own reach.
+ *                  This is the class that shipped wrong.
+ *
+ * Excluded from MAGNITUDE in v1: time units. "wait 30 seconds" is an
+ * instruction, not a claim, and a gate that false-positives is a gate that gets
+ * bypassed (`narrow > recall`, as with the credential floor).
+ *
+ * Deliberately NOT widened to more surfaces, and this was measured rather than
+ * assumed: `docs/benchmark.md` alone matches the ratio pattern on 54 lines — it
+ * is a methodology document whose job is to be full of statistics, and sweeping
+ * it would produce exactly the flood that teaches a maintainer to bypass the
+ * gate. `docs/proof.md` and `docs/comparison.yaml` are pointer-enforced by their
+ * own linters, so an unmarkered figure cannot originate there. The gap was never
+ * the surface list; it was the pattern.
+ */
+const RATIO = /\b\d+(?:\.\d+)?\s*(?:%|[x×](?![A-Za-z0-9]))/;
+// The optional `<word>-` allows a qualified unit: `13,881 GPT-tokens` is the
+// same claim shape as `13,881 tokens` and was missed without it.
+const MAGNITUDE = /\b\d[\d,._]*\s*(?:[A-Za-z]+-)?(?:tokens?|ms|USD|KB|MB|GB|chars?)\b/i;
+const SELF_COUNT = /\b\d+\+?\s+(?:host agents?|hosts|supported (?:agents?|hosts))\b/i;
+
+/** True when a line carries any figure shape that must bind to a claim. */
+export function is_quantified_claim(line: string): boolean {
+    return RATIO.test(line) || MAGNITUDE.test(line) || SELF_COUNT.test(line);
+}
 
 class ExitCode extends Error {
     code: number;
@@ -57,7 +110,7 @@ class ExitCode extends Error {
     }
 }
 
-interface LedgerEntry {
+export interface LedgerEntry {
     id: string;
     claim: string;
     kind: string;
@@ -170,10 +223,24 @@ function scan_markers(): { id: string; file: string }[] {
     return found;
 }
 
-/** Does an evidence pointer resolve? Returns null on success, else a reason. */
+/**
+ * Does an evidence pointer resolve? Returns null on success, else a reason.
+ *
+ * For `exec:` this is the STATIC half only — is the pointer well-formed and is
+ * the command allowlisted. A malformed or non-allowlisted command fails
+ * everywhere, including locally, because that is a defect in the ledger rather
+ * than a property of the environment. Re-execution is the separate CI-gated
+ * pass in `main`, so a laptop never runs an evidence command.
+ */
 function pointer_unresolved(evidence: string): string | null {
     const ev = evidence.trim();
     if (!ev) return 'empty evidence pointer';
+
+    const exec = parse_exec_pointer(ev);
+    if (exec !== null) {
+        return 'error' in exec ? exec.error : exec_static_error(exec);
+    }
+
     if (/^https?:\/\//.test(ev)) {
         return URL_DATED.test(ev) ? null : 'external URL missing a (YYYY-MM-DD) stamp';
     }
@@ -254,16 +321,100 @@ function main(argv: string[] = process.argv.slice(2)): number {
                 continue;
             }
             if (inFence) continue;
-            if (!QUANTIFIED_CLAIM.test(line)) continue;
-            if (/<!--\s*claim:/.test(line) || /unverified/i.test(line)) continue;
+            if (!is_quantified_claim(line)) continue;
+            if (/unverified/i.test(line)) continue;
+
+            // A marker on the line is not enough — it must be a marker that can
+            // LICENSE A NUMBER. This is how "7+ host agents" survived: the line
+            // already carried `claim:no-runtime-daemon`, a `kind: qual` claim
+            // about having no daemon, and any-marker-exempts-the-line let the
+            // unrelated figure ride along on it. A qualitative claim says nothing
+            // about a quantity, so only a `kind: quant` entry clears a figure.
+            const markers = [...line.matchAll(/<!--\s*claim:([A-Za-z0-9._-]+)\s*-->/g)].map((m) => m[1] as string);
+            const licensed = markers.some((id) => ledger.get(id)?.kind === 'quant');
+            if (licensed) continue;
+
+            const why =
+                markers.length > 0
+                    ? `figure carried only by non-quantitative claim marker(s) [${markers.join(', ')}] — a \`kind: qual\` claim cannot license a number`
+                    : 'quantified claim without a claim marker or \'unverified\' annotation';
             findings.push({
                 id: '(unmarkered)',
                 file: `${rel}:${i + 1}`,
-                reason: `quantified claim without a claim marker or 'unverified' annotation — ${line.trim().slice(0, 70)}`,
+                reason: `${why} — ${line.trim().slice(0, 70)}`,
             });
         }
     }
 
+    // A published denominator must match the live ledger.
+    //
+    // `exec-evidence-feasibility.json` records how many backed claims the
+    // feasibility measurement was taken over. That number was hand-written and
+    // drifted twice inside two days — 25 when the ledger held 26, then 26 when it
+    // held 27 — and CI stayed green both times, because the claim's evidence
+    // pointer resolved. Pointer-resolution is not truth: that is the very thing
+    // `ledger-exec-verifiability` says about the ledger, demonstrated by its own
+    // entry. The classification in that file is a human judgment and stays one;
+    // the denominator is mechanical, so it gets checked.
+    {
+        const rel = 'internal/reports/exec-evidence-feasibility.json';
+        const abs = path.join(REPO, rel);
+        if (fs.existsSync(abs)) {
+            try {
+                const stored = JSON.parse(fs.readFileSync(abs, 'utf8')) as { backed_claims?: number };
+                const live = [...ledger.values()].filter((e) => e.status === 'backed').length;
+                if (typeof stored.backed_claims === 'number' && stored.backed_claims !== live) {
+                    findings.push({
+                        id: '(derived-count)',
+                        file: rel,
+                        reason:
+                            `backed_claims is ${stored.backed_claims} but the ledger holds ${live} — ` +
+                            `a published denominator drifted from its source. Re-measure and update the report.`,
+                    });
+                }
+            } catch {
+                findings.push({
+                    id: '(derived-count)',
+                    file: rel,
+                    reason: 'unparseable JSON — the derived-count check cannot verify the published denominator',
+                });
+            }
+        }
+    }
+
+    // 4. Re-execution pass (`exec:` evidence). CI only — locally every exec
+    //    claim reports UNVERIFIED and nothing runs. A mismatch is a finding: the
+    //    command ran and disagreed with the claim. A skip is NOT a finding —
+    //    collapsing "could not run" into "failed" is how a verifier degrades
+    //    into a rubber stamp in the other direction.
+    const execEntries = [...ledger.values()]
+        .filter((e) => e.status === 'backed')
+        .map((e) => ({ entry: e, ptr: parse_exec_pointer(e.evidence) }))
+        .filter((x): x is { entry: LedgerEntry; ptr: { command: string; expected: number } } =>
+            x.ptr !== null && !('error' in x.ptr),
+        );
+
+    const execSkipped: string[] = [];
+    const execVerified: string[] = [];
+
+    if (execEntries.length > 0) {
+        const allowed = exec_allowed_here();
+        for (const { entry, ptr } of execEntries) {
+            if (!allowed) {
+                execSkipped.push(entry.id);
+                continue;
+            }
+            const outcome = run_exec_evidence(ptr, REPO);
+            if (outcome.mismatch) {
+                findings.push({ id: entry.id, file: LEDGER_REL, reason: outcome.reason });
+            } else if (outcome.verified) {
+                execVerified.push(entry.id);
+            } else {
+                execSkipped.push(entry.id);
+            }
+        }
+    }
+
     if (findings.length > 0) {
         process.stderr.write(`❌  check_claims: ${findings.length} unbacked/dangling claim(s):\n`);
         for (const f of findings) {
@@ -279,6 +430,14 @@ function main(argv: string[] = process.argv.slice(2)): number {
         process.stdout.write(
             `✅  check_claims: ${markers.length} markered claim(s) bound · ledger ${ledger.size} entries (${backed} backed, ${unbacked} unbacked inventory)\n`,
         );
+        if (execEntries.length > 0) {
+            const detail = execVerified.length > 0 ? ` (${execVerified.length} re-verified)` : '';
+            process.stdout.write(
+                execSkipped.length === execEntries.length
+                    ? `    exec: ${execEntries.length} claim(s) UNVERIFIED — re-execution is CI-only, skipped locally\n`
+                    : `    exec: ${execEntries.length} claim(s)${detail}, ${execSkipped.length} skipped\n`,
+            );
+        }
     }
     return 0;
 }
@@ -308,3 +467,24 @@ if (_isCliEntry()) {
 }
 
 export { REPO, LEDGER_REL, main, parse_args, load_ledger, scan_markers, pointer_unresolved, ExitCode };
+
+/**
+ * Live count of `status: backed` ledger entries.
+ *
+ * Exists so a published denominator can be DERIVED instead of typed. The
+ * `ledger-exec-verifiability` entry hard-coded this number twice and drifted
+ * within a day both times — first 25 when the ledger held 26, then 26 when it
+ * held 27 — while `check_claims` stayed green because its evidence pointer
+ * resolved. A number a human retypes on every ledger edit will drift; the only
+ * fix is to stop retyping it.
+ *
+ * Note this is not the same as `grep -c '^- status: backed'`, which also counts
+ * the entry-schema template in the document header.
+ */
+export function count_backed(): number {
+    let n = 0;
+    for (const e of load_ledger().values()) {
+        if (e.status === 'backed') n += 1;
+    }
+    return n;
+}

package/src/scripts/check_comparison.ts

@@ -29,6 +29,18 @@ export interface Row {
     our_evidence: string;
     their_evidence: string;
     checkable: boolean;
+    /**
+     * Optional — the failure mode this row's claim prevents, in adopter-facing
+     * voice. Populated rows additionally render as a second, non-comparative
+     * view; an empty field is the honest signal that a row is comparative only.
+     *
+     * The field exists so that "which rows appear where" is a **data property**
+     * rather than a curatorial decision. Hand-picking rows for a second surface
+     * would be manual curation with weaker enforcement than the projection this
+     * repo already runs — and either way the row's `our_evidence` pointer keeps
+     * being resolved, so no cell escapes the gate by appearing in the new view.
+     */
+    failure_mode?: string;
 }
 
 export function loadRows(repo: string = REPO): Row[] {