@event4u/agent-config 8.9.0 → 8.11.0

package/src/scripts/self_review_gate.ts

@@ -0,0 +1,298 @@
+/**
+ * Self-review gate — the package reviewing its own PRs with the exact machinery
+ * it sells (`adversarial-review` + `agent-security-review`).
+ *
+ * road-to-maintainer-bus-factor Phase 1. Ships ADVISORY + INERT-WITHOUT-SECRET:
+ *   - `--dry-run` (the PR gate job): no API, no spend, no secret. Assembles and
+ *     prints the review plan (which skills, which files, budget estimate) and
+ *     validates the harness. Always exit 0.
+ *   - live `--advisory` (the secret-gated job): loads the two review skill bodies
+ *     as the system prompt, sends the diff to the model, collects structured
+ *     findings, classifies them, records a verdict, and posts a PR review. In
+ *     advisory mode it NEVER blocks the merge (exit 0) — it records what WOULD
+ *     block. If no key is available it is a logged no-op (exit 0), never a failure.
+ *
+ * Turning the gate REQUIRED (block-on-verdict) and wiring the API secret + the
+ * per-PR budget are the maintainer's acts (blocker `self-review-gate-cost`); this
+ * script exposes the `--enforce` path so that flip is a one-flag change, but the
+ * shipped workflow runs advisory only.
+ *
+ * `classifyBlocking` + `gateVerdict` are pure and unit-tested (no API) — mirrors
+ * the `check_quality_regression.gateVerdict` pattern.
+ */
+import { spawnSync } from 'node:child_process';
+import { appendFileSync, existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { AnthropicClient, load_anthropic_key } from './ai_council/clients.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+// ── Finding model ─────────────────────────────────────────────────────
+export type Severity = 'critical' | 'high' | 'medium' | 'low';
+export type FindingKind = 'security' | 'claim' | 'correctness' | 'style';
+
+export interface Finding {
+    severity: Severity;
+    kind: FindingKind;
+    title: string;
+    detail: string;
+    file?: string;
+}
+
+/**
+ * A finding is merge-blocking iff it is security- or claim-affecting AND at
+ * least `high` severity. Style and correctness findings advise; low/medium
+ * security findings advise. Council 2026-07-08 (claude-sonnet-4-5 + gpt-4o):
+ * a 100 %-blocking gate at solo-maintainer token cost gets ignored or gamed —
+ * block ONLY on the narrow security/claim × {critical,high} intersection.
+ */
+export function classifyBlocking(f: Finding): boolean {
+    return (f.kind === 'security' || f.kind === 'claim') && (f.severity === 'critical' || f.severity === 'high');
+}
+
+/**
+ * Deterministic gate verdict. Mirrors `check_quality_regression.gateVerdict`:
+ * exit 2 blocks the merge, exit 0 passes.
+ *   - advisory (default shipped mode): always 0; the caller reports the
+ *     would-block set without failing the check.
+ *   - enforce (maintainer flip): 2 iff any finding is merge-blocking.
+ */
+export function gateVerdict(findings: readonly Finding[], opts: { enforce: boolean }): 0 | 2 {
+    if (!opts.enforce) return 0;
+    return findings.some(classifyBlocking) ? 2 : 0;
+}
+
+// ── Diff collection ───────────────────────────────────────────────────
+/** Generated projections + lockfiles a self-review should not re-read. */
+const SKIP_PREFIXES = ['dist/agent-src/', '.augment/', '.claude/', '.cursor/', '.clinerules/'];
+const SKIP_EXACT = new Set(['.windsurfrules', 'GEMINI.md', 'package-lock.json']);
+
+export function isReviewablePath(p: string): boolean {
+    if (SKIP_EXACT.has(p)) return false;
+    return !SKIP_PREFIXES.some((pre) => p.startsWith(pre));
+}
+
+function changedFiles(baseRef: string): string[] {
+    const r = spawnSync('git', ['diff', '--name-only', `${baseRef}...HEAD`], {
+        cwd: REPO_ROOT,
+        encoding: 'utf8',
+    });
+    if (r.status !== 0) {
+        throw new Error(`git diff failed: ${(r.stderr ?? '').toString().slice(0, 300)}`);
+    }
+    return (r.stdout ?? '')
+        .split('\n')
+        .map((s) => s.trim())
+        .filter(Boolean)
+        .filter(isReviewablePath);
+}
+
+function diffText(baseRef: string, files: string[]): string {
+    if (files.length === 0) return '';
+    const r = spawnSync('git', ['diff', `${baseRef}...HEAD`, '--', ...files], {
+        cwd: REPO_ROOT,
+        encoding: 'utf8',
+        maxBuffer: 32 * 1024 * 1024,
+    });
+    return (r.stdout ?? '').toString();
+}
+
+// ── Review skill bodies (the system prompt) ───────────────────────────
+const REVIEW_SKILLS = ['adversarial-review', 'agent-security-review'] as const;
+
+function loadSkillBody(name: string): string {
+    const p = path.join(REPO_ROOT, 'src', 'skills', name, 'SKILL.md');
+    return readFileSync(p, 'utf8');
+}
+
+function buildSystemPrompt(): string {
+    const bodies = REVIEW_SKILLS.map((s) => `# Skill: ${s}\n\n${loadSkillBody(s)}`).join('\n\n---\n\n');
+    return [
+        'You are the self-review gate for an AI-agent governance package. Apply the',
+        'two skills below to the supplied PR diff. Return ONLY a JSON object of the',
+        'shape {"findings":[{"severity":"critical|high|medium|low",',
+        '"kind":"security|claim|correctness|style","title":"...","detail":"...",',
+        '"file":"optional/path"}]}. Use kind="claim" for an unbacked headline number',
+        'or a proof/CLAIMS-affecting statement, kind="security" for an auth/tenant/',
+        'secret/egress/injection gap. No prose outside the JSON.',
+        '',
+        bodies,
+    ].join('\n');
+}
+
+// ── Plan (dry-run) ────────────────────────────────────────────────────
+export interface ReviewPlan {
+    skills: string[];
+    files: string[];
+    promptChars: number;
+    note: string;
+}
+
+export function buildPlan(baseRef: string): ReviewPlan {
+    const files = changedFiles(baseRef);
+    const diff = diffText(baseRef, files);
+    return {
+        skills: [...REVIEW_SKILLS],
+        files,
+        promptChars: buildSystemPrompt().length + diff.length,
+        note:
+            files.length === 0
+                ? 'No reviewable (non-generated) files changed — the live review would no-op.'
+                : `${files.length} reviewable file(s); live review would send ~${Math.ceil(
+                      (buildSystemPrompt().length + diff.length) / 4,
+                  )} input tokens.`,
+    };
+}
+
+// ── Live review ───────────────────────────────────────────────────────
+function parseFindings(text: string): Finding[] {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start < 0 || end <= start) return [];
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(text.slice(start, end + 1));
+    } catch {
+        return [];
+    }
+    const arr = (parsed as { findings?: unknown })?.findings;
+    if (!Array.isArray(arr)) return [];
+    const sev = new Set<Severity>(['critical', 'high', 'medium', 'low']);
+    const kind = new Set<FindingKind>(['security', 'claim', 'correctness', 'style']);
+    return arr
+        .filter((f): f is Record<string, unknown> => typeof f === 'object' && f !== null)
+        .filter((f) => sev.has(f.severity as Severity) && kind.has(f.kind as FindingKind))
+        .map((f) => ({
+            severity: f.severity as Severity,
+            kind: f.kind as FindingKind,
+            title: String(f.title ?? '').slice(0, 300),
+            detail: String(f.detail ?? '').slice(0, 2000),
+            ...(typeof f.file === 'string' ? { file: f.file } : {}),
+        }));
+}
+
+export function renderReview(findings: Finding[], enforce: boolean): string {
+    const banner =
+        '> HUMAN REVIEW REQUIRED — dogfooded AI adversarial-review + security gate. ' +
+        'Findings are decision support, not a guarantee; detection is probabilistic. ' +
+        'This is a floor, **not** independent human review.';
+    if (findings.length === 0) {
+        return `${banner}\n\n✅ Self-review gate: no findings.`;
+    }
+    const blocking = findings.filter(classifyBlocking);
+    const rows = findings
+        .map((f) => `| ${f.severity} | ${f.kind} | ${f.file ?? '—'} | ${f.title} |`)
+        .join('\n');
+    const verdictLine = blocking.length
+        ? enforce
+            ? `❌ ${blocking.length} merge-blocking finding(s) (security/claim × high+).`
+            : `⚠️ ${blocking.length} finding(s) WOULD block merge under an enforced gate (advisory now).`
+        : '✅ No merge-blocking findings (style/correctness advise only).';
+    return `${banner}\n\n${verdictLine}\n\n| severity | kind | file | finding |\n|---|---|---|---|\n${rows}`;
+}
+
+function postReview(body: string): void {
+    const pr = process.env.PR_NUMBER;
+    if (!pr) {
+        process.stdout.write('::notice::self-review-gate — no PR_NUMBER; printing review instead of posting.\n');
+        process.stdout.write(body + '\n');
+        return;
+    }
+    const r = spawnSync('gh', ['pr', 'comment', pr, '--body', body], { cwd: REPO_ROOT, encoding: 'utf8' });
+    if (r.status !== 0) {
+        process.stdout.write(`::warning::self-review-gate — gh pr comment failed: ${(r.stderr ?? '').toString().slice(0, 300)}\n`);
+        process.stdout.write(body + '\n');
+    }
+}
+
+function resolveKey(): string | null {
+    const env = process.env.ANTHROPIC_API_KEY;
+    if (env && env.trim()) return env.trim();
+    try {
+        return load_anthropic_key();
+    } catch {
+        return null;
+    }
+}
+
+// ── CLI ───────────────────────────────────────────────────────────────
+export function main(argv: string[]): 0 | 2 {
+    const dryRun = argv.includes('--dry-run');
+    const enforce = argv.includes('--enforce');
+    const baseIdx = argv.indexOf('--base');
+    const baseRef = (baseIdx >= 0 ? argv[baseIdx + 1] : undefined) ?? 'origin/main';
+
+    const plan = buildPlan(baseRef);
+
+    if (dryRun) {
+        process.stdout.write(
+            `self-review-gate (dry-run — no spend, advisory):\n` +
+                `  skills: ${plan.skills.join(', ')}\n` +
+                `  files:  ${plan.files.length}\n` +
+                `  ${plan.note}\n`,
+        );
+        return 0;
+    }
+
+    if (plan.files.length === 0) {
+        process.stdout.write('::notice::self-review-gate — no reviewable files changed; skipping.\n');
+        return 0;
+    }
+
+    const key = resolveKey();
+    if (!key) {
+        // Explicit NEUTRAL state — never a bare green that reads as "reviewed".
+        process.stdout.write(
+            '::warning::self-review-gate NEUTRAL — no ANTHROPIC_API_KEY configured, NOTHING was reviewed. ' +
+                'Set the repo secret to enable the live dogfooded review.\n',
+        );
+        const summary = process.env.GITHUB_STEP_SUMMARY;
+        if (summary) {
+            appendFileSync(
+                summary,
+                '### Self-review gate: NEUTRAL\n\nNo `ANTHROPIC_API_KEY` secret — nothing was reviewed. This is not a pass.\n',
+            );
+        }
+        return 0;
+    }
+
+    // Belt-and-suspenders: the WHOLE live path is wrapped so that ANYTHING going
+    // wrong — no API credit / balance (HTTP 402), rate-limit (429), network error,
+    // a client-ctor throw, a parse failure — resolves to a NEUTRAL exit 0, never a
+    // red CI. A missing key or no-credit must never become a merge blocker; the
+    // gate is advisory by design. (Only an --enforce run with real blocking
+    // findings returns 2 — an error is not a finding.)
+    try {
+        const client = new AnthropicClient({ api_key: key });
+        const resp = client.ask(buildSystemPrompt(), diffText(baseRef, plan.files), 4096);
+        if (resp.error) {
+            process.stdout.write(
+                `::warning::self-review-gate NEUTRAL — model call did not complete (${resp.error}); ` +
+                    'nothing was reviewed, not a blocker.\n',
+            );
+            return 0; // no credit / transport / API error → never blocks the merge
+        }
+        const findings = parseFindings(resp.text);
+        const body = renderReview(findings, enforce);
+        postReview(body);
+
+        const code = gateVerdict(findings, { enforce });
+        if (code === 2) {
+            process.stdout.write('::error::self-review-gate — merge-blocking findings under enforce mode.\n');
+        }
+        return code;
+    } catch (e) {
+        process.stdout.write(
+            `::warning::self-review-gate NEUTRAL — unexpected error (${e instanceof Error ? e.message : String(e)}); ` +
+                'nothing was reviewed, not a blocker.\n',
+        );
+        return 0;
+    }
+}
+
+if (existsSync(process.argv[1] ?? '') && import.meta.url === `file://${process.argv[1]}`) {
+    process.exit(main(process.argv.slice(2)));
+}

package/src/scripts/skill_eval_coverage.ts

@@ -26,6 +26,12 @@
  *                  drops below the pinned floor (internal/evals/coverage-floor.json).
  *                  A missing floor file is treated as an all-zero floor
  *                  (bootstrap-inert: never fails before the first --write-floor).
+ *                  ALSO enforces the tier floor (road-to-fable-feedback-5
+ *                  Phase 4a, council 2026-07-10): every `priority`-tier skill
+ *                  (rich + default-surface + router) MUST carry a behavioural
+ *                  evals.json, unless exempted WITH a reason in
+ *                  internal/evals/tier-floor-exemptions.json. No silent
+ *                  exemptions — the bright line replaces weighted scoring.
  *   --write-floor  pin the current coverage as the new floor. Maintainer action
  *                  after authoring evals — the ratchet only ever rises.
  *
@@ -47,6 +53,7 @@ export interface CoverageOptions {
     skillsDir?: string;
     profilesDir?: string;
     floorPath?: string;
+    exemptionsPath?: string;
 }
 
 export interface TierCoverage {
@@ -222,6 +229,52 @@ export function writeFloor(opts: CoverageOptions = {}): CoverageReport {
     return report;
 }
 
+export function _exemptionsPath(opts: CoverageOptions = {}): string {
+    return (
+        opts.exemptionsPath ??
+        path.join(REPO_ROOT, 'internal', 'evals', 'tier-floor-exemptions.json')
+    );
+}
+
+function _readExemptions(p: string): Record<string, string> {
+    try {
+        const raw = JSON.parse(fs.readFileSync(p, 'utf-8')) as {
+            exemptions?: Record<string, unknown>;
+        };
+        const out: Record<string, string> = {};
+        for (const [k, v] of Object.entries(raw.exemptions ?? {})) {
+            if (typeof v === 'string' && v.trim().length > 0) out[k] = v;
+        }
+        return out;
+    } catch {
+        return {};
+    }
+}
+
+/**
+ * Tier floor: every priority-tier skill (rich + default-surface + router)
+ * must carry evals.json, unless exempted with a reason. Stale exemptions
+ * (exempted but now covered, or no longer priority) are reported so the
+ * list cannot silently accumulate.
+ */
+export function checkTierFloor(opts: CoverageOptions = {}): {
+    ok: boolean;
+    violations: string[];
+    stale: string[];
+} {
+    const report = computeCoverage(opts);
+    const exemptions = _readExemptions(_exemptionsPath(opts));
+    const uncovered = report.tiers.priority.uncovered;
+    const violations = uncovered
+        .filter((n) => !(n in exemptions))
+        .map((n) => `priority-tier skill '${n}' has no evals/evals.json and no exemption`);
+    const uncoveredSet = new Set(uncovered);
+    const stale = Object.keys(exemptions)
+        .filter((n) => !uncoveredSet.has(n))
+        .map((n) => `exemption for '${n}' is stale (covered or no longer priority-tier)`);
+    return { ok: violations.length === 0, violations, stale };
+}
+
 /** Ratchet: current coverage must be >= floor, overall and per tier. */
 export function checkRatchet(opts: CoverageOptions = {}): { ok: boolean; regressions: string[] } {
     const report = computeCoverage(opts);
@@ -281,12 +334,27 @@ export function main(argv: readonly string[] = process.argv.slice(2)): number {
 
     if (argv.includes('--check')) {
         const { ok, regressions } = checkRatchet();
-        if (!ok) {
-            process.stdout.write('❌  eval-coverage ratchet regressed:\n');
-            for (const r of regressions) process.stdout.write(`    - ${r}\n`);
+        const floor = checkTierFloor();
+        if (!ok || !floor.ok) {
+            if (!ok) {
+                process.stdout.write('❌  eval-coverage ratchet regressed:\n');
+                for (const r of regressions) process.stdout.write(`    - ${r}\n`);
+            }
+            if (!floor.ok) {
+                process.stdout.write('❌  priority-tier eval floor violated:\n');
+                for (const v of floor.violations) process.stdout.write(`    - ${v}\n`);
+                process.stdout.write(
+                    '    Author a behavioural evals.json, or add an exemption WITH a reason\n' +
+                        '    to internal/evals/tier-floor-exemptions.json.\n',
+                );
+            }
             return 1;
         }
+        for (const s of floor.stale) {
+            process.stdout.write(`⚠️  ${s}\n`);
+        }
         process.stdout.write('✅  eval-coverage ratchet: no regression.\n');
+        process.stdout.write('✅  priority-tier eval floor: every rich/default-surface/router skill covered or exempted-with-reason.\n');
         return 0;
     }
 

package/src/scripts/skill_linter.ts

@@ -1416,6 +1416,7 @@ export function lint_skill(p: string, text: string): LintResult {
             'flux',
             'forecasting',
             'grafana',
+            'humanizer',
             'iconography',
             'laravel',
             'livewire',