@event4u/agent-config 8.9.0 → 8.10.0

package/src/scripts/orchestration_record.ts

@@ -11,6 +11,7 @@
  *     --spawn-count 1 --token-delta -72000 --provenance measured \
  *     --tier-chosen lite --tier-source inferred --task-class read-only-fanout \
  *     [--tiers sonnet,opus] [--wall-clock-ms 18500] [--dispatch-outcome DONE] \
+ *     [--first-pass-success true|false] [--escalated true|false] \
  *     [--dir <audit-dir>] [--dry-run]
  *
  * Read by `src/scripts/orchestration_savings_report.ts`.
@@ -65,6 +66,19 @@ function str(flags: Flags, key: string): string | undefined {
     return typeof v === 'string' ? v : undefined;
 }
 
+/**
+ * Parse an explicit boolean flag value. `--key true|false` → boolean; a bare
+ * `--key` counts as `true`; any other value is passed through so the lib's
+ * non-boolean validation rejects it with a clear error.
+ */
+function bool(flags: Flags, key: string): boolean | undefined {
+    const v = flags[key];
+    if (v === undefined) return undefined;
+    if (v === true || v === 'true') return true;
+    if (v === 'false') return false;
+    return v as unknown as boolean; // invalid string → caught by the lib's typeof validation
+}
+
 function int(flags: Flags, key: string): number | undefined {
     const v = str(flags, key);
     if (v === undefined) return undefined;
@@ -104,6 +118,8 @@ export function main(argv: string[] = process.argv.slice(2)): number {
         session_tier: str(flags, 'session-tier'),
         dispatch_outcome: str(flags, 'dispatch-outcome') as DispatchOutcome | undefined,
         verify_mode: str(flags, 'verify-mode') as VerifyMode | undefined,
+        first_pass_success: bool(flags, 'first-pass-success'),
+        escalated: bool(flags, 'escalated'),
         phase: str(flags, 'phase') as LinePhase | undefined,
         outcome: str(flags, 'outcome') as LineOutcome | undefined,
         confidence_band: str(flags, 'confidence-band') as Band | undefined,

package/src/scripts/orchestration_savings_report.ts

@@ -5,6 +5,11 @@
  * and prints an aggregate token-savings report. Read-only; no network, no
  * mutation. Handles a missing/empty audit dir gracefully.
  *
+ * QUALITY × COST PAIRING (council verdict): the savings figures never render
+ * without the quality columns (first_pass_success_rate / escalation_rate).
+ * When ≥ 20 lines in the window carry the quality fields the real rates
+ * render; below 20 the columns render as "n/a (n=<count>)" beside savings.
+ *
  * Usage:
  *   ./scripts-run src/scripts/orchestration_savings_report \
  *     [--dir <path>] [--format text|json] [--weights lite=1,medium=5,high=15]
@@ -77,13 +82,41 @@ export function readAuditLines(dir: string): AuditLine[] {
     return lines;
 }
 
-function renderText(r: SavingsReport, dir: string): string {
+/** Dispatches carrying the quality fields at/above this render the real rates; below, "n/a (n=<count>)". */
+export const QUALITY_GATE_MIN_LINES = 20;
+
+function pct(v: number): string {
+    return `${(v * 100).toFixed(0)}%`;
+}
+
+/**
+ * Render the paired cost × quality block. Council verdict: savings NEVER render
+ * without the quality columns — at n ≥ QUALITY_GATE_MIN_LINES the real rates
+ * render; below, the quality columns render as "n/a (n=<count>)" alongside the
+ * savings figures. There is no savings-only render path.
+ */
+function renderPairedCostQuality(r: SavingsReport): string[] {
+    const q = r.quality;
+    const gated = q.quality_lines >= QUALITY_GATE_MIN_LINES;
+    const fps = gated && q.first_pass_success_rate !== null
+        ? `${pct(q.first_pass_success_rate)} (n=${q.first_pass_lines})`
+        : `n/a (n=${q.first_pass_lines})`;
+    const esc = gated && q.escalation_rate !== null
+        ? `${pct(q.escalation_rate)} (n=${q.escalated_lines})`
+        : `n/a (n=${q.escalated_lines})`;
+    return [
+        '  cost × quality (paired — never savings alone):',
+        `    net token_delta: ${r.net_token_delta} (negative = net saved)   | first_pass_success_rate: ${fps}`,
+        `    tokens saved: ${r.tokens_saved} · tokens added: ${r.tokens_added}   | escalation_rate: ${esc}`,
+    ];
+}
+
+export function renderText(r: SavingsReport, dir: string): string {
     const out: string[] = [];
     out.push('Orchestration savings report');
     out.push(`  source: ${dir}`);
     out.push(`  dispatches: ${r.dispatches}  (total spawns: ${r.total_spawns})`);
-    out.push(`  net token_delta: ${r.net_token_delta}  (negative = net saved)`);
-    out.push(`  tokens saved: ${r.tokens_saved}   tokens added: ${r.tokens_added}`);
+    out.push(...renderPairedCostQuality(r));
     out.push(`  provenance: measured ${r.by_provenance.measured.dispatches} (Δ ${r.by_provenance.measured.net_token_delta}) · estimated ${r.by_provenance.estimated.dispatches} (Δ ${r.by_provenance.estimated.net_token_delta})`);
     out.push(`  measured share: ${(r.measured_share * 100).toFixed(0)}%`);
     if (Object.keys(r.by_tier).length) {

package/src/scripts/second_brain_retrieval.ts

@@ -38,6 +38,7 @@ import {
     _setMemoryRoot,
     retrieve,
 } from './memory_lookup.js';
+import { sanitize_text } from './_lib/retrieval_sanitize.js';
 import { scoreTask } from './second_brain_score.js';
 
 const _HERE = fileURLToPath(import.meta.url);
@@ -67,11 +68,19 @@ export function bindStore(): void {
     _setIntakeRoot(path.join(STORE_DIR, 'intake-none'));
 }
 
-export function loadCorpus(): { type: string; k: number; tasks: RetrievalTask[] } {
+export function loadCorpus(): {
+    type: string;
+    k: number;
+    tasks: RetrievalTask[];
+    stale_ids?: string[];
+    poisoned_ids?: string[];
+} {
     const doc = YAML.parse(fs.readFileSync(path.join(STORE_DIR, 'retrieval-corpus.yml'), 'utf-8')) as {
         type: string;
         k: number;
         tasks: RetrievalTask[];
+        stale_ids?: string[];
+        poisoned_ids?: string[];
     };
     return doc;
 }
@@ -162,11 +171,17 @@ export function signTestP(wins: number, losses: number): number {
 
 export async function run(opts: { mode: 'dry-run' | 'live'; host: string; seeds: number }): Promise<Record<string, unknown>> {
     bindStore();
-    const { type, k, tasks } = loadCorpus();
+    const { type, k, tasks, stale_ids = [], poisoned_ids = [] } = loadCorpus();
+    const staleSet = new Set(stale_ids);
+    const poisonedSet = new Set(poisoned_ids);
     const apiKey = opts.mode === 'live' ? load_anthropic_key() : '';
     const placebo = _placeboBodies(type);
 
     let precisionHits = 0;
+    let topkSlots = 0;
+    let staleHits = 0;
+    let poisonedRetrieved = 0;
+    let poisonedNeutralized = 0;
     let tieSetSum = 0;
     const perTask: Array<Record<string, unknown>> = [];
     const armPass: Record<Arm, number> = { 'retrieval-off': 0, 'retrieval-on': 0, placebo: 0 };
@@ -177,6 +192,18 @@ export async function run(opts: { mode: 'dry-run' | 'live'; host: string; seeds:
     for (const task of tasks) {
         const { topk, rank, tieSetSize } = retrieveFor(task, type, k);
         const ids = topk.map((r) => r.id);
+        topkSlots += topk.length;
+        for (const hit of topk) {
+            if (staleSet.has(hit.id)) staleHits += 1;
+            if (poisonedSet.has(hit.id)) {
+                poisonedRetrieved += 1;
+                // Rejection = the sanitize floor strips every hidden-instruction
+                // vector before the body could reach a prompt.
+                const clean = sanitize_text(hit.body);
+                if (!/[​-‏‪-‮⁦-⁩]/u.test(clean)) poisonedNeutralized += 1;
+            }
+        }
+
         const neededInTopK = rank >= 0 && rank < k;
         if (neededInTopK) precisionHits += 1;
         tieSetSum += tieSetSize;
@@ -226,6 +253,21 @@ export async function run(opts: { mode: 'dry-run' | 'live'; host: string; seeds:
         k,
         store: 'internal/bench/second-brain/retrieval-store/ (5 needed + 19 distractors, keyword-overlapping confusers)',
         precision_at_k: precisionAtK,
+        // road-to-proof-under-real-conditions Phase 5 — retrieval quality pair:
+        // stale_hit_rate: fraction of retrieved top-k slots occupied by
+        // SUPERSEDED store entries (lower is better; the substrate has no
+        // recency/supersede weighting yet, so a non-zero rate is the honest
+        // baseline the FTS5/recency activation path improves on).
+        stale_hit_rate: topkSlots ? staleHits / topkSlots : 0,
+        stale_hits: staleHits,
+        topk_slots: topkSlots,
+        // poisoned_rejection_rate: of poisoned entries that WERE retrieved,
+        // the fraction whose hidden-instruction vectors the sanitize floor
+        // strips (target 1.0; null when none was retrieved this run).
+        poisoned_rejection_rate: poisonedRetrieved ? poisonedNeutralized / poisonedRetrieved : null,
+        poisoned_retrieved: poisonedRetrieved,
+        poisoned_neutralized: poisonedNeutralized,
+
         precision_hits: precisionHits,
         mean_tie_set_size: meanTieSet,
         discrimination_note:

package/src/scripts/self_review_gate.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+    classifyBlocking,
+    gateVerdict,
+    isReviewablePath,
+    renderReview,
+    type Finding,
+} from './self_review_gate.js';
+
+const f = (severity: Finding['severity'], kind: Finding['kind']): Finding => ({
+    severity,
+    kind,
+    title: `${severity} ${kind}`,
+    detail: 'x',
+});
+
+describe('classifyBlocking', () => {
+    it('blocks security + high and claim + critical', () => {
+        expect(classifyBlocking(f('high', 'security'))).toBe(true);
+        expect(classifyBlocking(f('critical', 'claim'))).toBe(true);
+    });
+    it('does not block low/medium security, or any style/correctness', () => {
+        expect(classifyBlocking(f('medium', 'security'))).toBe(false);
+        expect(classifyBlocking(f('low', 'claim'))).toBe(false);
+        expect(classifyBlocking(f('critical', 'style'))).toBe(false);
+        expect(classifyBlocking(f('high', 'correctness'))).toBe(false);
+    });
+});
+
+describe('gateVerdict', () => {
+    const blocking = [f('high', 'security')];
+    it('advisory (default shipped mode) never blocks, even with a blocking finding', () => {
+        expect(gateVerdict(blocking, { enforce: false })).toBe(0);
+    });
+    it('enforce blocks only on a merge-blocking finding', () => {
+        expect(gateVerdict(blocking, { enforce: true })).toBe(2);
+        expect(gateVerdict([f('critical', 'style')], { enforce: true })).toBe(0);
+        expect(gateVerdict([], { enforce: true })).toBe(0);
+    });
+});
+
+describe('isReviewablePath', () => {
+    it('skips generated projections + lockfiles, keeps source', () => {
+        expect(isReviewablePath('dist/agent-src/skills/x/SKILL.md')).toBe(false);
+        expect(isReviewablePath('.augment/rules/x.md')).toBe(false);
+        expect(isReviewablePath('.windsurfrules')).toBe(false);
+        expect(isReviewablePath('package-lock.json')).toBe(false);
+        expect(isReviewablePath('src/rules/x.md')).toBe(true);
+        expect(isReviewablePath('src/scripts/y.ts')).toBe(true);
+    });
+});
+
+describe('renderReview', () => {
+    it('carries the HUMAN REVIEW REQUIRED banner + the floor-not-human-review caveat', () => {
+        const out = renderReview([], false);
+        expect(out).toContain('HUMAN REVIEW REQUIRED');
+        expect(out).toContain('not** independent human review');
+    });
+    it('advisory phrasing says WOULD block, enforce phrasing says blocking', () => {
+        const blocking = [f('high', 'security')];
+        expect(renderReview(blocking, false)).toContain('WOULD block');
+        expect(renderReview(blocking, true)).toContain('merge-blocking');
+    });
+});

package/src/scripts/self_review_gate.ts

@@ -0,0 +1,272 @@
+/**
+ * Self-review gate — the package reviewing its own PRs with the exact machinery
+ * it sells (`adversarial-review` + `agent-security-review`).
+ *
+ * road-to-maintainer-bus-factor Phase 1. Ships ADVISORY + INERT-WITHOUT-SECRET:
+ *   - `--dry-run` (the PR gate job): no API, no spend, no secret. Assembles and
+ *     prints the review plan (which skills, which files, budget estimate) and
+ *     validates the harness. Always exit 0.
+ *   - live `--advisory` (the secret-gated job): loads the two review skill bodies
+ *     as the system prompt, sends the diff to the model, collects structured
+ *     findings, classifies them, records a verdict, and posts a PR review. In
+ *     advisory mode it NEVER blocks the merge (exit 0) — it records what WOULD
+ *     block. If no key is available it is a logged no-op (exit 0), never a failure.
+ *
+ * Turning the gate REQUIRED (block-on-verdict) and wiring the API secret + the
+ * per-PR budget are the maintainer's acts (blocker `self-review-gate-cost`); this
+ * script exposes the `--enforce` path so that flip is a one-flag change, but the
+ * shipped workflow runs advisory only.
+ *
+ * `classifyBlocking` + `gateVerdict` are pure and unit-tested (no API) — mirrors
+ * the `check_quality_regression.gateVerdict` pattern.
+ */
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { AnthropicClient, load_anthropic_key } from './ai_council/clients.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+// ── Finding model ─────────────────────────────────────────────────────
+export type Severity = 'critical' | 'high' | 'medium' | 'low';
+export type FindingKind = 'security' | 'claim' | 'correctness' | 'style';
+
+export interface Finding {
+    severity: Severity;
+    kind: FindingKind;
+    title: string;
+    detail: string;
+    file?: string;
+}
+
+/**
+ * A finding is merge-blocking iff it is security- or claim-affecting AND at
+ * least `high` severity. Style and correctness findings advise; low/medium
+ * security findings advise. Council 2026-07-08 (claude-sonnet-4-5 + gpt-4o):
+ * a 100 %-blocking gate at solo-maintainer token cost gets ignored or gamed —
+ * block ONLY on the narrow security/claim × {critical,high} intersection.
+ */
+export function classifyBlocking(f: Finding): boolean {
+    return (f.kind === 'security' || f.kind === 'claim') && (f.severity === 'critical' || f.severity === 'high');
+}
+
+/**
+ * Deterministic gate verdict. Mirrors `check_quality_regression.gateVerdict`:
+ * exit 2 blocks the merge, exit 0 passes.
+ *   - advisory (default shipped mode): always 0; the caller reports the
+ *     would-block set without failing the check.
+ *   - enforce (maintainer flip): 2 iff any finding is merge-blocking.
+ */
+export function gateVerdict(findings: readonly Finding[], opts: { enforce: boolean }): 0 | 2 {
+    if (!opts.enforce) return 0;
+    return findings.some(classifyBlocking) ? 2 : 0;
+}
+
+// ── Diff collection ───────────────────────────────────────────────────
+/** Generated projections + lockfiles a self-review should not re-read. */
+const SKIP_PREFIXES = ['dist/agent-src/', '.augment/', '.claude/', '.cursor/', '.clinerules/'];
+const SKIP_EXACT = new Set(['.windsurfrules', 'GEMINI.md', 'package-lock.json']);
+
+export function isReviewablePath(p: string): boolean {
+    if (SKIP_EXACT.has(p)) return false;
+    return !SKIP_PREFIXES.some((pre) => p.startsWith(pre));
+}
+
+function changedFiles(baseRef: string): string[] {
+    const r = spawnSync('git', ['diff', '--name-only', `${baseRef}...HEAD`], {
+        cwd: REPO_ROOT,
+        encoding: 'utf8',
+    });
+    if (r.status !== 0) {
+        throw new Error(`git diff failed: ${(r.stderr ?? '').toString().slice(0, 300)}`);
+    }
+    return (r.stdout ?? '')
+        .split('\n')
+        .map((s) => s.trim())
+        .filter(Boolean)
+        .filter(isReviewablePath);
+}
+
+function diffText(baseRef: string, files: string[]): string {
+    if (files.length === 0) return '';
+    const r = spawnSync('git', ['diff', `${baseRef}...HEAD`, '--', ...files], {
+        cwd: REPO_ROOT,
+        encoding: 'utf8',
+        maxBuffer: 32 * 1024 * 1024,
+    });
+    return (r.stdout ?? '').toString();
+}
+
+// ── Review skill bodies (the system prompt) ───────────────────────────
+const REVIEW_SKILLS = ['adversarial-review', 'agent-security-review'] as const;
+
+function loadSkillBody(name: string): string {
+    const p = path.join(REPO_ROOT, 'src', 'skills', name, 'SKILL.md');
+    return readFileSync(p, 'utf8');
+}
+
+function buildSystemPrompt(): string {
+    const bodies = REVIEW_SKILLS.map((s) => `# Skill: ${s}\n\n${loadSkillBody(s)}`).join('\n\n---\n\n');
+    return [
+        'You are the self-review gate for an AI-agent governance package. Apply the',
+        'two skills below to the supplied PR diff. Return ONLY a JSON object of the',
+        'shape {"findings":[{"severity":"critical|high|medium|low",',
+        '"kind":"security|claim|correctness|style","title":"...","detail":"...",',
+        '"file":"optional/path"}]}. Use kind="claim" for an unbacked headline number',
+        'or a proof/CLAIMS-affecting statement, kind="security" for an auth/tenant/',
+        'secret/egress/injection gap. No prose outside the JSON.',
+        '',
+        bodies,
+    ].join('\n');
+}
+
+// ── Plan (dry-run) ────────────────────────────────────────────────────
+export interface ReviewPlan {
+    skills: string[];
+    files: string[];
+    promptChars: number;
+    note: string;
+}
+
+export function buildPlan(baseRef: string): ReviewPlan {
+    const files = changedFiles(baseRef);
+    const diff = diffText(baseRef, files);
+    return {
+        skills: [...REVIEW_SKILLS],
+        files,
+        promptChars: buildSystemPrompt().length + diff.length,
+        note:
+            files.length === 0
+                ? 'No reviewable (non-generated) files changed — the live review would no-op.'
+                : `${files.length} reviewable file(s); live review would send ~${Math.ceil(
+                      (buildSystemPrompt().length + diff.length) / 4,
+                  )} input tokens.`,
+    };
+}
+
+// ── Live review ───────────────────────────────────────────────────────
+function parseFindings(text: string): Finding[] {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start < 0 || end <= start) return [];
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(text.slice(start, end + 1));
+    } catch {
+        return [];
+    }
+    const arr = (parsed as { findings?: unknown })?.findings;
+    if (!Array.isArray(arr)) return [];
+    const sev = new Set<Severity>(['critical', 'high', 'medium', 'low']);
+    const kind = new Set<FindingKind>(['security', 'claim', 'correctness', 'style']);
+    return arr
+        .filter((f): f is Record<string, unknown> => typeof f === 'object' && f !== null)
+        .filter((f) => sev.has(f.severity as Severity) && kind.has(f.kind as FindingKind))
+        .map((f) => ({
+            severity: f.severity as Severity,
+            kind: f.kind as FindingKind,
+            title: String(f.title ?? '').slice(0, 300),
+            detail: String(f.detail ?? '').slice(0, 2000),
+            ...(typeof f.file === 'string' ? { file: f.file } : {}),
+        }));
+}
+
+export function renderReview(findings: Finding[], enforce: boolean): string {
+    const banner =
+        '> HUMAN REVIEW REQUIRED — dogfooded AI adversarial-review + security gate. ' +
+        'Findings are decision support, not a guarantee; detection is probabilistic. ' +
+        'This is a floor, **not** independent human review.';
+    if (findings.length === 0) {
+        return `${banner}\n\n✅ Self-review gate: no findings.`;
+    }
+    const blocking = findings.filter(classifyBlocking);
+    const rows = findings
+        .map((f) => `| ${f.severity} | ${f.kind} | ${f.file ?? '—'} | ${f.title} |`)
+        .join('\n');
+    const verdictLine = blocking.length
+        ? enforce
+            ? `❌ ${blocking.length} merge-blocking finding(s) (security/claim × high+).`
+            : `⚠️ ${blocking.length} finding(s) WOULD block merge under an enforced gate (advisory now).`
+        : '✅ No merge-blocking findings (style/correctness advise only).';
+    return `${banner}\n\n${verdictLine}\n\n| severity | kind | file | finding |\n|---|---|---|---|\n${rows}`;
+}
+
+function postReview(body: string): void {
+    const pr = process.env.PR_NUMBER;
+    if (!pr) {
+        process.stdout.write('::notice::self-review-gate — no PR_NUMBER; printing review instead of posting.\n');
+        process.stdout.write(body + '\n');
+        return;
+    }
+    const r = spawnSync('gh', ['pr', 'comment', pr, '--body', body], { cwd: REPO_ROOT, encoding: 'utf8' });
+    if (r.status !== 0) {
+        process.stdout.write(`::warning::self-review-gate — gh pr comment failed: ${(r.stderr ?? '').toString().slice(0, 300)}\n`);
+        process.stdout.write(body + '\n');
+    }
+}
+
+function resolveKey(): string | null {
+    const env = process.env.ANTHROPIC_API_KEY;
+    if (env && env.trim()) return env.trim();
+    try {
+        return load_anthropic_key();
+    } catch {
+        return null;
+    }
+}
+
+// ── CLI ───────────────────────────────────────────────────────────────
+export function main(argv: string[]): 0 | 2 {
+    const dryRun = argv.includes('--dry-run');
+    const enforce = argv.includes('--enforce');
+    const baseIdx = argv.indexOf('--base');
+    const baseRef = (baseIdx >= 0 ? argv[baseIdx + 1] : undefined) ?? 'origin/main';
+
+    const plan = buildPlan(baseRef);
+
+    if (dryRun) {
+        process.stdout.write(
+            `self-review-gate (dry-run — no spend, advisory):\n` +
+                `  skills: ${plan.skills.join(', ')}\n` +
+                `  files:  ${plan.files.length}\n` +
+                `  ${plan.note}\n`,
+        );
+        return 0;
+    }
+
+    if (plan.files.length === 0) {
+        process.stdout.write('::notice::self-review-gate — no reviewable files changed; skipping.\n');
+        return 0;
+    }
+
+    const key = resolveKey();
+    if (!key) {
+        process.stdout.write(
+            '::notice::self-review-gate skipped — set the ANTHROPIC_API_KEY repo secret to enable the live dogfooded review.\n',
+        );
+        return 0;
+    }
+
+    const client = new AnthropicClient({ api_key: key });
+    const resp = client.ask(buildSystemPrompt(), diffText(baseRef, plan.files), 4096);
+    if (resp.error) {
+        process.stdout.write(`::warning::self-review-gate — model call failed: ${resp.error}\n`);
+        return 0; // advisory: a transport failure never blocks the merge
+    }
+    const findings = parseFindings(resp.text);
+    const body = renderReview(findings, enforce);
+    postReview(body);
+
+    const code = gateVerdict(findings, { enforce });
+    if (code === 2) {
+        process.stdout.write('::error::self-review-gate — merge-blocking findings under enforce mode.\n');
+    }
+    return code;
+}
+
+if (existsSync(process.argv[1] ?? '') && import.meta.url === `file://${process.argv[1]}`) {
+    process.exit(main(process.argv.slice(2)));
+}