@event4u/agent-config 14.9.0 → 14.11.0

package/src/scripts/council_attendance_metrics.ts

@@ -0,0 +1,183 @@
+#!/usr/bin/env tsx
+/**
+ * Replay the four registered council attendance metrics over an events log.
+ *
+ * Step 3.2 of `road-to-council-evidence-integrity`. The metrics were registered
+ * in `src/config/quorum-attendance-budget.json` on 2026-08-10 and nothing in the
+ * tree ever computed them, so "the four registered metrics reproduce unchanged
+ * across the schema bump" was a claim with no instrument. This is the instrument.
+ *
+ * ## The exclusion rule, which is the whole point of the schema version
+ *
+ * `stance_agreement` arrived at schema v5. A line written at v4 or earlier does
+ * not carry the field and NO honest value can be inferred for it — a v4 pass may
+ * have had a stance tally whose verdict was never recorded, or none at all, and
+ * the line cannot tell you which. So the agreement rate's denominator is
+ * v5-and-later lines ONLY, and this reports the excluded count next to the rate
+ * rather than folding it in. Defaulting the missing field to `not_tallied` would
+ * manufacture a measurement out of an absence, which is the defect class this
+ * whole roadmap exists to close.
+ *
+ * The four pre-existing metrics are computed over their own documented
+ * populations, unchanged by the bump: they read fields that exist at every
+ * version they were registered against.
+ *
+ * ## Scope, stated rather than discovered later
+ *
+ * `agents/runtime/council/events.log` is gitignored and machine-local, so every
+ * reading is one machine's traffic and no rate from it generalises. A rate
+ * printed here describes the log it was given and nothing else.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+/** The version at which `stance_agreement` began to be written. */
+export const STANCE_AGREEMENT_SINCE = 5;
+
+type Line = Record<string, unknown>;
+
+export interface AttendanceMetrics {
+    /** post_run, command=run lines — the population every rate below is scoped to. */
+    readonly passes: number;
+    /** present / total, summed over passes. `null` when there are none. */
+    readonly attendance_rate: number | null;
+    /** share of passes with total < configured_total. */
+    readonly roster_shortfall_rate: number | null;
+    /** share of passes with `solo: true`. */
+    readonly solo_conclusion_rate: number | null;
+    /** share of passes with `floor_would_hold: true`. */
+    readonly shadow_floor_fire_rate: number | null;
+    /** count per `absent[].reason`. */
+    readonly absent_reason_distribution: Record<string, number>;
+    /**
+     * Share of AGREEMENT-ELIGIBLE passes that reached consensus. The denominator
+     * is `agreement_eligible`, never `passes` — see the module header.
+     */
+    readonly consensus_rate: number | null;
+    /** Passes at schema >= STANCE_AGREEMENT_SINCE. The agreement denominator. */
+    readonly agreement_eligible: number;
+    /** Passes excluded from the agreement rate because their schema predates the field. */
+    readonly agreement_excluded: number;
+}
+
+export function parseLog(text: string): Line[] {
+    const out: Line[] = [];
+    for (const raw of text.split('\n')) {
+        const line = raw.trim();
+        if (line === '') continue;
+        try {
+            const rec = JSON.parse(line) as unknown;
+            if (rec !== null && typeof rec === 'object' && !Array.isArray(rec)) {
+                out.push(rec as Line);
+            }
+        } catch {
+            // A malformed line is skipped, never guessed at. It is counted
+            // nowhere, which is the honest treatment: it is not a pass that
+            // happened one way, it is a line nobody can read.
+        }
+    }
+    return out;
+}
+
+export function computeMetrics(lines: readonly Line[]): AttendanceMetrics {
+    // The documented population: post_run proves providers were actually
+    // called; `command: run` excludes the spend-free estimate preview.
+    const passes = lines.filter(
+        (l) => l['action'] === 'quorum_result' && l['phase'] === 'post_run' && l['command'] === 'run',
+    );
+    const n = passes.length;
+    const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
+    const share = (pred: (l: Line) => boolean): number | null =>
+        n === 0 ? null : passes.filter(pred).length / n;
+
+    const attendance_rate =
+        n === 0
+            ? null
+            : passes.reduce((acc, l) => {
+                  const total = num(l['total']);
+                  return acc + (total === 0 ? 0 : num(l['present']) / total);
+              }, 0) / n;
+
+    const absent_reason_distribution: Record<string, number> = {};
+    for (const l of passes) {
+        const absent = l['absent'];
+        if (!Array.isArray(absent)) continue;
+        for (const a of absent) {
+            if (a === null || typeof a !== 'object') continue;
+            const reason = String((a as Line)['reason'] ?? 'unavailable');
+            absent_reason_distribution[reason] = (absent_reason_distribution[reason] ?? 0) + 1;
+        }
+    }
+
+    // The exclusion, applied on the SCHEMA VERSION rather than on the presence
+    // of the key. A v5 line whose field is somehow absent is a defect worth
+    // seeing; keying on presence would silently reclassify it as old data.
+    const eligible = passes.filter((l) => num(l['schema_version']) >= STANCE_AGREEMENT_SINCE);
+    const consensus_rate =
+        eligible.length === 0 ? null : eligible.filter((l) => l['stance_agreement'] === 'consensus').length / eligible.length;
+
+    return {
+        passes: n,
+        attendance_rate,
+        roster_shortfall_rate: share((l) => num(l['total']) < num(l['configured_total'])),
+        solo_conclusion_rate: share((l) => l['solo'] === true),
+        shadow_floor_fire_rate: share((l) => l['floor_would_hold'] === true),
+        absent_reason_distribution,
+        consensus_rate,
+        agreement_eligible: eligible.length,
+        agreement_excluded: n - eligible.length,
+    };
+}
+
+function repoRoot(): string {
+    return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+}
+
+export function main(argv: string[]): number {
+    const asJson = argv.includes('--json');
+    const pathArg = argv.find((a) => !a.startsWith('--'));
+    const logPath = pathArg ?? path.join(repoRoot(), 'agents/runtime/council/events.log');
+    if (!fs.existsSync(logPath)) {
+        process.stderr.write(
+            `council_attendance_metrics: no log at ${logPath}\n` +
+                '  This is an honest null, not a failure: the log is gitignored and\n' +
+                '  machine-local, so a fresh checkout legitimately has none.\n',
+        );
+        return 0;
+    }
+    const m = computeMetrics(parseLog(fs.readFileSync(logPath, 'utf8')));
+    if (asJson) {
+        process.stdout.write(`${JSON.stringify(m, null, 2)}\n`);
+        return 0;
+    }
+    const pct = (v: number | null): string => (v === null ? 'n/a (no passes)' : `${(v * 100).toFixed(1)}%`);
+    process.stdout.write(`council attendance metrics · ${logPath}\n`);
+    process.stdout.write(`  (machine-local log — this describes THIS log, and generalises to nothing)\n\n`);
+    process.stdout.write(`  post_run/command=run passes   ${String(m.passes)}\n`);
+    process.stdout.write(`  attendance_rate               ${pct(m.attendance_rate)}\n`);
+    process.stdout.write(`  roster_shortfall_rate         ${pct(m.roster_shortfall_rate)}\n`);
+    process.stdout.write(`  solo_conclusion_rate          ${pct(m.solo_conclusion_rate)}\n`);
+    process.stdout.write(`  shadow_floor_fire_rate        ${pct(m.shadow_floor_fire_rate)}\n`);
+    process.stdout.write(
+        `  consensus_rate                ${pct(m.consensus_rate)}  (denominator ${String(m.agreement_eligible)} v${String(STANCE_AGREEMENT_SINCE)}+ lines; ${String(m.agreement_excluded)} older lines EXCLUDED, not defaulted)\n`,
+    );
+    const dist = Object.entries(m.absent_reason_distribution).sort(([a], [b]) => a.localeCompare(b));
+    process.stdout.write(`  absent_reason_distribution    ${dist.length === 0 ? '(none)' : dist.map(([k, v]) => `${k}=${String(v)}`).join(' · ')}\n`);
+    return 0;
+}
+
+if (process.argv[1] !== undefined) {
+    try {
+        const here = fs.realpathSync(fileURLToPath(import.meta.url));
+        const argv1 = fs.realpathSync(path.resolve(process.argv[1]));
+        if (here === argv1) {
+            process.exitCode = main(process.argv.slice(2));
+        }
+    } catch {
+        const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+        if (import.meta.url === argvUrl) {
+            process.exitCode = main(process.argv.slice(2));
+        }
+    }
+}

package/src/scripts/council_cli.ts

@@ -82,8 +82,10 @@ import { classifyCliFailure, type AbsentReason } from './ai_council/transport_re
 import { evaluateQuorum, type QuorumResult } from './ai_council/quorum.js';
 import {
     _emitQuorumEvent,
+    annotateRenderedQuorum,
     _format_quorum_line,
     _postRunQuorum,
+    stanceAgreementOf,
     _quorum_min_present_from,
     _quorum_setting_from,
 } from './ai_council/quorum_wiring.js';
@@ -165,7 +167,7 @@ import {
     render_deanonymization_block,
     with_chairman_fields,
 } from './ai_council/blind_review.js';
-import { tally_stances } from './ai_council/stance_tally.js';
+import { tallyFromResponses } from './ai_council/stance_tally.js';
 import { buildHandoffFromStanceTally, type HandoffEnvelope } from './ai_council/handoff.js';
 
 // ── argparse-style exit plumbing ────────────────────────────────────
@@ -2657,6 +2659,9 @@ function cmd_run(
     // with 3 configured members where 2 fail to construct and 1 answers emits
     // `{total: 1, present: 1}` and reads as full attendance.
     const configured_total = quorum_out.result?.total ?? null;
+    // Hoisted above the post-run emit so agreement rides the same line as attendance.
+    const stance_tally_result = tallyFromResponses(responses, stance_tally_on);
+    const stance_agreement = stanceAgreementOf(stance_tally_result);
     const post_run = _postRunQuorum(members, responses, ai_cfg);
     quorum_out.result = post_run.quorum;
     skipped.push(...post_run.absent);
@@ -2667,6 +2672,7 @@ function cmd_run(
     // skips already went out under `phase: 'pre_run'` from `build_members`,
     // and merging them here would double-count a member absent in both.
     _emitQuorumEvent('post_run', post_run.quorum, post_run.absent, {
+        ...(stance_agreement === undefined ? {} : { stanceAgreement: stance_agreement }),
         command: 'run',
         // Whether the roster actually shrank — see `dispatch_shape`. Without
         // this the line is byte-identical to a configured one-member council,
@@ -2698,13 +2704,7 @@ function cmd_run(
     // the payload in the first place). `buildHandoffFromStanceTally` returns
     // the honest all-null envelope when stance tally never ran or split, so
     // this is always additive, never a fabricated decision.
-    const handoff: HandoffEnvelope = buildHandoffFromStanceTally(
-        stance_tally_on
-            ? tally_stances(
-                  responses.filter((r) => !r.error).map((r) => ({ member: `${r.provider}:${r.model}`, text: r.text })),
-              )
-            : null,
-    );
+    const handoff: HandoffEnvelope = buildHandoffFromStanceTally(stance_tally_result);
     const persona_labels = build_persona_labels(advisor_plans, billable);
     const peer_review = _maybe_run_peer_review(
         ai_cfg,
@@ -2718,6 +2718,8 @@ function cmd_run(
         { persona_labels },
     );
     const consensus = _maybe_run_consensus(ai_cfg, question, members, responses, budget, table, project, args);
+    // Rendered attendance, re-derived after the parser. The event does not move.
+    quorum_out.result = annotateRenderedQuorum(quorum_out.result, consensus?.parse_outcomes);
     const chairman = _maybe_run_chairman(
         ai_cfg,
         question,

package/src/scripts/council_parse_rate.ts

@@ -0,0 +1,116 @@
+#!/usr/bin/env tsx
+/**
+ * Findings-parse outcome rates over the recorded fixture corpus.
+ *
+ * Step 2.4 of `road-to-council-evidence-integrity`. Phase 2 made the three
+ * outcomes distinguishable (`parsed` / `empty` / `parse_failed`); this makes the
+ * distinction *measurable*, over a population a reader can open.
+ *
+ * ## What the denominator is, and what it is not
+ *
+ * The denominator is `tests/fixtures/council-parse-corpus/` — recorded verbatim
+ * member answers, one per file. It is **not live traffic**, and no number this
+ * script prints may be quoted as if it were. The corpus was assembled to cover
+ * the distinct failure SHAPES, so it over-represents them by construction: two
+ * of seven answers are unparseable here because there are two unparseable
+ * shapes, not because two in seven real answers fail.
+ *
+ * Saying that in the same breath as the number is the whole point. A
+ * `parse_empty_rate` with an unstated denominator is exactly the "looks more
+ * settled than the run was" defect this roadmap exists to close, one layer up.
+ *
+ * ## Why it also acts as a gate
+ *
+ * `expected.json` pins the outcome per fixture, so a change to the extractor
+ * that silently reclassifies a recorded answer fails here rather than being
+ * discovered by a paid re-ask in production. Exit 1 on any drift.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import { parse_findings_outcome, type FindingsParseOutcome } from './ai_council/consensus.js';
+
+const CORPUS_REL = 'tests/fixtures/council-parse-corpus';
+
+function repoRoot(): string {
+    return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+}
+
+interface Row {
+    readonly file: string;
+    readonly outcome: FindingsParseOutcome;
+    readonly findings: number;
+}
+
+export function scoreCorpus(dir: string): Row[] {
+    const files = fs
+        .readdirSync(dir)
+        .filter((f) => f.endsWith('.txt'))
+        .sort();
+    return files.map((f) => {
+        const text = fs.readFileSync(path.join(dir, f), 'utf8');
+        const ex = parse_findings_outcome(text, { source: `fixture:${f}` });
+        return { file: f, outcome: ex.outcome, findings: ex.findings.length };
+    });
+}
+
+export function main(argv: string[]): number {
+    const asJson = argv.includes('--json');
+    const dir = path.join(repoRoot(), CORPUS_REL);
+    if (!fs.existsSync(dir)) {
+        process.stderr.write(`council_parse_rate: corpus not found at ${CORPUS_REL}\n`);
+        return 1;
+    }
+    const rows = scoreCorpus(dir);
+    const total = rows.length;
+    const count = (o: FindingsParseOutcome): number => rows.filter((r) => r.outcome === o).length;
+    const failed = count('parse_failed');
+    const empty = count('empty');
+    const parsed = count('parsed');
+
+    const expectedPath = path.join(dir, 'expected.json');
+    let drift: string[] = [];
+    if (fs.existsSync(expectedPath)) {
+        const expected = JSON.parse(fs.readFileSync(expectedPath, 'utf8')) as Record<string, string>;
+        drift = rows
+            .filter((r) => expected[r.file] !== undefined && expected[r.file] !== r.outcome)
+            .map((r) => `${r.file}: expected ${String(expected[r.file])}, got ${r.outcome}`);
+        const missing = rows.filter((r) => expected[r.file] === undefined).map((r) => `${r.file}: not pinned in expected.json`);
+        drift = [...drift, ...missing];
+    }
+
+    if (asJson) {
+        process.stdout.write(
+            `${JSON.stringify({ denominator: CORPUS_REL, total, parsed, empty, parse_failed: failed, rows, drift }, null, 2)}\n`,
+        );
+    } else {
+        process.stdout.write(`council parse outcomes · denominator: ${CORPUS_REL} (${String(total)} recorded answers, NOT live traffic)\n\n`);
+        for (const r of rows) {
+            process.stdout.write(`  ${r.outcome.padEnd(13)} ${r.file} (${String(r.findings)} finding(s))\n`);
+        }
+        const pct = (n: number): string => (total === 0 ? 'n/a' : `${((n / total) * 100).toFixed(1)}%`);
+        process.stdout.write(`\n  parse_failed_rate  ${String(failed)}/${String(total)}  ${pct(failed)}\n`);
+        process.stdout.write(`  parse_empty_rate   ${String(empty)}/${String(total)}  ${pct(empty)}\n`);
+        process.stdout.write(`  parsed_rate        ${String(parsed)}/${String(total)}  ${pct(parsed)}\n`);
+        for (const d of drift) {
+            process.stderr.write(`  DRIFT ${d}\n`);
+        }
+    }
+    return drift.length > 0 ? 1 : 0;
+}
+
+if (process.argv[1] !== undefined) {
+    try {
+        const here = fs.realpathSync(fileURLToPath(import.meta.url));
+        const argv1 = fs.realpathSync(path.resolve(process.argv[1]));
+        if (here === argv1) {
+            process.exitCode = main(process.argv.slice(2));
+        }
+    } catch {
+        const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+        if (import.meta.url === argvUrl) {
+            process.exitCode = main(process.argv.slice(2));
+        }
+    }
+}