@event4u/agent-config 12.0.0 → 13.0.0

package/src/scripts/ai_council/config.ts

@@ -81,6 +81,7 @@ import * as user_global_paths from '../_lib/user_global_paths.js';
 // `import type` and is erased at build, so there is no runtime cycle today.
 // Converting it to a value import — or adding any other runtime import from
 // quorum into config — creates a real ESM cycle in the loader's init path.
+import { OPENAI_CLI_VENDOR_DEFAULT } from './clients.js';
 import { SOLO_FLOOR_MIN_PRESENT } from './quorum.js';
 
 const _VALID_PROVIDERS: ReadonlySet<string> = new Set([
@@ -1843,7 +1844,21 @@ function _build_member(
             }
             entries.push(entry);
         }
-        if (member_enabled && model && !entries.includes(model)) {
+        // `codex-default` is a SENTINEL, not a model id — "let the transport
+        // choose". Requiring it to appear on a downgrade ladder of concrete ids
+        // is a category error, and it is the check that blocks the one config
+        // that actually works on a subscription account: the shipped template
+        // cannot pin an id (every id this package shipped is on
+        // `CODEX_MEASURED_UNSERVABLE`), and its ladder is an api-transport
+        // concern the CLI's own default cannot be ordered against. Exempting
+        // the sentinel is narrower than dropping the check — a real pin is
+        // still required to be on its own ladder.
+        if (
+            member_enabled &&
+            model &&
+            model !== OPENAI_CLI_VENDOR_DEFAULT &&
+            !entries.includes(model)
+        ) {
             throw new CouncilConfigError(
                 `members.${name}.model_ladder must include the active ` +
                     `\`model\` (${_pyReprStr(model)}); got ${_pyRepr(entries)}.`,

package/src/scripts/ai_council/low_impact.ts

@@ -35,6 +35,7 @@
 // - dict insertion order + `dict(sorted(...))` for the stats aggregation.
 
 import { pyRound } from '../_lib/value_ladder.js';
+import { OPENAI_CLI_VENDOR_DEFAULT } from './clients.js';
 import type { CouncilResponse, ExternalAIClient } from './clients.js';
 import type { LowImpactFastPathConfig, MemberConfig } from './config.js';
 import type { ImpactVerdict } from './necessity.js';
@@ -630,7 +631,19 @@ export function resolve_low_impact(
             const pinned = model_downgrade.model_tier_override?.[member.name];
             const cheapest = member.model_ladder.length > 0 ? member.model_ladder[0] : null;
             const target = pinned && pinned.length > 0 ? pinned : cheapest;
-            if (target && target !== client.model) {
+            // A member left on the vendor-default sentinel is NOT downgradable
+            // off the ladder. There is nothing to downgrade FROM — the
+            // transport picks the model — and writing a rung onto it re-creates
+            // the pinned-id shape the seat repair removed. Concretely: the
+            // shipped ladder's cheapest rung is `gpt-4o-mini` and its
+            // neighbours are `gpt-4o` / `gpt-4.1`, so this line could hand the
+            // codex transport an id `CODEX_MEASURED_UNSERVABLE` already records
+            // as refused — on a path whose `model_downgrade.enabled` defaults
+            // to true. An explicit per-run `model_tier_override` still wins,
+            // because that is a human naming a model for this run.
+            const unpinned = client.model === OPENAI_CLI_VENDOR_DEFAULT;
+            const overrideWins = pinned !== undefined && pinned.length > 0;
+            if (target && target !== client.model && (!unpinned || overrideWins)) {
                 client.model = target;
             }
         }

package/src/scripts/ai_council/orchestrator.ts

@@ -2029,7 +2029,20 @@ export function render(responses: CouncilResponse[], opts: RenderOptions = {}):
 function _render_quorum_line(q: QuorumResult): string {
     const verdict = q.status === 'concluded' ? 'concluded' : 'INCONCLUSIVE — release gate holds';
     const solo = isSoloConcluded(q) ? ' · **solo** — one voice concluded this pass' : '';
-    return `**Quorum:** ${q.present}/${q.total} present, needed ${q.threshold} — ${verdict}.${solo}`;
+    // The DEGRADED marker shipped on stdout (`council_cli.ts::_format_quorum_line`,
+    // round 7 § 5.3) and did NOT ship here, although this function is the one
+    // the mirror was copied FROM. The two then diverged in the worse direction:
+    // stdout scrolls away, this string is what gets committed as evidence and
+    // cited months later. `1/2 present, needed 1 — concluded` is literally true
+    // and reads as agreement; the solo marker alone covers only present === 1,
+    // so 2-of-4 rendered as plain "concluded" with nothing to stop a reader
+    // inferring convergence. Same sentence as stdout on purpose — one wording,
+    // so neither surface can drift into being the softer one again.
+    const degraded =
+        q.present < q.total
+            ? `  ⚠️  DEGRADED — ${String(q.total - q.present)} member(s) did not answer; this is not convergence.`
+            : '';
+    return `**Quorum:** ${q.present}/${q.total} present, needed ${q.threshold} — ${verdict}.${solo}${degraded}`;
 }
 
 /**

package/src/scripts/ai_council/transport_resolver.ts

@@ -356,6 +356,7 @@ export type CliFailureClass =
     | 'binary_missing'
     | 'auth_rejected'
     | 'cli_unsupported'
+    | 'model_unservable'
     | 'timeout'
     | 'server_error'
     | 'quota_exhausted'
@@ -376,6 +377,20 @@ const FALLBACK_ELIGIBLE: ReadonlySet<CliFailureClass> = new Set([
     'binary_missing',
     'auth_rejected',
     'cli_unsupported',
+    // `model_unservable` satisfies the same no-double-charge property in both
+    // of its shapes: the pre-spend gate refuses before any subprocess exists,
+    // and the call-time shape is a provider 400 — rejected at the request
+    // boundary, no generation performed.
+    //
+    // HONEST SCOPE: this set is not consumed by any shipped path today.
+    // `isFallbackEligible` and `MidFlightFallback` have zero call sites outside
+    // this module and its tests, so membership here is a stated property of the
+    // class, NOT a behaviour a caller performs. Said plainly because the
+    // opposite reading is the attractive one — the constraint really is the
+    // subscription transport's rather than the model's, so the api rung really
+    // would serve the same id, and it would be easy to write that as if it
+    // happened. It does not happen yet.
+    'model_unservable',
 ]);
 
 /** Normalise a raw client error / skip reason into a `CliFailureClass`. */
@@ -388,6 +403,15 @@ export function classifyCliFailure(raw: string): CliFailureClass {
     if (s === 'cli_unsupported' || s === 'unsupported' || s === 'parse_failed') {
         return 'cli_unsupported';
     }
+    // Prefix match, not equality: the detail carries the operator-facing reason
+    // after the code, and that sentence IS the value of this class — collapsing
+    // it back to a bare token would restore the opaque `exit_1` this replaced.
+    // Also matched on the provider's own wording, so a model refused at call
+    // time (one the deny-list has never seen) classifies the same as one the
+    // pre-spend gate caught.
+    if (s.startsWith('model_unservable') || s.includes('is not supported when using codex')) {
+        return 'model_unservable';
+    }
     if (s === 'timeout') return 'timeout';
     if (s === 'cli_quota_exhausted' || s === 'quota_exhausted') return 'quota_exhausted';
     if (/^exit_5\d\d$/.test(s) || s === 'server_error' || s === 'os') return 'server_error';
@@ -406,8 +430,14 @@ export function isFallbackEligible(failure: CliFailureClass): boolean {
  * can bucket every absent member — whether it never got a transport at all
  * or lost one partway through — under one vocabulary. Returns `null` for a
  * failure class outside the four-value enum (`cli_unsupported`,
- * `server_error`, `other`); a caller falls back to the raw failure detail
- * for those rather than mis-classifying them.
+ * `server_error`, `model_unservable`, `other`); a caller falls back to the raw
+ * failure detail for those rather than mis-classifying them.
+ *
+ * `model_unservable` is deliberately unmapped rather than folded into
+ * `no_auth`: the credential is fine, the model is not, and the `detail` string
+ * carries the specific reason. So the bucket a reader sees is still the generic
+ * `unavailable`, while the line beside it now names the cause — an incomplete
+ * improvement, stated as one.
  */
 export function absentReasonFromCliFailure(failure: CliFailureClass): AbsentReason | null {
     switch (failure) {

package/src/scripts/bench_ab_v2_complexity.ts

@@ -0,0 +1,252 @@
+#!/usr/bin/env -S npx tsx
+/**
+ * Offline re-scorer for the Phase-3 metric pair — S0.3 delta #11.
+ *
+ * Reads a finished `ab-v2` report, walks every trial's **preserved workspace**
+ * (delta #7 keyed `task|arm|seed`), and writes two endpoints into each trial's
+ * `metrics`:
+ *
+ *   - `added_lines` — T1's endpoint. Added only, not added+removed: the
+ *     discipline oracle's `max_lines_changed` wants the sum, the size claim
+ *     wants what the run *wrote*.
+ *   - `median_cognitive_complexity` — T2's endpoint, over the functions in the
+ *     files that trial actually changed.
+ *
+ * WHY OFFLINE RATHER THAN IN THE RUNNER. Three reasons, in order of weight.
+ * The metric is **retro-fittable**: the workspaces from every completed sweep
+ * are on disk, so a run that already cost money can gain the endpoint without
+ * being re-run — which is what makes the anti-golfing gate nearly free, and what
+ * the roadmap's own text promises. It is **deterministic and model-free**, so
+ * re-scoring the same workspace twice cannot produce two numbers. And the
+ * grammar load is **async** while `trajectory_metrics` is synchronous and called
+ * from four places — threading async through the live path would be a larger
+ * diff than the endpoint itself, for no measurement it does not already get.
+ *
+ * The report is rewritten in place only with `--write`; the default prints the
+ * table and touches nothing, because silently rewriting a pinned artefact is how
+ * a report stops matching the numbers already quoted from it.
+ *
+ * Usage:
+ *   bench_ab_v2_complexity.ts <report.json> [--write] [--json]
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { parse as parseYaml } from 'yaml';
+
+import { medianComplexityPerChangedFunction } from './_lib/bench_ab_complexity.js';
+import { changed_files, diff_line_counts } from './_lib/bench_ab_scoring_v2.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+const FIXTURES_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
+const CORPUS_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'corpora', 'ab-trackb-v2.yaml');
+
+type Dict = Record<string, unknown>;
+
+function _dictOr(v: unknown): Dict {
+    return v && typeof v === 'object' && !Array.isArray(v) ? (v as Dict) : {};
+}
+
+/**
+ * task id → fixture path. The corpus is the ONLY resolver, deliberately.
+ *
+ * An earlier draft also had the runner stamp a `fixture` key onto each record so
+ * reports would be self-describing, and this function read it first. That is two
+ * mechanisms for one fact: the corpus lookup already covers every report,
+ * including every one written before the key would have existed. The extra field
+ * was dropped rather than kept as a fallback — a reader with no producer is the
+ * shape this branch's own completion review flagged elsewhere.
+ *
+ * A missing or unreadable corpus is not an error: it degrades every affected
+ * trial to `skipped_reason: 'task id not in the corpus'`, which is a reportable
+ * state, rather than aborting a re-score that may still cover most of the report.
+ */
+function loadCorpusFixtures(corpusPath: string | null): Map<string, string> {
+    const byId = new Map<string, string>();
+    if (!corpusPath || !fs.existsSync(corpusPath)) return byId;
+    try {
+        const corpus = _dictOr(parseYaml(fs.readFileSync(corpusPath, 'utf8'), { version: '1.1' }));
+        const tasks = Array.isArray(corpus['tasks']) ? (corpus['tasks'] as Dict[]) : [];
+        for (const t of tasks) {
+            if (t['id'] !== undefined && t['fixture'] !== undefined) {
+                byId.set(String(t['id']), String(t['fixture']));
+            }
+        }
+    } catch {
+        return byId;
+    }
+    return byId;
+}
+
+export interface TrialRescore {
+    task: string;
+    arm: string;
+    seed: unknown;
+    /** `null` when the workspace or the fixture is no longer on disk. */
+    added_lines: number | null;
+    median_cognitive_complexity: number | null;
+    n_functions: number;
+    unsupported_files: string[];
+    skipped_reason: string | null;
+}
+
+/**
+ * Re-score one report payload. Pure apart from the filesystem reads it needs;
+ * mutates `payload` only when `write` is true. The fixture each trial is diffed
+ * against is resolved from the corpus by task id — see `loadCorpusFixtures`.
+ */
+export async function rescoreReport(
+    payload: Dict,
+    opts: { fixturesRoot?: string; write?: boolean; corpusPath?: string | null } = {},
+): Promise<TrialRescore[]> {
+    const fixturesRoot = opts.fixturesRoot ?? FIXTURES_ROOT;
+    let wrote = 0;
+    const byId = loadCorpusFixtures(opts.corpusPath === undefined ? CORPUS_PATH : opts.corpusPath);
+    const out: TrialRescore[] = [];
+    const records = Array.isArray(payload['records']) ? (payload['records'] as Dict[]) : [];
+
+    for (const rec of records) {
+        const taskId = String(rec['id'] ?? '<unknown>');
+        const fixtureRel = byId.get(taskId) ?? null;
+        const arms = _dictOr(rec['arms']);
+        for (const [arm, runsRaw] of Object.entries(arms)) {
+            const runs = Array.isArray(runsRaw) ? (runsRaw as Dict[]) : [];
+            for (const run of runs) {
+                const row: TrialRescore = {
+                    task: taskId,
+                    arm,
+                    seed: run['seed'] ?? null,
+                    added_lines: null,
+                    median_cognitive_complexity: null,
+                    n_functions: 0,
+                    unsupported_files: [],
+                    skipped_reason: null,
+                };
+
+                const workspace = run['workspace'] !== undefined ? String(run['workspace']) : '';
+                if (!workspace || !fs.existsSync(workspace)) {
+                    // Not a zero — a trial whose workspace was pruned is
+                    // unmeasurable, and `compare()` reads a missing metric as
+                    // "not measured on this pair" rather than as a value.
+                    row.skipped_reason = workspace ? 'workspace missing on disk' : 'no workspace recorded';
+                    out.push(row);
+                    continue;
+                }
+                if (fixtureRel === null) {
+                    row.skipped_reason = 'task id not in the corpus';
+                    out.push(row);
+                    continue;
+                }
+                const fixture = path.join(fixturesRoot, fixtureRel);
+                if (!fs.existsSync(fixture)) {
+                    row.skipped_reason = 'fixture missing on disk';
+                    out.push(row);
+                    continue;
+                }
+
+                const changed = changed_files(fixture, workspace);
+                row.added_lines = diff_line_counts(fixture, workspace, changed).added;
+                const rollup = await medianComplexityPerChangedFunction(workspace, changed);
+                row.median_cognitive_complexity = rollup.median;
+                row.n_functions = rollup.n_functions;
+                row.unsupported_files = rollup.unsupported_files;
+
+                if (opts.write) {
+                    const metrics = _dictOr(run['metrics']);
+                    metrics['added_lines'] = row.added_lines;
+                    // `median` is null when the diff contained no scorable
+                    // function, and null is written on purpose: `compare()` reads
+                    // a null as "not measured on this pair", while a 0 would claim
+                    // the run produced maximally simple code. The count sits
+                    // beside it so a reader can tell "nothing to score" from
+                    // "scored, and it was trivial".
+                    metrics['median_cognitive_complexity'] = row.median_cognitive_complexity;
+                    metrics['complexity_n_functions'] = row.n_functions;
+                    run['metrics'] = metrics;
+                    wrote += 1;
+                }
+                out.push(row);
+            }
+        }
+    }
+    if (opts.write) lastWriteCount = wrote;
+    return out;
+}
+
+/**
+ * How many trials the last `rescoreReport({ write: true })` actually touched.
+ *
+ * The CLI uses it to skip rewriting the report when nothing was measured:
+ * re-serialising a pinned artefact for a zero-row pass changes its bytes (and
+ * its formatting) without changing a single number, which is exactly the kind of
+ * silent churn that makes a report stop matching the figures quoted from it.
+ */
+let lastWriteCount = 0;
+export function trialsWrittenByLastRescore(): number {
+    return lastWriteCount;
+}
+
+function renderTable(rows: TrialRescore[]): string {
+    const L: string[] = [];
+    L.push('task | arm | seed | added | median cc | fns | note');
+    L.push('-----|-----|------|-------|-----------|-----|-----');
+    for (const r of rows) {
+        L.push(
+            [
+                r.task,
+                r.arm,
+                String(r.seed ?? ''),
+                r.added_lines === null ? '-' : String(r.added_lines),
+                r.median_cognitive_complexity === null ? '-' : String(r.median_cognitive_complexity),
+                String(r.n_functions),
+                r.skipped_reason ??
+                    (r.unsupported_files.length > 0
+                        ? `unsupported: ${r.unsupported_files.join(',')}`
+                        : ''),
+            ].join(' | '),
+        );
+    }
+    const measured = rows.filter((r) => r.median_cognitive_complexity !== null).length;
+    L.push('');
+    L.push(`${measured}/${rows.length} trials carry a complexity observation.`);
+    return L.join('\n');
+}
+
+async function main(argv: string[]): Promise<number> {
+    const args = argv.filter((a) => !a.startsWith('--'));
+    const write = argv.includes('--write');
+    const asJson = argv.includes('--json');
+    const reportPath = args[0];
+    if (!reportPath) {
+        process.stderr.write(
+            'usage: bench_ab_v2_complexity.ts <report.json> [--write] [--json]\n',
+        );
+        return 1;
+    }
+    if (!fs.existsSync(reportPath)) {
+        process.stderr.write(`report not found: ${reportPath}\n`);
+        return 1;
+    }
+    const payload = JSON.parse(fs.readFileSync(reportPath, 'utf8')) as Dict;
+    const rows = await rescoreReport(payload, { write });
+    const written = trialsWrittenByLastRescore();
+    if (write && written > 0) {
+        fs.writeFileSync(reportPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    } else if (write) {
+        process.stderr.write('no trial could be measured — report left untouched\n');
+    }
+    process.stdout.write(asJson ? `${JSON.stringify(rows, null, 2)}\n` : `${renderTable(rows)}\n`);
+    return 0;
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(_HERE)) {
+    main(process.argv.slice(2)).then(
+        (code) => process.exit(code),
+        (err: unknown) => {
+            process.stderr.write(`${String(err)}\n`);
+            process.exit(1);
+        },
+    );
+}

package/src/scripts/bench_ab_v2_run.ts

@@ -45,7 +45,6 @@
 import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import * as fs from 'node:fs';
-import * as os from 'node:os';
 import * as path from 'node:path';
 import process from 'node:process';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -53,6 +52,7 @@ import { parse as parseYaml } from 'yaml';
 
 import * as v1 from './bench_ab_task_runner.js';
 import * as scoring from './_lib/bench_ab_scoring_v2.js';
+import { preserve_transcript, reset_fixture } from './_lib/bench_ab_workspace.js';
 import {
     SweepBudget,
     activation_verdict,
@@ -75,7 +75,6 @@ const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
 const ROUTER_PATH = path.join(REPO_ROOT, 'dist', 'router.json');
 const RULES_DIR = path.join(REPO_ROOT, 'dist', 'agent-src', 'rules');
 const CORPUS_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'corpora', 'ab-trackb-v2.yaml');
-const FIXTURES_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
 const REPORTS_DIR = path.join(REPO_ROOT, 'internal', 'bench', 'reports', 'ab-v2');
 const CHECKPOINTS_DIR = path.join(REPORTS_DIR, 'checkpoints');
 const PRICING_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'pricing.yaml');
@@ -87,7 +86,6 @@ const PRICING_PATH = path.join(REPO_ROOT, 'internal', 'bench', 'pricing.yaml');
 // vanilla ≈ package and invalidated every prior null. A /tmp clone has no
 // agent-config ancestor → vanilla is truly plain; `package` still activates via
 // the USER-scope global plugin regardless of cwd.
-const WORK_ROOT = path.join(os.tmpdir(), 'agent-config-bench-v2-clones');
 
 type Dict = Record<string, unknown>;
 
@@ -329,45 +327,6 @@ export function injected_text(inject: string | null, placebo_chars: number): str
     return null;
 }
 
-/**
- * Workspace directory for one trial — delta #7 of the S0.3 spike.
- *
- * Keyed by `task|arm|seed`, not by task alone. The old task-only key meant every
- * arm and every seed of a task reused ONE directory that the next trial deleted,
- * so at the end of a sweep exactly one workspace survived per task — the last one
- * written, with no record of which arm or seed it belonged to.
- *
- * That is not a tidiness problem. Phase 3's anti-golfing gate is specified as
- * retro-fittable onto already-completed runs by offline re-scoring, and the
- * roadmap calls that gate cheap *because* the workspaces are preserved. Under the
- * old key there was nothing to re-score, so the claim was false. A distinct
- * directory per trial is what makes it true.
- *
- * `arm` is sanitised because arm names are used verbatim as a path segment and
- * one of them would otherwise be free to escape the root.
- */
-export function workspace_dir(task_id: string, arm: string, seed: number): string {
-    const safe_arm = arm.replace(/[^A-Za-z0-9._-]/g, '_');
-    return path.join(WORK_ROOT, `${task_id}__${safe_arm}__seed${seed}`);
-}
-
-/**
- * Copy the task's pristine fixture into this trial's own working clone.
- *
- * The clone is re-created from the fixture on every call, so a resumed or
- * repeated trial still starts pristine — the per-trial key changes *which*
- * directory that is, never whether it is clean.
- */
-export function reset_fixture(task: Dict, arm: string, seed: number): [string, string] {
-    const fixture = path.join(FIXTURES_ROOT, String(task['fixture']));
-    const dest = workspace_dir(String(task['id']), arm, seed);
-    if (fs.existsSync(dest)) {
-        fs.rmSync(dest, { recursive: true, force: true });
-    }
-    fs.mkdirSync(path.dirname(dest), { recursive: true });
-    fs.cpSync(fixture, dest, { recursive: true });
-    return [dest, fixture];
-}
 
 /** Map a run outcome to an AgentBench-style trajectory bucket. */
 export function status_bucket(run: Dict): string {
@@ -689,6 +648,7 @@ export function run_one(task: Dict, arm: string, opts: RunOneOpts): Dict {
         // rather than recomputed later, so an offline re-scorer reads the path the
         // sweep actually used instead of re-deriving a key that may have changed.
         workspace: clone,
+        transcript_path: preserve_transcript(clone, String(run['transcript'] ?? '')),
         ...integrity_fields(run, spec, sp_text ? sp_text.length : 0, opts.model),
     };
 }
@@ -803,6 +763,7 @@ export function selftest_run(
         metrics: trajectory_metrics(run, score),
         injected_chars: sp_text ? sp_text.length : 0,
         workspace: clone,
+        // No transcript_path — a synthetic transcript would give T5 a score of the harness.
         ...integrity_fields(run, spec, sp_text ? sp_text.length : 0, opts.model),
     };
 }