@event4u/agent-config 12.1.0 → 14.0.0

package/src/scripts/_lib/bench_ab_workspace.ts

@@ -0,0 +1,111 @@
+/**
+ * Per-trial workspace lifecycle — the clone, and the transcript beside it.
+ *
+ * The two belong together: `reset_fixture` re-creates the clone AND drops the
+ * sibling transcript, and both are the delta-#7 evidence an offline re-scorer
+ * reads. They moved out of `bench_ab_v2_run.ts` because the source-size ratchet
+ * is shrink-only and extraction is its intended answer — raising the baseline
+ * is a defect, and trimming the prose out of load-bearing comments is worse.
+ *
+ * ## Transcript preservation — delta #7's sibling, for T5.
+ *
+ * T5 scores what a run did BEFORE it wrote, which leaves no residue in the
+ * final tree: the transcript is the only evidence, and the report discards it.
+ * These two functions are what make a completed sweep re-scorable for search
+ * adherence instead of needing a re-run.
+ *
+ * BESIDE the clone, never inside it. `changed_files` diffs the clone against
+ * the pristine fixture, so a transcript written into the workspace would appear
+ * as a file the run created: T5's evidence would corrupt T1's, and the size
+ * endpoint would gain one added file per trial for a reason no reader could see.
+ *
+ * A file rather than a field on the trial record because transcripts are the
+ * largest thing a sweep produces, and inlining them would multiply a pinned
+ * report's size for data only the T5 re-scorer ever reads. Truncating instead
+ * was the other option and is worse: a rubric scored on a clipped transcript
+ * measures the clip.
+ *
+ * KNOWN COVERAGE GAP, stated rather than silently carried: the
+ * `package-recursive` arm records neither `workspace` nor `transcript_path`, so
+ * T1/T2/T4 and now T5 all skip it. The workspace half predates this module and
+ * closing it would alter what the existing endpoints cover, which is a
+ * different decision from adding one — so it is named here and in the T5
+ * re-scorer's "no transcript recorded" reason instead of being half-fixed.
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const _HERE = fileURLToPath(import.meta.url);
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..', '..');
+const FIXTURES_ROOT = path.join(REPO_ROOT, 'internal', 'bench', 'ab');
+const WORK_ROOT = path.join(os.tmpdir(), 'agent-config-bench-v2-clones');
+
+type Dict = Record<string, unknown>;
+
+/** Where this trial's transcript is preserved — a sibling of the clone. */
+export function transcript_path_for(clone: string): string {
+    return `${clone}.transcript.txt`;
+}
+
+/**
+ * Write the transcript beside the clone; return the path, or `null` when there
+ * was nothing to preserve or the write failed.
+ *
+ * A null propagates onto the trial as `transcript_path: null`, which the T5
+ * re-scorer reads as "not measured on this trial" — never as a zero score. A
+ * failed write must not be able to look like a run that searched for nothing.
+ */
+export function preserve_transcript(clone: string, transcript: string): string | null {
+    if (!transcript) return null;
+    const dest = transcript_path_for(clone);
+    try {
+        fs.writeFileSync(dest, transcript, { encoding: 'utf-8' });
+    } catch {
+        return null;
+    }
+    return dest;
+}
+
+
+/**
+ * Workspace directory for one trial — delta #7 of the S0.3 spike.
+ *
+ * Keyed by `task|arm|seed`, not by task alone. The old task-only key meant every
+ * arm and every seed of a task reused ONE directory that the next trial deleted,
+ * so at the end of a sweep exactly one workspace survived per task — the last one
+ * written, with no record of which arm or seed it belonged to.
+ *
+ * That is not a tidiness problem. Phase 3's anti-golfing gate is specified as
+ * retro-fittable onto already-completed runs by offline re-scoring, and the
+ * roadmap calls that gate cheap *because* the workspaces are preserved. Under the
+ * old key there was nothing to re-score, so the claim was false. A distinct
+ * directory per trial is what makes it true.
+ *
+ * `arm` is sanitised because arm names are used verbatim as a path segment and
+ * one of them would otherwise be free to escape the root.
+ */
+export function workspace_dir(task_id: string, arm: string, seed: number): string {
+    const safe_arm = arm.replace(/[^A-Za-z0-9._-]/g, '_');
+    return path.join(WORK_ROOT, `${task_id}__${safe_arm}__seed${seed}`);
+}
+
+/**
+ * Copy the task's pristine fixture into this trial's own working clone.
+ *
+ * The clone is re-created from the fixture on every call, so a resumed or
+ * repeated trial still starts pristine — the per-trial key changes *which*
+ * directory that is, never whether it is clean.
+ */
+export function reset_fixture(task: Dict, arm: string, seed: number): [string, string] {
+    const fixture = path.join(FIXTURES_ROOT, String(task['fixture']));
+    const dest = workspace_dir(String(task['id']), arm, seed);
+    if (fs.existsSync(dest)) {
+        fs.rmSync(dest, { recursive: true, force: true });
+    }
+    fs.rmSync(transcript_path_for(dest), { force: true });
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.cpSync(fixture, dest, { recursive: true });
+    return [dest, fixture];
+}

package/src/scripts/_lib/evaluator_contract.ts

@@ -0,0 +1,159 @@
+/**
+ * The evaluator-output contract: types, the registry of wrapped verifiers, and
+ * the validator both the gate and any future loop share.
+ *
+ * Contract: `docs/contracts/evaluator-output.md`.
+ * Schema:   `src/scripts/schemas/evaluator-output.schema.json`.
+ *
+ * The adapters below are PURE — each maps a verifier's already-captured
+ * `{stdout, stderr, exitCode}` onto the contract shape. That is deliberate and
+ * it is what makes the gate cheap: the mapping is the part that can be wrong,
+ * and it can be tested against recorded fixtures without paying to re-run three
+ * verifiers CI already runs once. Spike s02 established that none of the three
+ * needs a change on its own side for this to work.
+ */
+import { load_schema, validate, type YamlValue } from '../validate_frontmatter.js';
+
+export type MetricState = 'present' | 'absent' | 'unreadable';
+export type Direction = 'maximize' | 'minimize';
+
+export interface EvaluatorOutput {
+    readonly schema_version: 1;
+    readonly name: string;
+    readonly pass: boolean;
+    readonly score: number;
+    readonly metric?: number;
+    readonly metric_state?: MetricState;
+    readonly direction?: Direction;
+    readonly error?: string;
+}
+
+/** What a verifier invocation produced, before any interpretation. */
+export interface RawRun {
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly exitCode: number;
+}
+
+export interface EvaluatorAdapter {
+    readonly name: string;
+    /** The verifier this adapter wraps, as its CI-identical script name. */
+    readonly wraps: string;
+    readonly direction: Direction;
+    readonly parse: (raw: RawRun) => EvaluatorOutput;
+}
+
+/**
+ * Read a count off either stream.
+ *
+ * Both streams on purpose: `check_references` writes its count to stderr
+ * (measured, spike s02), and a stdout-only reader returned a null metric beside
+ * `pass: true` — a degraded reading that looked clean. A wrapper reads both; it
+ * still emits its own verdict on stdout alone, per the contract.
+ */
+function readCount(raw: RawRun, re: RegExp): number | null {
+    const m = `${raw.stdout}\n${raw.stderr}`.match(re);
+    if (m === null || m[1] === undefined) return null;
+    const n = Number(m[1]);
+    return Number.isFinite(n) ? n : null;
+}
+
+/**
+ * Fold a possibly-missing count into the three contract fields.
+ *
+ * `expected` says whether this verifier is supposed to have a number at all.
+ * When it is and the number is missing, the result is `unreadable` — never
+ * `absent`, and never a silent zero. That collapse is the exact defect the
+ * `metric_state` field exists to prevent.
+ */
+function withMetric(
+    base: Omit<EvaluatorOutput, 'metric' | 'metric_state' | 'score'>,
+    count: number | null,
+    expected: boolean,
+): EvaluatorOutput {
+    if (count !== null) {
+        return { ...base, metric: count, metric_state: 'present', score: -count };
+    }
+    // `metric` is OMITTED rather than set to null: the subset validator this
+    // estate ships rejects an array-typed `type`, so the schema spells the field
+    // as an optional number. Absence is the null.
+    if (!expected) {
+        return { ...base, metric_state: 'absent', score: base.pass ? 0 : -1 };
+    }
+    return {
+        ...base,
+        metric_state: 'unreadable',
+        score: -1,
+        error: 'expected a count in the verifier output and found none',
+    };
+}
+
+export const EVALUATORS: readonly EvaluatorAdapter[] = [
+    {
+        name: 'validate_frontmatter',
+        wraps: 'validate_frontmatter',
+        direction: 'minimize',
+        parse: (raw) =>
+            withMetric(
+                { schema_version: 1, name: 'validate_frontmatter', pass: raw.exitCode === 0, direction: 'minimize' },
+                // Group 2 is the failing count; group 1 is the corpus size, which
+                // is the denominator and not the thing being minimized.
+                readCount(raw, /\d+\s+artefacts,\s+(\d+)\s+failing/),
+                true,
+            ),
+    },
+    {
+        name: 'lint_output_slop',
+        wraps: 'lint_output_slop',
+        direction: 'minimize',
+        parse: (raw) => {
+            const text = `${raw.stdout}\n${raw.stderr}`;
+            const clean = /clean\s+—\s+no placeholder-prose patterns found/.test(text);
+            const count = clean ? 0 : (text.match(/^\s*\S+:\d+/gm) ?? []).length;
+            return withMetric(
+                { schema_version: 1, name: 'lint_output_slop', pass: raw.exitCode === 0, direction: 'minimize' },
+                count,
+                true,
+            );
+        },
+    },
+    {
+        name: 'check_references',
+        wraps: 'check_references',
+        direction: 'minimize',
+        parse: (raw) => {
+            const text = `${raw.stdout}\n${raw.stderr}`;
+            const scanned = readCount(raw, /scanned:\s*(\d+)/);
+            // A gate that scanned nothing has not measured a clean tree — it has
+            // not measured. Report that as unreadable rather than as zero broken
+            // references, which is the manufactured-green shape this estate names.
+            if (scanned === null || scanned === 0) {
+                return withMetric(
+                    { schema_version: 1, name: 'check_references', pass: raw.exitCode === 0, direction: 'minimize' },
+                    null,
+                    true,
+                );
+            }
+            const clean = /No broken references found/.test(text);
+            return withMetric(
+                { schema_version: 1, name: 'check_references', pass: raw.exitCode === 0, direction: 'minimize' },
+                clean ? 0 : (text.match(/^\s*broken:/gim) ?? []).length || 1,
+                true,
+            );
+        },
+    },
+];
+
+/**
+ * Schema violations for one evaluator output. Empty array = conformant.
+ *
+ * `SchemaError.format()` rather than `String(e)`: the class carries no
+ * `toString`, so stringifying it yields `[object Object]` — a violation list
+ * that names nothing, which is indistinguishable from a broken validator.
+ */
+export function validateEvaluatorOutput(value: unknown): string[] {
+    const schema = load_schema('evaluator-output');
+    return validate(value as YamlValue, schema)
+        .filter((e) => e.severity !== 'warning')
+        .map((e) => e.format());
+}

package/src/scripts/_lib/gh_transient.ts

@@ -0,0 +1,80 @@
+/**
+ * Transient-failure handling for `gh` calls in the release pipeline.
+ *
+ * `gh` does not retry GitHub's 5xx responses — including the pre-flight
+ * "does a PR already exist" GraphQL query it runs before every `pr create`.
+ * The release run therefore dies at whichever step happens to catch the
+ * outage, after the commit and the push already landed.
+ *
+ * Pure over strings and an injected executor so the classifier and the retry
+ * loop are exercised by the fixture tests without a network or a real sleep.
+ */
+
+export interface GhRunResult {
+    returncode: number;
+    stdout: string;
+    stderr: string;
+}
+
+/** Backoff between attempts. Length is the retry budget: three retries, ~23s. */
+export const GH_RETRY_DELAYS_MS = [2_000, 6_000, 15_000] as const;
+
+/**
+ * Retry-worthy failures only. A 4xx is the caller's fault and repeating it
+ * just delays the real message — the one exception is 429, which is the same
+ * wait-and-repeat shape as a 5xx.
+ */
+const _TRANSIENT_RE =
+    /HTTP 5\d\d|HTTP 429|No server is currently available|Something went wrong while executing your query|exceeded a secondary rate limit|\b(?:ETIMEDOUT|ECONNRESET|EAI_AGAIN)\b|TLS handshake timeout/;
+
+/**
+ * Whether `stderr` describes a transient GitHub failure.
+ *
+ * Deliberately reads stderr ALONE. `gh` writes errors there, while stdout on a
+ * failing call can still carry payload the caller asked for (`pr view --json
+ * body`) — and a changelog body that happens to quote `HTTP 500` must not turn
+ * a permanent failure into three pointless retries.
+ */
+export function is_transient_gh_failure(stderr: string): boolean {
+    return _TRANSIENT_RE.test(stderr);
+}
+
+/**
+ * `gh` argv rendered for an error line, long values elided.
+ *
+ * The release-PR body is passed as a single `--body` argument and runs to tens
+ * of thousands of characters. Rendering argv verbatim is what buried the real
+ * one-line `gh` error under a 22k dump on 14.0.0.
+ */
+export function gh_argv_label(args: readonly string[], max = 60): string {
+    return ['gh', ...args].map((a) => (a.length > max ? `${a.slice(0, max - 1)}…` : a)).join(' ');
+}
+
+/**
+ * Run `gh <args>` through `exec`, repeating while the failure is transient and
+ * the budget lasts. Returns the last result — classifying it is the caller's
+ * job, so a `check: false` probe keeps its own fallback behaviour.
+ *
+ * `sleep` and `notify` are injected rather than called directly so the loop is
+ * testable without a real 23-second wait, and so a caller running under a test
+ * exec-override can pass a no-op sleep while the delay sequence itself still
+ * runs and can be asserted.
+ */
+export function gh_retry(
+    args: readonly string[],
+    exec: (argv: readonly string[]) => GhRunResult,
+    opts: { sleep?: (ms: number) => void; notify?: (message: string) => void } = {},
+): GhRunResult {
+    let result = exec(['gh', ...args]);
+    for (let i = 0; result.returncode !== 0 && i < GH_RETRY_DELAYS_MS.length; i++) {
+        if (!is_transient_gh_failure(result.stderr)) {
+            break;
+        }
+        const ms = GH_RETRY_DELAYS_MS[i]!;
+        const first = result.stderr.trim().split('\n')[0] ?? '';
+        opts.notify?.(`↻  ${gh_argv_label(args, 24)} — transient GitHub failure, retry in ${ms / 1_000}s: ${first}`);
+        opts.sleep?.(ms);
+        result = exec(['gh', ...args]);
+    }
+    return result;
+}

package/src/scripts/_lib/release_material.ts

@@ -21,7 +21,18 @@ export interface ChangelogSection {
     body: string;
 }
 
-const _NEXT_HEADING_RE = /^##\s+\[?\d+\.\d+\.\d+/m;
+/**
+ * Where a changelog section ends: the next release entry OR the next era
+ * banner. Era banners are level 1 and release entries level 2+ — the same
+ * invariant the release-validation `changelog-entry` gate relies on, and the
+ * one `lint_changelog_rollback` / `lint_breaking_changes_index` already encode
+ * as `/^##? /`. Without the era arm the newest entry runs to end-of-file after
+ * a split, which leaves it as the last `##` heading above archived banners.
+ *
+ * Exported because release.ts bounds the previous entry the same way, and two
+ * copies of a boundary is how one of them stays wrong.
+ */
+export const NEXT_SECTION_RE = /^(?:##\s+\[?\d+\.\d+\.\d+|# Era:)/m;
 
 function _reEscape(s: string): string {
     return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
@@ -29,9 +40,9 @@ function _reEscape(s: string): string {
 
 /**
  * Extract the CHANGELOG section for `version` — heading plus body up to the
- * next version heading (or end of file). Returns null when no heading for
- * `version` exists. Matches both heading shapes the generator emits
- * (`## X.Y.Z (date)` and `## [X.Y.Z](compare-url) (date)`).
+ * next version heading or era banner (or end of file). Returns null when no
+ * heading for `version` exists. Matches both heading shapes the generator
+ * emits (`## X.Y.Z (date)` and `## [X.Y.Z](compare-url) (date)`).
  */
 export function extract_changelog_section(text: string, version: string): ChangelogSection | null {
     const headingRe = new RegExp(`^##\\s+\\[?${_reEscape(version)}\\b.*$`, 'm');
@@ -41,7 +52,7 @@ export function extract_changelog_section(text: string, version: string): Change
     }
     const bodyStart = m.index + m[0].length;
     const rest = text.slice(bodyStart);
-    const next = _NEXT_HEADING_RE.exec(rest);
+    const next = NEXT_SECTION_RE.exec(rest);
     const sectionEnd = bodyStart + (next ? next.index : rest.length);
     const body = text
         .slice(bodyStart, sectionEnd)

package/src/scripts/_lib/review_independence.ts

@@ -0,0 +1,111 @@
+/**
+ * Review independence — a recorded property of a review, never a promise.
+ *
+ * The failure this closes: a review whose members all share a model family
+ * produces an artifact byte-indistinguishable from one backed by independent
+ * models, so a consumer reads "no findings" as acceptance either way. The
+ * self-review gate is the live instance — it calls ONE Anthropic client, and
+ * the ledger it writes says nothing about that.
+ *
+ * Derived, never set by hand: `acceptance_status` follows from
+ * `review_independence`, so a same-family set carrying `accepted` is
+ * unrepresentable rather than merely forbidden. Two fields a producer can set
+ * inconsistently would reintroduce the ambiguity the pair exists to remove.
+ *
+ * Evidence: agents/evidence/eval-findings/metric-loop-s03.md.
+ */
+
+/** The five providers `ai_council/config.ts::_VALID_PROVIDERS` admits. */
+const PROVIDER_FAMILY: Readonly<Record<string, string>> = {
+    anthropic: 'anthropic',
+    openai: 'openai',
+    gemini: 'google',
+    xai: 'xai',
+    perplexity: 'perplexity',
+};
+
+export type ReviewIndependence = 'cross-family' | 'same-family' | 'single-member' | 'unknown';
+export type AcceptanceStatus = 'accepted' | 'provisional';
+
+/**
+ * How much INDEPENDENT evidence backs the verdict — deliberately orthogonal to
+ * how much effort the reviewer spent.
+ *
+ * Collapsing the two is what lets a long, careful, single-model pass read as
+ * acceptance: effort is visible in the artifact (findings, detail, rationale)
+ * and independence is not, so a reader with only one axis substitutes the one
+ * they can see. A five-hour same-family review is `single-pass`, and a terse
+ * two-family one is `independent`. That inversion is the point.
+ */
+export type Assurance = 'unreviewed' | 'single-pass' | 'independent';
+
+export function reviewIndependence(memberNames: readonly string[]): ReviewIndependence {
+    const present = memberNames.map((n) => n.trim()).filter((n) => n.length > 0);
+    if (present.length === 0) return 'unknown';
+    if (present.length === 1) return 'single-member';
+    const families = new Set(present.map((n) => PROVIDER_FAMILY[n] ?? `unknown:${n}`));
+    return families.size > 1 ? 'cross-family' : 'same-family';
+}
+
+/**
+ * Only a cross-family set may be described as acceptance.
+ *
+ * `unknown` maps to `provisional`, not to a third value: an absent member set
+ * is not evidence of independence, and the safe direction on an integrity field
+ * is the weaker claim.
+ */
+export function acceptanceStatus(independence: ReviewIndependence): AcceptanceStatus {
+    return independence === 'cross-family' ? 'accepted' : 'provisional';
+}
+
+export function assuranceFor(independence: ReviewIndependence): Assurance {
+    if (independence === 'cross-family') return 'independent';
+    if (independence === 'unknown') return 'unreviewed';
+    return 'single-pass';
+}
+
+export interface IndependenceFields {
+    readonly review_independence: ReviewIndependence;
+    readonly acceptance_status: AcceptanceStatus;
+    readonly assurance: Assurance;
+    readonly reviewers: readonly string[];
+}
+
+/** The three fields plus the member set they were derived from. */
+export function independenceFields(memberNames: readonly string[]): IndependenceFields {
+    const independence = reviewIndependence(memberNames);
+    return {
+        review_independence: independence,
+        acceptance_status: acceptanceStatus(independence),
+        assurance: assuranceFor(independence),
+        reviewers: [...memberNames],
+    };
+}
+
+/**
+ * Consistency violations for an artifact that already carries the fields.
+ *
+ * The gate's teeth: a producer that hand-set `accepted` beside a same-family
+ * independence is caught here rather than trusted.
+ */
+export function independenceViolations(doc: Record<string, unknown>): string[] {
+    const out: string[] = [];
+    const indep = doc['review_independence'];
+    const accept = doc['acceptance_status'];
+    const assur = doc['assurance'];
+    if (indep === undefined) {
+        out.push('review_independence is absent — an artifact that does not declare its independence reads as acceptance by default');
+        return out;
+    }
+    const expectedAccept = acceptanceStatus(indep as ReviewIndependence);
+    if (accept !== expectedAccept) {
+        out.push(
+            `acceptance_status is ${String(accept)} but review_independence ${String(indep)} derives ${expectedAccept}`,
+        );
+    }
+    const expectedAssur = assuranceFor(indep as ReviewIndependence);
+    if (assur !== undefined && assur !== expectedAssur) {
+        out.push(`assurance is ${String(assur)} but review_independence ${String(indep)} derives ${expectedAssur}`);
+    }
+    return out;
+}

package/src/scripts/_lib/session_register.ts

@@ -212,6 +212,20 @@ export interface SessionRecord {
     started_at: string;
     /** ISO-8601 UTC, rewritten every heartbeat. The liveness signal. */
     last_seen: string;
+    /**
+     * Per-detector turn-end refusals THIS session has taken, or absent when it
+     * has taken none.
+     *
+     * `road-to-stop-gate-honesty` § D-2: the gate refuses turn-ends with "no
+     * per-session visibility into how often it happens", and a refused turn-end
+     * costs the user at least one extra model turn. The counts ride this
+     * existing write — the reader is one small `readFileSync` of a record the
+     * gate already maintains, so the heartbeat gains no spawn.
+     *
+     * Optional on purpose: a record written before this field existed, or by a
+     * session that was never refused, is still a valid record.
+     */
+    turn_end_refusals?: Record<string, number>;
 }
 
 /**