@event4u/agent-config 16.5.0 → 17.0.0

package/src/scripts/_lib/test_delta.ts

@@ -47,6 +47,10 @@ export interface FileDiff {
     readonly path: string;
     readonly added: readonly string[];
     readonly removed: readonly string[];
+    /** Per added line: the line above it in the new file is the previous added line. */
+    readonly addedFollows?: readonly boolean[];
+    /** Per removed line: the line above it in the old file is the previous removed line. */
+    readonly removedFollows?: readonly boolean[];
 }
 
 /**
@@ -63,10 +67,17 @@ export function parseUnifiedDiff(diff: string): FileDiff[] {
     let oldPath: string | null = null;
     let added: string[] = [];
     let removed: string[] = [];
+    let addedFollows: boolean[] = [];
+    let removedFollows: boolean[] = [];
+    // Whether the last line seen in each file's sequence was a changed one.
+    let lastNew = false;
+    let lastOld = false;
     const flush = (): void => {
-        if (path !== null) out.push({ path, added, removed });
+        if (path !== null) out.push({ path, added, removed, addedFollows, removedFollows });
         added = [];
         removed = [];
+        addedFollows = [];
+        removedFollows = [];
     };
     for (const line of diff.split('\n')) {
         if (line.startsWith('diff --git ')) {
@@ -92,10 +103,22 @@ export function parseUnifiedDiff(diff: string): FileDiff[] {
             path = p === '/dev/null' ? oldPath : p.replace(/^b\//, '');
             continue;
         }
-        if (line.startsWith('@@')) continue;
+        if (line.startsWith('@@')) {
+            lastNew = lastOld = false;
+            continue;
+        }
         if (path === null) continue;
-        if (line.startsWith('+')) added.push(line.slice(1));
-        else if (line.startsWith('-')) removed.push(line.slice(1));
+        if (line.startsWith('+')) {
+            added.push(line.slice(1));
+            addedFollows.push(lastNew);
+            lastNew = true;
+        } else if (line.startsWith('-')) {
+            removed.push(line.slice(1));
+            removedFollows.push(lastOld);
+            lastOld = true;
+        } else {
+            lastNew = lastOld = false;
+        }
     }
     flush();
     return out;
@@ -132,6 +155,57 @@ const SUPPRESSION =
  */
 const DIFF_LINE_AS_DATA = /['"`][+-]/;
 
+/**
+ * A line the language never executes — prose, not code.
+ *
+ * Found on road-to-an-analysis-family-that-deepens-itself 6.2b: a rename
+ * rewrote a doc comment whose old wording carried "assertion failures" and
+ * "this expectation moves into the list" and whose new wording carried
+ * neither. `ASSERTION` matched both old lines, so a file whose executed
+ * assertions GREW by two array entries was reported as having net-removed two
+ * of them. A gate that reds a test edit which strengthened the test is the
+ * false positive this gate's own docstring says erodes it.
+ *
+ * Strictly a PRECISION fix, never a hole: commenting an assertion out now
+ * counts as the loss it is — the executed line leaves the removed side and the
+ * prose that replaces it no longer cancels it on the added side.
+ *
+ * Deliberately line-leading only. A trailing `// why` after a real assertion
+ * leaves the assertion executable and still counted; the markers cover the
+ * comment syntax of every framework `ASSERTION` spreads across (JS/TS/PHP
+ * `//`, block-comment continuation `*`, Python/shell `#`, SQL/Lua `--`).
+ */
+const COMMENT_LINE = /^\s*(\/\/|\/\*|\*|#|--)/;
+
+/**
+ * A runtime-conditional skip (`it.skipIf(c)`, `describe.runIf(c)`).
+ *
+ * Counted apart from `SUPPRESSION` because it is legitimate far more often — a
+ * tool that is absent, a live flag that is off — and because it is also the
+ * shape that hid a happy path nothing ever ran. So an ADDED one is a
+ * suppression unless the same added line, or the added line directly above it
+ * in the file, carries a non-empty `skip-reason:`. Diff-only by design: the
+ * sites already in the tree are never re-counted.
+ */
+const CONDITIONAL_SKIP = /\.\s*(skipIf|runIf)\s*\(/;
+const SKIP_REASON = /\bskip-reason:\s*\S/;
+
+/** `follows[i]` false means the previous list entry is not the line above (another hunk). */
+function unreasonedConditionalSkips(added: readonly string[], follows?: readonly boolean[]): number {
+    let n = 0;
+    added.forEach((line, i) => {
+        if (!CONDITIONAL_SKIP.test(line) || DIFF_LINE_AS_DATA.test(line)) return;
+        if (COMMENT_LINE.test(line)) return;
+        const above = follows?.[i] === false ? '' : (added[i - 1] ?? '');
+        if (SKIP_REASON.test(line) || SKIP_REASON.test(above)) return;
+        n += 1;
+    });
+    return n;
+}
+
+/** A suppression that only exists inside a comment: a PHPUnit docblock annotation. */
+const DOCBLOCK_SUPPRESSION = /@group\s+skip/;
+
 /** Counted evidence about one test file's diff. */
 export interface WeakeningSignal {
     readonly path: string;
@@ -152,11 +226,20 @@ export interface WeakeningSignal {
  */
 export function weakeningSignal(file: FileDiff): WeakeningSignal {
     const count = (lines: readonly string[], re: RegExp): number =>
-        lines.filter((l) => re.test(l) && !DIFF_LINE_AS_DATA.test(l)).length;
+        lines.filter(
+            (l) =>
+                re.test(l) &&
+                !DIFF_LINE_AS_DATA.test(l) &&
+                (!COMMENT_LINE.test(l) || (re === SUPPRESSION && DOCBLOCK_SUPPRESSION.test(l))),
+        ).length;
     return {
         path: file.path,
         assertionsLost: count(file.removed, ASSERTION) - count(file.added, ASSERTION),
-        suppressionsGained: count(file.added, SUPPRESSION) - count(file.removed, SUPPRESSION),
+        suppressionsGained:
+            count(file.added, SUPPRESSION) +
+            unreasonedConditionalSkips(file.added, file.addedFollows) -
+            count(file.removed, SUPPRESSION) -
+            unreasonedConditionalSkips(file.removed, file.removedFollows),
     };
 }
 

package/src/scripts/_lib/tier_downshift_verdict.ts

@@ -0,0 +1,222 @@
+/**
+ * Verdict for the two pre-registered tier-downshift claims
+ * (`tier-downshift-quality-held`, `tier-downshift-limit-relief` in
+ * `docs/CLAIMS.md`) over a paired run: every slice once on `lite`, once on the
+ * inherited tier, both verified by the same deterministic check.
+ *
+ * Pure: the bars are constants here, the rows come from the caller, and an
+ * underpowered run backs nothing in either direction.
+ */
+
+/** Quality bar: the lite arm's fail-or-escalate rate may exceed inherit's by at most this. */
+export const QUALITY_MARGIN_PP = 5;
+/** The escalation kill of `ask_economy` in `src/config/dispatch-economy-metrics.json`. */
+export const ESCALATION_KILL = 0.3;
+/** Pair cap of the paired run (decision A2). */
+export const PAIR_CAP = 60;
+/** Below this many complete pairs no rate is read at all. */
+export const MIN_PAIRS = 30;
+
+export type Arm = 'lite' | 'inherit';
+
+/** Token usage of one leg, as the host reports it. */
+export interface LegUsage {
+    input: number;
+    cache_read: number;
+    cache_write: number;
+    output: number;
+}
+
+/** USD per million tokens for one model family. */
+export interface Price {
+    input: number;
+    output: number;
+    cache_write: number;
+    cache_read: number;
+}
+
+export interface LegResult {
+    slice_id: string;
+    arm: Arm;
+    /** The deterministic check passed. */
+    verified: boolean;
+    /** The leg could not finish and the head re-ran it one tier up. */
+    escalated: boolean;
+    served_model: string;
+    usage: LegUsage;
+    /** Usage of the head's re-run one tier up, charged to this leg's arm. */
+    rerun?: { served_model: string; usage: LegUsage; verified: boolean };
+    /** Other model usage the leg caused (an advisor it consulted), charged to this leg's arm. */
+    extra?: { served_model: string; usage: LegUsage }[];
+}
+
+export interface ArmSummary {
+    slices: number;
+    failed_or_escalated: number;
+    escalated: number;
+    verified: number;
+    cost_usd: number;
+    cost_per_verified_usd: number | null;
+}
+
+export type QualityVerdict = 'held' | 'breached' | 'underpowered';
+export type CostVerdict = 'relief' | 'no-relief' | 'underpowered';
+
+export interface TierDownshiftVerdict {
+    pairs: number;
+    lite: ArmSummary;
+    inherit: ArmSummary;
+    /** lite minus inherit fail-or-escalate rate, in percentage points. */
+    diff_pp: number | null;
+    /** Half-width of the 95 % interval of the paired difference, in percentage points. */
+    ci_half_width_pp: number | null;
+    escalation_rate_lite: number | null;
+    quality: QualityVerdict;
+    cost: CostVerdict;
+    /** Both claims backed: the kind may take lite as its default dispatch. */
+    flip: boolean;
+    notes: string[];
+}
+
+const FAMILIES = ['haiku', 'sonnet', 'opus', 'fable'] as const;
+
+function priceFor(model: string, prices: Record<string, Price>): Price | null {
+    const m = model.toLowerCase();
+    const family = FAMILIES.find((f) => m.includes(f));
+    return family ? (prices[family] ?? null) : null;
+}
+
+function legCost(model: string, usage: LegUsage, prices: Record<string, Price>, notes: string[]): number {
+    const p = priceFor(model, prices);
+    if (p === null) {
+        notes.push(`no price for served model ${model}; its leg is priced at 0 and the cost verdict is underpowered`);
+        return Number.NaN;
+    }
+    return (
+        (usage.input * p.input + usage.cache_read * p.cache_read + usage.cache_write * p.cache_write + usage.output * p.output) /
+        1_000_000
+    );
+}
+
+function summarise(legs: LegResult[], prices: Record<string, Price>, notes: string[]): ArmSummary {
+    let cost = 0;
+    let verified = 0;
+    let failedOrEscalated = 0;
+    let escalated = 0;
+    for (const leg of legs) {
+        cost += legCost(leg.served_model, leg.usage, prices, notes);
+        for (const x of leg.extra ?? []) cost += legCost(x.served_model, x.usage, prices, notes);
+        if (leg.escalated) escalated++;
+        if (!leg.verified || leg.escalated) failedOrEscalated++;
+        if (leg.rerun) {
+            cost += legCost(leg.rerun.served_model, leg.rerun.usage, prices, notes);
+            if (leg.rerun.verified) verified++;
+        } else if (leg.verified) {
+            verified++;
+        }
+    }
+    return {
+        slices: legs.length,
+        failed_or_escalated: failedOrEscalated,
+        escalated,
+        verified,
+        cost_usd: cost,
+        cost_per_verified_usd: verified > 0 && Number.isFinite(cost) ? cost / verified : null,
+    };
+}
+
+/**
+ * Read the paired run. A slice counts only when both arms ran it once; extra
+ * or one-sided rows are noted and left out rather than guessed at.
+ */
+export function tierDownshiftVerdict(rows: readonly LegResult[], prices: Record<string, Price>): TierDownshiftVerdict {
+    const notes: string[] = [];
+    const bySlice = new Map<string, Partial<Record<Arm, LegResult>>>();
+    for (const r of rows) {
+        const entry = bySlice.get(r.slice_id) ?? {};
+        if (entry[r.arm]) {
+            notes.push(`slice ${r.slice_id} ran twice on ${r.arm}; the second row is ignored`);
+            continue;
+        }
+        entry[r.arm] = r;
+        bySlice.set(r.slice_id, entry);
+    }
+    const pairs: [LegResult, LegResult][] = [];
+    for (const [id, e] of bySlice) {
+        if (e.lite && e.inherit) pairs.push([e.lite, e.inherit]);
+        else notes.push(`slice ${id} has one arm only and is left out`);
+    }
+    if (pairs.length > PAIR_CAP) notes.push(`${pairs.length} pairs exceed the cap of ${PAIR_CAP}; all are read`);
+
+    const lite = summarise(pairs.map((p) => p[0]), prices, notes);
+    const inherit = summarise(pairs.map((p) => p[1]), prices, notes);
+    const n = pairs.length;
+
+    let diffPp: number | null = null;
+    let halfWidthPp: number | null = null;
+    let escalationRate: number | null = null;
+    if (n > 0) {
+        // Paired difference of the fail-or-escalate indicator: only discordant pairs move it.
+        let b = 0;
+        let c = 0;
+        for (const [l, i] of pairs) {
+            const lf = !l.verified || l.escalated;
+            const inf = !i.verified || i.escalated;
+            if (lf && !inf) b++;
+            if (!lf && inf) c++;
+        }
+        const diff = (b - c) / n;
+        const variance = (b + c - (b - c) ** 2 / n) / (n * n);
+        diffPp = diff * 100;
+        halfWidthPp = 1.96 * Math.sqrt(Math.max(variance, 0)) * 100;
+        // No discordant pair gives a zero-width interval; the rule of three bounds
+        // an unseen disagreement rate at 3/n instead.
+        if (b + c === 0) halfWidthPp = (3 / n) * 100;
+        escalationRate = lite.escalated / n;
+    }
+
+    let quality: QualityVerdict;
+    if (n < MIN_PAIRS || diffPp === null || halfWidthPp === null || escalationRate === null) {
+        quality = 'underpowered';
+        notes.push(`${n} complete pairs, below the floor of ${MIN_PAIRS}`);
+    } else if (escalationRate >= ESCALATION_KILL || diffPp - halfWidthPp > QUALITY_MARGIN_PP) {
+        quality = 'breached';
+    } else if (diffPp + halfWidthPp <= QUALITY_MARGIN_PP) {
+        quality = 'held';
+    } else {
+        quality = 'underpowered';
+        notes.push('the interval straddles the 5 pp margin');
+    }
+
+    let cost: CostVerdict;
+    if (quality === 'underpowered' || lite.cost_per_verified_usd === null || inherit.cost_per_verified_usd === null) {
+        cost = 'underpowered';
+    } else {
+        cost = lite.cost_per_verified_usd < inherit.cost_per_verified_usd ? 'relief' : 'no-relief';
+    }
+
+    return {
+        pairs: n,
+        lite,
+        inherit,
+        diff_pp: diffPp,
+        ci_half_width_pp: halfWidthPp,
+        escalation_rate_lite: escalationRate,
+        quality,
+        cost,
+        flip: quality === 'held' && cost === 'relief',
+        notes,
+    };
+}
+
+/**
+ * A kind already on lite reverts the moment a later window breaches the kill;
+ * a kind not on lite moves only on a verdict that backs both claims.
+ */
+export function kindTierAfter(current: 'lite' | 'inherit', verdict: TierDownshiftVerdict): 'lite' | 'inherit' {
+    if (current === 'lite') {
+        const breached = verdict.quality === 'breached' || (verdict.escalation_rate_lite ?? 0) >= ESCALATION_KILL;
+        return breached ? 'inherit' : 'lite';
+    }
+    return verdict.flip ? 'lite' : 'inherit';
+}

package/src/scripts/_lib/tool_result_census.ts

@@ -0,0 +1,62 @@
+/**
+ * Reader for `agents/runtime/state/tool-result-census.jsonl`
+ * (`road-to-bytes-the-model-actually-receives` step 1.3).
+ *
+ * The census holds two line shapes. Lines written before the two-field schema
+ * carry `{ts, tool, bytes}`; lines from 2026-09-30 on carry
+ * `{ts, tool, raw_bytes, delivered_bytes, measurable}`. The census is
+ * append-only, so both shapes stay in the file for good.
+ *
+ * A legacy `bytes` counted the same thing `raw_bytes` counts — the host's hook
+ * envelope. It says nothing about what was delivered, so a legacy line reads
+ * as `delivered_bytes: null`. Reading it as equal would invent a figure for
+ * every line written before the second field existed.
+ *
+ * Neither shape counts what reached the model; that is
+ * `tool-result-payload-bytes`, read from transcripts (`cc_transcript.ts`).
+ */
+
+export type CensusShape = 'legacy' | 'two-field';
+
+export interface CensusReading {
+    ts: string;
+    tool: string | null;
+    /** Hook-envelope bytes, or `null` when the line recorded none. */
+    raw_bytes: number | null;
+    /** Hook-envelope bytes after any rewrite; `null` when unknown — always so for a legacy line. */
+    delivered_bytes: number | null;
+    shape: CensusShape;
+}
+
+function numOrNull(v: unknown): number | null {
+    return typeof v === 'number' && Number.isFinite(v) ? v : null;
+}
+
+/** One census line, or `null` for a blank, unparseable or shapeless line. */
+export function readCensusLine(line: string): CensusReading | null {
+    if (line.trim() === '') return null;
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(line);
+    } catch {
+        return null;
+    }
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return null;
+    const rec = parsed as Record<string, unknown>;
+    const ts = typeof rec['ts'] === 'string' ? rec['ts'] : null;
+    if (ts === null) return null;
+    const tool = typeof rec['tool'] === 'string' ? rec['tool'] : null;
+    if ('raw_bytes' in rec || 'delivered_bytes' in rec) {
+        return {
+            ts,
+            tool,
+            raw_bytes: numOrNull(rec['raw_bytes']),
+            delivered_bytes: numOrNull(rec['delivered_bytes']),
+            shape: 'two-field',
+        };
+    }
+    if ('bytes' in rec) {
+        return { ts, tool, raw_bytes: numOrNull(rec['bytes']), delivered_bytes: null, shape: 'legacy' };
+    }
+    return null;
+}

package/src/scripts/_lib/trigger_routers.ts

@@ -1,3 +1,4 @@
+// egress-content: fixed-corpus — sends the trigger-eval corpus (skill descriptions and the eval's own prompts), never session content
 // fetch-based TriggerRouter implementations for the cross-model eval smoke
 // (roadmap T-004). These mirror the SDK-based AnthropicRouter in
 // skill_trigger_eval.ts but talk plain HTTP via an injectable `fetch`, so the