@event4u/agent-config 14.12.0 → 14.13.0

package/src/scripts/ai_council/judge_position_bias.ts

@@ -0,0 +1,197 @@
+/**
+ * Per-judge position-consistency — order-swap over sampled pairwise judgments.
+ *
+ * `road-to-inbox-harvest-2026-08-e-council-topology-evidence` Phase 3, step 3.5:
+ * "repeat sampled pairwise judgments with candidate order reversed; emit a
+ * per-judge position-consistency metric", verified by "the metric exists per
+ * judge and is reported with the verdict".
+ *
+ * ── What already existed, and why this is not a second copy of it ────────────
+ * `check_quality_regression.evaluatePair` (`:84-108`) already judges every pair
+ * in both orders and resolves a flip to `inconsistent` rather than to a winner,
+ * and `_lib/judge_hygiene.ts:5-9` records that as stronger than the step that
+ * asked for it. Two things it does not do, and both are the point here:
+ *
+ *   1. It is SINGLE-JUDGE. `aggregate` returns one run-wide `inconsistency_rate`
+ *      (`:216`), so two judges of differing reliability average into one number
+ *      that describes neither. The step asks for the metric PER JUDGE, which is
+ *      what makes it actionable: you can drop the flipper.
+ *   2. It reports presence, not DIRECTION. A judge that flips at random and a
+ *      judge that always prefers whatever it was shown first produce the same
+ *      inconsistency rate and need opposite remedies — the first is noise, the
+ *      second is a systematic bias that survives averaging and quietly decides
+ *      verdicts. `first_position_rate` separates them.
+ *
+ * That module is the thin-vs-eager token gate and is not council code; this one
+ * is council-side and stays a pure library — no transport, no fixture path, no
+ * `main`. The judge is an injected function, so the whole surface is testable
+ * against a scripted judge and costs nothing to exercise.
+ *
+ * ── Honest scope ────────────────────────────────────────────────────────────
+ * The shipped council has NO pairwise judging stage (`grep -rn pairwise
+ * src/scripts/ai_council` returns nothing): members deliberate, peer-review and
+ * are scored on findings, never compared two at a time. So this module is the
+ * metric and its renderer, exercised by the provider-leakage bench that Phase 3
+ * builds; it does not claim a live council verdict currently carries the line,
+ * because there is currently no live council judgment for it to describe.
+ */
+
+import { _sha256_hex } from './blind_review.js';
+
+/** A judge's preference between the two PRESENTED candidates — order-relative. */
+export type PairwiseVerdict = 'first' | 'second' | 'tie';
+
+/**
+ * The pluggable judge. `first`/`second` are the candidate bodies in PRESENTED
+ * order, so a judge implementation cannot tell which arm it is looking at
+ * unless the bodies themselves leak it.
+ */
+export type PairwiseJudge = (ctx: { readonly id: string }, first: string, second: string) => PairwiseVerdict;
+
+/** How a pair resolved once both orders were seen. */
+export type PairResolution = 'a' | 'b' | 'tie' | 'inconsistent';
+
+export interface SwapObservation {
+    readonly id: string;
+    /** Verdict with A presented first. */
+    readonly forward: PairwiseVerdict;
+    /** Verdict with B presented first. */
+    readonly reverse: PairwiseVerdict;
+    readonly resolution: PairResolution;
+}
+
+/**
+ * Judge one (a, b) pair in BOTH orders.
+ *
+ * A verdict that does not survive the swap resolves to `inconsistent`, never to
+ * a winner — a flagged tie would still be a tie in somebody's denominator,
+ * while `inconsistent` is its own bucket with its own reported rate.
+ */
+export function judgeBothOrders(id: string, a: string, b: string, judge: PairwiseJudge): SwapObservation {
+    const ctx = { id };
+    const forward = judge(ctx, a, b);
+    const reverse = judge(ctx, b, a);
+    // Normalise each verdict to the CANDIDATE it names, not the position.
+    const wf: PairResolution = forward === 'first' ? 'a' : forward === 'second' ? 'b' : 'tie';
+    const wr: PairResolution = reverse === 'first' ? 'b' : reverse === 'second' ? 'a' : 'tie';
+    const resolution: PairResolution = wf === wr ? wf : 'inconsistent';
+    return { id, forward, reverse, resolution };
+}
+
+export interface JudgeConsistency {
+    readonly judge: string;
+    /** Pairs actually repeated in both orders. */
+    readonly sampled: number;
+    /** Both orders named the same candidate, or both said tie. */
+    readonly consistent: number;
+    /** The two orders disagreed. */
+    readonly inconsistent: number;
+    /**
+     * `consistent / sampled`. `null` at zero pairs — a rate over an empty
+     * denominator is not 1.0, and reporting it as 1.0 would make an unrun swap
+     * indistinguishable from a perfectly stable judge.
+     */
+    readonly position_consistency: number | null;
+    /**
+     * Share of DECISIVE verdicts (`first`/`second`, both orders pooled) that
+     * named whichever candidate was shown FIRST. ~0.5 is unbiased; near 1.0 is
+     * primacy, near 0.0 recency. `null` when nothing decisive was said.
+     *
+     * This is the direction an inconsistency rate cannot show, and the reason a
+     * judge that flips at random is a different problem from one that always
+     * prefers the top of the list.
+     */
+    readonly first_position_rate: number | null;
+    /** Decisive verdicts counted for `first_position_rate`. */
+    readonly decisive_verdicts: number;
+}
+
+/** Per-judge metric over that judge's own swapped observations. */
+export function positionConsistency(judge: string, obs: readonly SwapObservation[]): JudgeConsistency {
+    let consistent = 0;
+    let inconsistent = 0;
+    let firstWins = 0;
+    let decisive = 0;
+    for (const o of obs) {
+        if (o.resolution === 'inconsistent') {
+            inconsistent += 1;
+        } else {
+            consistent += 1;
+        }
+        for (const v of [o.forward, o.reverse]) {
+            if (v === 'first') {
+                decisive += 1;
+                firstWins += 1;
+            } else if (v === 'second') {
+                decisive += 1;
+            }
+        }
+    }
+    const sampled = obs.length;
+    return {
+        judge,
+        sampled,
+        consistent,
+        inconsistent,
+        position_consistency: sampled > 0 ? consistent / sampled : null,
+        first_position_rate: decisive > 0 ? firstWins / decisive : null,
+        decisive_verdicts: decisive,
+    };
+}
+
+/**
+ * Deterministic sample of `ids` at `rate`, seeded by `seed`.
+ *
+ * The step says "SAMPLED pairwise judgments", because the swap doubles judge
+ * calls and doubling every one of them to measure the judge is a cost the
+ * measurement does not need. Deterministic so a published consistency figure is
+ * reproducible from the report's own seed — a sample nobody can re-draw is a
+ * number nobody can check.
+ *
+ * `rate <= 0` samples nothing; `rate >= 1` samples everything.
+ */
+export function sampleForSwap(ids: readonly string[], rate: number, seed: string): string[] {
+    if (!(rate > 0)) {
+        return [];
+    }
+    if (rate >= 1) {
+        return [...ids];
+    }
+    return ids.filter((id) => _unitHash(`${seed} ${id}`) < rate);
+}
+
+/**
+ * Stable [0,1) hash of a string.
+ *
+ * Reuses `blind_review._sha256_hex` rather than carrying a local one. That
+ * export exists for exactly this reason — `blind_review.ts:38-44` records that
+ * two hashes of one input are two answers to one question and the second is the
+ * one nobody updates. A hand-rolled FNV-1a was tried here first and its low bits
+ * were skewed enough that a rate of 0.25 sampled half the corpus; the test that
+ * caught it is kept.
+ */
+function _unitHash(s: string): number {
+    return parseInt(_sha256_hex(s).slice(0, 8), 16) / 4294967296;
+}
+
+/**
+ * One report line per judge, for emission BESIDE a verdict.
+ *
+ * `n/a` rather than a number when nothing was sampled: the step's verify is
+ * that the metric is reported, and a report that silently prints nothing for an
+ * unrun swap satisfies it only by looking like it ran.
+ */
+export function renderPositionConsistency(rows: readonly JudgeConsistency[]): string {
+    if (rows.length === 0) {
+        return 'position consistency: no judge sampled — order-swap NOT RUN.';
+    }
+    const pct = (v: number | null): string => (v === null ? 'n/a' : `${(v * 100).toFixed(0)}%`);
+    return rows
+        .map(
+            (r) =>
+                `position consistency · ${r.judge}: ${pct(r.position_consistency)} ` +
+                `(${String(r.consistent)}/${String(r.sampled)} pairs survived the swap; ` +
+                `first-position rate ${pct(r.first_position_rate)} over ${String(r.decisive_verdicts)} decisive verdicts)`,
+        )
+        .join('\n');
+}

package/src/scripts/ai_council/orchestrator.ts

@@ -21,10 +21,14 @@
 // Failure normalisation (one member's exception → `error`-set
 // CouncilResponse, never raise) is unchanged.
 
-import { ConsensusResult, CouncilQuestion, PeerReviewResult } from './orchestrator_results.js';
-import type { RunConsensusScoringOptions, RunPeerReviewOptions } from './orchestrator_results.js';
+import type { ConsensusResult } from './orchestrator_results.js';
+import { CouncilQuestion, PeerReviewResult } from './orchestrator_results.js';
+import type { RunPeerReviewOptions } from './orchestrator_results.js';
 export { ConsensusResult, CouncilQuestion, PeerReviewResult } from './orchestrator_results.js';
 export type { RunConsensusScoringOptions, RunPeerReviewOptions } from './orchestrator_results.js';
+// Moved to its own module for the source-size budget; re-exported so every
+// existing importer of `run_consensus_scoring` is untouched.
+export { run_consensus_scoring } from './consensus_round.js';
 import {
     record_spend as _record_daily_spend,
     today_spend_usd as _today_spend_usd,
@@ -39,16 +43,9 @@ import {
 import type {
     ConsensusMetadata,
     Finding} from './consensus.js';
-import {
-    ConsensusBucket,
-    FindingScore,
-    aggregate_scores,
-    anonymize_findings,
-    anonymize_responses,
-    bucket_by_threshold,
-    parse_findings_outcome,
-    parse_scores_response,
-} from './consensus.js';
+import type { ConsensusBucket } from './consensus.js';
+import { anonymize_responses } from './consensus.js';
+import { composeFinalRoundSuffix } from './inline_findings.js';
 import type {
     CostEstimate,
     PriceTable} from './pricing.js';
@@ -64,9 +61,7 @@ import {
     ANTI_CONFORMITY_DIRECTIVE,
     describe_verdict_mismatch,
     STANCE_LINE_CONTRACT,
-    build_extraction_user_prompt,
     build_peer_review_user_prompt,
-    build_scoring_user_prompt,
     peer_review_synthesis_addendum,
     synthesis_template,
     system_prompt_for,
@@ -366,6 +361,8 @@ export interface ConsultOptions {
      * context). Default `null` → byte-identical to today.
      */
     no_project_context_members?: ReadonlySet<string> | null;
+    /** Phase 1B — `composeFinalRoundSuffix`. Default `false` → prompt unchanged. */
+    inline_findings?: boolean;
     /** Mid-flight cli→api fallback; absent → no retry. See `mid_flight_fallback`. */
     cli_fallback?: CliFallbackOptions | null;
 }
@@ -435,15 +432,13 @@ export function consult(
     let current_suffix = '';
 
     const stance_tally = opts.stance_tally ?? false;
+    const inline_findings = opts.inline_findings ?? false;
     for (let round_idx = 0; round_idx < rounds; round_idx++) {
-        // Phase 1: the FINAL round carries the mandatory STANCE closing-line
-        // contract when the tally is enabled. Off (default) → the original
-        // question object flows through untouched, byte-identical to today.
+        // The FINAL round carries the opt-in closing contracts (STANCE line,
+        // inline findings). Both off → the question flows through untouched,
+        // byte-identical to today. Order lives in the composer.
         const is_final = round_idx === rounds - 1;
-        const suffix_for_round =
-            is_final && stance_tally
-                ? `${current_suffix}\n\n---\n\n${STANCE_LINE_CONTRACT}`
-                : current_suffix;
+        const suffix_for_round = composeFinalRoundSuffix(current_suffix, { is_final, inline_findings, stance_tally });
         const prompt_for_round = `${question.user_prompt}${suffix_for_round}`;
         const round_question =
             round_idx === 0 && prompt_for_round === question.user_prompt
@@ -1581,188 +1576,6 @@ export function run_peer_review(
 
 
 
-/**
- * Two-pass consensus round (Phase 4 / F3).
- *
- * Pass 1 — extraction: each member re-emits its own deliberation as
- * a JSON array of `{id, text}` findings. Pass 2 — scoring: each
- * member sees the *other* members' findings under anonymous labels
- * and rates them 1-10 + agree/disagree + reason.
- *
- * The cost budget is shared across both passes; the daily ledger
- * receives both. Errors in one member's extraction or scoring tag
- * that member but never abort the round.
- */
-export function run_consensus_scoring(
-    members: ExternalAIClient[],
-    deliberation_responses: CouncilResponse[],
-    opts: RunConsensusScoringOptions = {},
-): ConsensusResult {
-    const budget = opts.budget ?? null;
-    const table = opts.table ?? null;
-    const on_overrun = opts.on_overrun ?? null;
-    const project = opts.project ?? null;
-    const original_ask = opts.original_ask ?? '';
-    const max_tokens = opts.max_tokens ?? DEFAULT_MAX_TOKENS;
-    const strong_threshold = opts.strong_threshold ?? 0.7;
-    const minority_threshold = opts.minority_threshold ?? 0.4;
-
-    if (members.length === 0 || deliberation_responses.length === 0) {
-        return new ConsensusResult({
-            bucket: new ConsensusBucket(),
-            findings: [],
-            scores: [],
-            metadata: new Map(),
-            extraction_responses: [],
-            scoring_responses: [],
-        });
-    }
-
-    // ── Pass 1: extraction ──────────────────────────────────────────
-    const member_by_name = new Map<string, ExternalAIClient>();
-    for (const m of members) {
-        member_by_name.set(m.name, m);
-    }
-    const extraction_responses: CouncilResponse[] = [];
-    const all_findings: Finding[] = [];
-    // Per-member extraction outcome — step 2.2. Recorded rather than derived from the
-    // findings count, because "zero findings" and "could not be read" are different facts
-    // and the count cannot tell them apart. `parsed-after-reask` is kept distinct from
-    // `parsed`: a member that needed a second ask is a signal about the prompt or the
-    // member, and collapsing it into `parsed` would hide the only evidence that the
-    // re-ask is doing anything.
-    const parse_outcomes = new Map<string, string>();
-    for (const resp of deliberation_responses) {
-        const member = member_by_name.get(resp.provider);
-        if (member === undefined || resp.error || !_pyStrip(resp.text)) {
-            continue;
-        }
-        const question = new CouncilQuestion({
-            mode: 'prompt',
-            user_prompt: build_extraction_user_prompt(resp.text),
-            max_tokens,
-        });
-        const extracted = consult([member], question, budget, {
-            table,
-            on_overrun,
-            project,
-            original_ask,
-        });
-        extraction_responses.push(...extracted);
-        if (extracted.length === 0 || extracted[0]!.error) {
-            continue;
-        }
-        const source = `${member.name}:${member.model}`;
-        let ex = parse_findings_outcome(extracted[0]!.text, { source });
-
-        // Step 2.2 — ONE bounded re-ask on `parse_failed`, and one only.
-        //
-        // A member that said something no parser could read used to be indistinguishable
-        // from a member that found nothing: `parse_findings_response` returned `[]` for
-        // both, and this loop pushed the empty array with no branch. So an unparseable
-        // answer counted as a clean zero-findings review.
-        //
-        // ONE re-ask, not a loop. A re-ask is a paid call, and an unbounded retry turns
-        // one unparseable answer into open-ended spend — the failure mode the N=3
-        // validation budget exists for, reached here by a different road. `empty` is NOT
-        // re-asked: a member that said nothing has nothing to restate, and re-asking it
-        // buys a second silence.
-        if (ex.outcome === 'parse_failed') {
-            const retry_q = new CouncilQuestion({
-                mode: 'prompt',
-                user_prompt:
-                    build_extraction_user_prompt(resp.text) +
-                    '\n\nYour previous answer could not be parsed. Reply with ONLY a JSON ' +
-                    'array of {"id": string, "text": string} objects, no prose before or ' +
-                    'after it. An empty array [] is a valid answer meaning you found nothing.',
-                max_tokens,
-            });
-            const retried = consult([member], retry_q, budget, {
-                table,
-                on_overrun,
-                project,
-                original_ask,
-            });
-            extraction_responses.push(...retried);
-            if (retried.length > 0 && !retried[0]!.error) {
-                ex = parse_findings_outcome(retried[0]!.text, { source });
-            }
-            parse_outcomes.set(source, ex.outcome === 'parsed' ? 'parsed-after-reask' : 'parse_failed');
-        } else {
-            parse_outcomes.set(source, ex.outcome);
-        }
-        all_findings.push(...ex.findings);
-    }
-
-    if (all_findings.length === 0) {
-        return new ConsensusResult({
-            bucket: new ConsensusBucket(),
-            findings: [],
-            scores: [],
-            metadata: new Map(),
-            parse_outcomes,
-            extraction_responses,
-            scoring_responses: [],
-        });
-    }
-
-    // ── Pass 2: scoring (each member rates the OTHERS' findings) ────
-    const scoring_responses: CouncilResponse[] = [];
-    const all_scores: FindingScore[] = [];
-    for (const member of members) {
-        const scorer = `${member.name}:${member.model}`;
-        const others = all_findings.filter((f) => f.source !== scorer);
-        if (others.length === 0) {
-            continue;
-        }
-        const anon = anonymize_findings(others);
-        const label_to_id = new Map<string, string>();
-        const anon_text = new Map<string, string>();
-        for (const [label, f] of anon) {
-            label_to_id.set(label, f.id);
-            anon_text.set(label, f.text);
-        }
-        const question = new CouncilQuestion({
-            mode: 'prompt',
-            user_prompt: build_scoring_user_prompt(anon_text),
-            max_tokens,
-        });
-        const scored = consult([member], question, budget, {
-            table,
-            on_overrun,
-            project,
-            original_ask,
-        });
-        scoring_responses.push(...scored);
-        if (scored.length === 0 || scored[0]!.error) {
-            continue;
-        }
-        for (const s of parse_scores_response(scored[0]!.text, { scorer })) {
-            const real_id = label_to_id.get(s.finding_id);
-            if (real_id === undefined) {
-                continue;
-            }
-            all_scores.push(
-                new FindingScore(real_id, s.scorer, s.score, s.agree, s.reason),
-            );
-        }
-    }
-
-    const metadata = aggregate_scores(all_findings, all_scores);
-    const bucket = bucket_by_threshold(all_findings, metadata, {
-        strong: strong_threshold,
-        minority: minority_threshold,
-    });
-    return new ConsensusResult({
-        bucket,
-        findings: all_findings,
-        scores: all_scores,
-        metadata,
-        parse_outcomes,
-        extraction_responses,
-        scoring_responses,
-    });
-}
 
 /**
  * Format the per-member meta line — tokens, cost (or subscription), latency.

package/src/scripts/ai_council/orchestrator_results.ts

@@ -25,7 +25,14 @@ import type { ProjectContext } from './project_context.js';
 // The alternative was leaving the two options interfaces behind, which would have meant
 // finding 19 lines elsewhere for no design reason.
 import type { OnOverrunCallback } from './orchestrator.js';
-import type { ConsensusBucket, ConsensusMetadata, Finding, FindingScore } from './consensus.js';
+import type {
+    ConsensusBucket,
+    ConsensusMetadata,
+    Finding,
+    FindingScore,
+    FindingsExtraction,
+} from './consensus.js';
+import type { RecordedExtractionOutcome } from './inline_findings.js';
 
 /**
  * Bundle returned by `run_peer_review()` (Phase 5 / F1).
@@ -121,7 +128,7 @@ export class ConsensusResult {
      *
      * Additive and optional, so no existing constructor call changes.
      */
-    parse_outcomes: Map<string, string>;
+    parse_outcomes: Map<string, RecordedExtractionOutcome>;
     extraction_responses: CouncilResponse[];
     scoring_responses: CouncilResponse[];
 
@@ -130,7 +137,7 @@ export class ConsensusResult {
         findings: Finding[];
         scores: FindingScore[];
         metadata: Map<string, ConsensusMetadata>;
-        parse_outcomes?: Map<string, string>;
+        parse_outcomes?: Map<string, RecordedExtractionOutcome>;
         extraction_responses: CouncilResponse[];
         scoring_responses: CouncilResponse[];
     }) {
@@ -163,4 +170,17 @@ export interface RunConsensusScoringOptions {
     max_tokens?: number;
     strong_threshold?: number;
     minority_threshold?: number;
+    /**
+     * Phase 1B: findings already harvested from each member's own deliberation
+     * reply by `harvest_inline_findings`, keyed `provider:model`. A member with
+     * an entry skips the separate extraction call entirely; a member without one
+     * takes the shipped extraction path unchanged. Absent → extraction-always,
+     * call-for-call identical to today.
+     *
+     * A pre-computed map rather than a boolean because the harvest must run
+     * BEFORE peer review and synthesis read the responses, which is upstream of
+     * this function — see `harvest_inline_findings` for why that ordering is
+     * load-bearing.
+     */
+    inline_extractions?: ReadonlyMap<string, FindingsExtraction> | null;
 }

package/src/scripts/ai_council/pricing.ts

@@ -148,6 +148,112 @@ export function estimate_cost(
     };
 }
 
+// ── billable-aware aggregation ─────────────────────────────────────
+//
+// A vendor-official CLI seat (anthropic / openai / gemini) runs under the
+// user's subscription auth and is `billable = false`; the contract states it
+// in as many words (`docs/contracts/ai-council-config.md:173`). Pricing such
+// an answer at API rates and reporting the figure as spend is the defect these
+// two helpers exist to close, found 2026-08-27: `council_cli.ts` ran
+// `estimate_cost` over every non-errored response with no `billable` check, so
+// one run printed `TOTAL: $0.0000` from the pre-run path (which filters) and
+// `actual $0.1055` from the post-run path, two lines apart.
+//
+// Why the coercion below is not paranoia: `_serialise_responses` stringifies
+// every metadata value with `String(v)`, so a persisted artefact carries
+// `"billable": "false"` — and `Boolean("false")` is `true` in JavaScript. A
+// consumer reading the field back with a bare `Boolean()` sees a subscription
+// seat as billable, which is the same wrong answer arrived at from the other
+// direction.
+
+/** The subset of a `CouncilResponse` the cost aggregation reads. */
+export interface BillableCostInput {
+    provider: string;
+    model: string;
+    input_tokens: number;
+    output_tokens: number;
+    error?: string | null;
+    metadata?: Record<string, unknown> | null;
+}
+
+/**
+ * Does this response represent money actually spent?
+ *
+ * ABSENT means billable. A missing flag is not evidence of a free call, and
+ * defaulting the other way would let a client that forgot to stamp its
+ * transport silently zero its own spend.
+ */
+export function isBillableResponse(r: BillableCostInput): boolean {
+    const raw = (r.metadata ?? {})['billable'];
+    if (raw === undefined || raw === null) {
+        return true;
+    }
+    if (typeof raw === 'boolean') {
+        return raw;
+    }
+    if (typeof raw === 'string') {
+        // The persisted form. Only the two literals a `String(v)` round-trip
+        // can produce are interpreted; anything else falls through to the
+        // conservative default rather than being guessed at.
+        const t = raw.trim().toLowerCase();
+        if (t === 'false') {
+            return false;
+        }
+        return true;
+    }
+    // Every other type is BILLABLE, not truthiness-of-value. The first draft of
+    // this function ended `return Boolean(raw)` while its own comment promised a
+    // conservative default, and a council review (2/2) caught the contradiction:
+    // `Boolean(0)` and `Boolean([])` are `false`, so a numeric or empty-array
+    // value would have zeroed a billable seat's spend. The serialiser should
+    // never emit those, which is exactly why guessing at them is wrong.
+    return true;
+}
+
+/**
+ * Did every answered response come from a non-billable transport?
+ *
+ * Exported because the CLI's output line needs it and must NOT infer it from a
+ * zero total. A council review (2/2) rated that inference the highest-severity
+ * finding in the first draft, and the reasoning is concrete: `estimate_cost`
+ * returns `0.0` when `lookup` finds no price row, so an unpriced BILLABLE model
+ * produces a zero total. So do a billable seat that errored and one that
+ * reported zero tokens. All three would have printed "all seats
+ * subscription-authed" over real spend.
+ *
+ * Vacuously false on an empty or all-errored set: with nothing answered there is
+ * no transport to make a claim about, and silence is the honest output.
+ */
+export function allSeatsNonBillable(responses: BillableCostInput[]): boolean {
+    const answered = responses.filter((r) => !r.error);
+    if (answered.length === 0) {
+        return false;
+    }
+    return answered.every((r) => !isBillableResponse(r));
+}
+
+/**
+ * Total USD actually spent across a set of responses.
+ *
+ * Errored responses contribute nothing (no tokens were billed for an answer
+ * that never arrived), and non-billable transports contribute nothing (the
+ * subscription already paid). What remains is spend.
+ */
+export function sumBillableCost(responses: BillableCostInput[], table: PriceTable): number {
+    let total = 0.0;
+    for (const r of responses) {
+        if (r.error) {
+            continue;
+        }
+        if (!isBillableResponse(r)) {
+            continue;
+        }
+        const ce = estimate_cost(r.provider, r.model, r.input_tokens, r.output_tokens, table);
+        total += ce.input_usd + ce.output_usd;
+    }
+    return total;
+}
+
 // ── prompt-cache repricing ─────────────────────────────────────────
 //
 // Prompt-cache multipliers derived from the base input rate (Anthropic, GA —