@event4u/agent-config 14.11.0 → 14.13.0

package/src/scripts/ai_council/inline_findings.ts

@@ -0,0 +1,340 @@
+// Phase 1B — inline findings, the whole concern in one module.
+//
+// The consensus round's pass 1 asks every member a SECOND time, in a separate
+// paid call, to restate its own analysis as a JSON findings array. Phase 1B of
+// `road-to-inbox-harvest-2026-08-e-council-topology-evidence` asks for that
+// array in the FIRST reply instead, and keeps the extraction call as the
+// repair path.
+//
+// It lives here rather than spread across `orchestrator.ts`, `consensus.ts` and
+// `council_cli.ts` for a reason the source-size budget made concrete: all three
+// are far past the 1500-line ceiling, so a new concern added to them is new
+// budget excess with nowhere to go. Concentrating it also puts the four pieces
+// that have to agree — the contract text's promise, the locator, the harvest
+// and the config predicate — where a reader meets them together.
+import type { ExternalAIClient, CouncilResponse } from './clients.js';
+import type { Finding, FindingsExtraction } from './consensus.js';
+import { _BARE_ARRAY_SRC, _JSON_BLOCK_SRC, parse_findings_outcome } from './consensus.js';
+import { INLINE_FINDINGS_CONTRACT, STANCE_LINE_CONTRACT } from './prompts.js';
+
+/** Python `str.strip()` parity, as everywhere else in this package. */
+const _strip = (s: string): string => s.replace(/^\s+|\s+$/g, '');
+
+/**
+ * Phase 1B — the deliberation reply split into argument and findings block.
+ *
+ * `deliberation_text` is what every downstream consumer should see: peer
+ * review, chairman synthesis, and the rendered artifact evaluate the member's
+ * REASONING, and a schema block restating selected conclusions is scaffolding,
+ * not argument. Leaving it in amplifies a concise finding simply because it
+ * appears twice — the AI council's stated reason (2026-08-30, 2 of 2 seats) for
+ * choosing to strip.
+ *
+ * `block` is the exact consumed span. One locator produces both halves, so the
+ * text that is parsed and the text that is removed cannot disagree — the
+ * council's other requirement, and the reason there is no separate
+ * "find and remove a JSON fence" function anywhere in this change.
+ */
+export interface InlineFindingsSplit {
+    /** The reply with the consumed block removed. Equals the input when `found` is false. */
+    readonly deliberation_text: string;
+    /** The consumed span, ready for `parse_findings_outcome`. `''` when `found` is false. */
+    readonly block: string;
+    readonly found: boolean;
+}
+
+/**
+ * Split a full analysis reply at its TRAILING findings block.
+ *
+ * A locator, deliberately not a second parser: `parse_findings_outcome` stays
+ * the only thing that decides whether a candidate IS a findings block, and this
+ * narrows where it looks.
+ *
+ * The narrowing is not cosmetic. `_extract_json_array` returns the FIRST match,
+ * which is correct for an extraction reply (the reply IS the array) and wrong
+ * for an analysis reply, where the prose above may legitimately quote a JSON
+ * array — the analysis lens exists to critique analyser OUTPUT, so a quoted
+ * array is an ordinary thing to find there. First-match would read the quoted
+ * array as the member's findings and never reach the real block.
+ *
+ * `INLINE_FINDINGS_CONTRACT` asks for the block LAST, so LAST is what this
+ * reads. `found: false` when nothing array-shaped is present, which the caller
+ * treats as "no inline block" and repairs with the extraction call — the same
+ * outcome as today, never worse.
+ *
+ * **Precisely: "last FENCED, else last bare" — not "last in the text."** The
+ * two patterns are tried in `_extract_json_array`'s precedence order and the
+ * first pattern that matches at all wins, so a reply carrying a quoted FENCED
+ * array above its own trailing BARE array selects the quoted one. Stated
+ * because the docstring used to claim the simpler rule, which a completion
+ * review of 2026-08-30 found untrue against that ordering; the precedence test
+ * puts the bare array first and so never exercises the collision.
+ *
+ * The consequence is bounded rather than eliminated. `_isOwnFindingsBlock`
+ * below now rejects any array whose items are not all well-formed findings, so
+ * the common shape of a quoted array — evidence rows, log lines, a schema
+ * fragment — no longer wins. What survives is a quoted array of well-formed
+ * findings sitting fenced above the member's own bare one, and that is recorded
+ * as a known residual of 1B.4's gate rather than claimed away.
+ */
+export function split_inline_findings(text: string): InlineFindingsSplit {
+    const none: InlineFindingsSplit = { deliberation_text: text, block: '', found: false };
+    if (!text) {
+        return none;
+    }
+    // Deliberately the same two patterns `_extract_json_array` uses, in the same
+    // precedence order — fenced beats bare — so a block this locator returns is
+    // one that function can also read. Only the match SELECTION differs: last,
+    // not first.
+    for (const src of [_JSON_BLOCK_SRC, _BARE_ARRAY_SRC]) {
+        const re = new RegExp(src, 'gs');
+        let last: RegExpExecArray | null = null;
+        let m: RegExpExecArray | null;
+        while ((m = re.exec(text)) !== null) {
+            last = m;
+            // A zero-length match would spin forever; the patterns cannot produce
+            // one (both require at least `[]`), but the guard costs nothing and a
+            // future pattern edit is exactly when it would matter.
+            if (m.index === re.lastIndex) {
+                re.lastIndex += 1;
+            }
+        }
+        if (last) {
+            const start = last.index;
+            const end = start + last[0].length;
+            return {
+                deliberation_text: `${text.slice(0, start)}${text.slice(end)}`,
+                block: text.slice(start, end),
+                found: true,
+            };
+        }
+    }
+    return none;
+}
+
+/**
+ * How a member's findings were obtained — the RECORDED provenance, wider than
+ * `FindingsParseOutcome`, which describes one parse of one piece of text.
+ *
+ * Typed rather than a bare string because Phase 1B's promotion gate is computed
+ * off exactly these values: an untyped `Map<string, string>` makes the number
+ * that decides the experiment vulnerable to a misspelling no compiler would
+ * catch. The AI council (2026-08-30, 2 of 2 seats) called the pre-existing bare
+ * `'parsed-after-reask'` technical debt rather than a precedent to extend.
+ *
+ * `'parsed'` keeps its shipped meaning — read from the separate extraction
+ * call — and is unambiguous now that `'parsed-inline'` names the other source.
+ * One seat asked for a rename to `'parsed-extraction'`; renaming a value that
+ * is already recorded and compared against is a separate change with its own
+ * migration, and is recorded here as declined-with-a-reason, not overlooked.
+ */
+export type RecordedExtractionOutcome =
+    | 'parsed'
+    | 'empty'
+    | 'parse_failed'
+    | 'parsed-after-reask'
+    | 'parsed-inline';
+
+/**
+ * Phase 1B — harvest the inline findings block out of each deliberation reply.
+ *
+ * Runs BETWEEN the deliberation and every consumer of its text. That placement
+ * is the load-bearing part and it was a defect the AI council caught
+ * (2026-08-30, 2 of 2 seats): peer review and chairman synthesis read the
+ * responses BEFORE the consensus round does, so parsing inside
+ * `run_consensus_scoring` would have left the schema block in the text those
+ * two evaluate — which is precisely the amplification the strip exists to
+ * prevent.
+ *
+ * MUTATES `text` on each response whose block parsed, replacing the consumed
+ * span with a one-line marker. Mutation rather than a parallel clean copy
+ * because two representations of one response means every downstream consumer
+ * has to know which one it holds, and the one that forgets is a silent bug.
+ * `CouncilResponse.text` is already mutated on the stance-repair path, so this
+ * is the established seam rather than a new one.
+ *
+ * The marker is deliberately visible. Stripping without an observable trace is
+ * a silent mutation of the artifact a reader takes for a transcript, and one
+ * seat made an observable marker a condition of its verdict; the other reached
+ * the same concern from the auditability side. ~90 characters replaces ~300 of
+ * JSON, so the scaffolding is gone and the fact of its removal is not.
+ *
+ * Returns the per-member extraction keyed by `provider:model` — the same key
+ * `run_consensus_scoring` records outcomes under. A member whose block did not
+ * parse gets NO entry and its `text` is left exactly as the model wrote it, so
+ * the repair extraction call downstream still sees the raw reply.
+ */
+/**
+ * Is this extraction the member's OWN findings block, or just some JSON it quoted?
+ *
+ * `outcome === 'parsed'` was the whole test until a completion review of
+ * 2026-08-30 showed it decides nothing: `parse_findings_outcome` returns
+ * `'parsed'` for ANY syntactically valid JSON array, and `_collect_findings`
+ * silently skips items missing `id` or `text` while the outcome stays `'parsed'`.
+ * So a reply carrying no findings block at all, but quoting
+ * `[{"file":"src/a.ts","line":42}]` as evidence, was recorded as a successful
+ * inline extraction with zero findings — and its quoted evidence was DELETED
+ * from the text the peer review, the chairman and the artifact read. Worse, the
+ * failure was recorded as a success in the rate the 1B.4 promotion gate reads,
+ * so the gate could not see it.
+ *
+ * Two conjuncts, and both are needed:
+ *
+ * - **it parsed** — unchanged;
+ * - **every item it saw became a finding** — `findings.length === item_count`.
+ *   A findings block from a member following the contract has well-formed items
+ *   throughout. An array where some or all items lack `id`/`text` is something
+ *   else, whatever it parsed to.
+ *
+ * The bare-`[]` answer keeps working: `item_count` is 0 and `findings.length`
+ * is 0, so the conjunction holds and "I found nothing" is still an answer
+ * rather than a re-ask.
+ *
+ * **What this does NOT decide, stated rather than implied away.** A member that
+ * quotes ANOTHER member's well-formed `[{"id":…,"text":…}]` array, as the last
+ * block in its reply, is indistinguishable from its own by shape — and would
+ * still be attributed to the quoting member. Shape cannot separate those; only
+ * provenance could, and the reply carries none. It is narrower than the case
+ * fixed here (the quote must be well-formed AND last AND the member must have
+ * emitted no block of its own), it is visible in the emitted marker, and it is
+ * carried into 1B.4's gate as a known residual rather than counted as a success.
+ */
+function _isOwnFindingsBlock(ex: {
+    outcome: string;
+    findings: readonly unknown[];
+    item_count: number;
+}): boolean {
+    return ex.outcome === 'parsed' && ex.findings.length === ex.item_count;
+}
+
+export function harvest_inline_findings(
+    members: readonly ExternalAIClient[],
+    responses: readonly CouncilResponse[],
+): Map<string, FindingsExtraction> {
+    const out = new Map<string, FindingsExtraction>();
+    const model_by_name = new Map<string, string>();
+    for (const m of members) {
+        model_by_name.set(m.name, m.model);
+    }
+    for (const resp of responses) {
+        const model = model_by_name.get(resp.provider);
+        if (model === undefined || resp.error || !_strip(resp.text)) {
+            continue;
+        }
+        const split = split_inline_findings(resp.text);
+        if (!split.found) {
+            continue;
+        }
+        const source = `${resp.provider}:${model}`;
+        const ex = parse_findings_outcome(split.block, { source });
+        if (!_isOwnFindingsBlock(ex)) {
+            // Not a findings block after all — most likely a JSON array the member
+            // quoted from the artifact. Leave the reply untouched: stripping text
+            // we could not read would remove evidence and buy nothing.
+            continue;
+        }
+        out.set(source, ex);
+        // Retain the reply BEFORE rewriting it — this is the single write of
+        // `CouncilResponse.raw_text`, and the reason that field exists.
+        //
+        // The marker appended below promises "the raw reply is retained in the
+        // session record". Until 2026-08-30 that promise was false, and a
+        // completion review is what caught it: the persisted record is
+        // serialized from these same objects AFTER this harvest runs
+        // (`council_cli._serialise_responses`), so the stripped text was the
+        // only text that survived anywhere, and the auditability condition the
+        // council attached to this feature was unmet.
+        //
+        // `null` stays the honest value for a reply nothing rewrote — it means
+        // "`text` IS the raw reply", never "the raw reply was lost" — which is
+        // also why the serializer omits the key entirely in that case rather
+        // than writing a duplicate of the reply into every session record.
+        resp.raw_text = resp.text;
+        const marker = `_[inline findings block extracted: ${String(ex.findings.length)} item(s); the raw reply is retained in the session record.]_`;
+        resp.text = `${_strip(split.deliberation_text)}\n\n${marker}`;
+    }
+    return out;
+}
+
+
+/**
+ * The FINAL deliberation round's volatile suffix.
+ *
+ * Two opt-in contracts ride it and their ORDER is load-bearing:
+ * `STANCE_LINE_CONTRACT` requires the stance to be the last line of the reply,
+ * so it is appended last and the findings block sits above it. A findings block
+ * appended after the stance line would contradict the instruction the member is
+ * reading, which is the one composition bug this function exists to make
+ * impossible to reintroduce.
+ *
+ * Both off → the caller's suffix is returned unchanged, so the composed default
+ * is byte-identical to a run without either feature.
+ */
+export function composeFinalRoundSuffix(
+    suffix: string,
+    opts: { is_final: boolean; inline_findings: boolean; stance_tally: boolean },
+): string {
+    if (!opts.is_final) {
+        return suffix;
+    }
+    let out = suffix;
+    if (opts.inline_findings) {
+        out = `${out}\n\n---\n\n${INLINE_FINDINGS_CONTRACT}`;
+    }
+    if (opts.stance_tally) {
+        out = `${out}\n\n---\n\n${STANCE_LINE_CONTRACT}`;
+    }
+    return out;
+}
+
+/**
+ * Consume a member's harvested findings, or report that it has none.
+ *
+ * Returns `true` when the caller should SKIP its extraction call. An entry in
+ * the map is always a short-circuit and never a partial one, because
+ * `harvest_inline_findings` records only a member whose block PARSED — a member
+ * with no entry had its reply left raw and takes the shipped extraction path
+ * over the full text.
+ */
+export function takeInlineFindings(
+    inline_extractions: ReadonlyMap<string, FindingsExtraction> | null,
+    source: string,
+    parse_outcomes: Map<string, RecordedExtractionOutcome>,
+    all_findings: Finding[],
+): boolean {
+    const inline = inline_extractions?.get(source);
+    if (inline === undefined) {
+        return false;
+    }
+    parse_outcomes.set(source, 'parsed-inline');
+    all_findings.push(...inline.findings);
+    return true;
+}
+
+/**
+ * Is the inline-findings contract active for this run?
+ *
+ * Read BEFORE the deliberation, because the contract is a prompt change and the
+ * prompt is built first; the harvest reads the same predicate so the two cannot
+ * disagree about whether a member was asked for the block. Three conjuncts, all
+ * required: consensus scoring on, this lens in its lens list, and
+ * `inline_findings` explicitly on. Any missing → `false`, and `false` is a
+ * byte-identical prompt.
+ *
+ * Reads the SYNTHESISED settings dict, not the typed config. That distinction
+ * is not academic: `_lib/council_settings_block.ts` projects the typed config
+ * into this dict, and when it did not carry the key a `true` in the YAML
+ * resolved to `undefined` and the feature was silently off with every unit test
+ * green — found by a live analysis run on 2026-08-30, not by review.
+ */
+export function inlineFindingsActive(
+    ai_cfg: Record<string, unknown>,
+    mode: string,
+): boolean {
+    const cs = (ai_cfg['consensus_scoring'] as Record<string, unknown> | undefined) ?? {};
+    if (cs['enabled'] !== true || cs['inline_findings'] !== true) {
+        return false;
+    }
+    const lenses = (cs['lenses'] as string[] | undefined) ?? ['analysis'];
+    return lenses.includes(mode);
+}

package/src/scripts/ai_council/judge_position_bias.ts

@@ -0,0 +1,197 @@
+/**
+ * Per-judge position-consistency — order-swap over sampled pairwise judgments.
+ *
+ * `road-to-inbox-harvest-2026-08-e-council-topology-evidence` Phase 3, step 3.5:
+ * "repeat sampled pairwise judgments with candidate order reversed; emit a
+ * per-judge position-consistency metric", verified by "the metric exists per
+ * judge and is reported with the verdict".
+ *
+ * ── What already existed, and why this is not a second copy of it ────────────
+ * `check_quality_regression.evaluatePair` (`:84-108`) already judges every pair
+ * in both orders and resolves a flip to `inconsistent` rather than to a winner,
+ * and `_lib/judge_hygiene.ts:5-9` records that as stronger than the step that
+ * asked for it. Two things it does not do, and both are the point here:
+ *
+ *   1. It is SINGLE-JUDGE. `aggregate` returns one run-wide `inconsistency_rate`
+ *      (`:216`), so two judges of differing reliability average into one number
+ *      that describes neither. The step asks for the metric PER JUDGE, which is
+ *      what makes it actionable: you can drop the flipper.
+ *   2. It reports presence, not DIRECTION. A judge that flips at random and a
+ *      judge that always prefers whatever it was shown first produce the same
+ *      inconsistency rate and need opposite remedies — the first is noise, the
+ *      second is a systematic bias that survives averaging and quietly decides
+ *      verdicts. `first_position_rate` separates them.
+ *
+ * That module is the thin-vs-eager token gate and is not council code; this one
+ * is council-side and stays a pure library — no transport, no fixture path, no
+ * `main`. The judge is an injected function, so the whole surface is testable
+ * against a scripted judge and costs nothing to exercise.
+ *
+ * ── Honest scope ────────────────────────────────────────────────────────────
+ * The shipped council has NO pairwise judging stage (`grep -rn pairwise
+ * src/scripts/ai_council` returns nothing): members deliberate, peer-review and
+ * are scored on findings, never compared two at a time. So this module is the
+ * metric and its renderer, exercised by the provider-leakage bench that Phase 3
+ * builds; it does not claim a live council verdict currently carries the line,
+ * because there is currently no live council judgment for it to describe.
+ */
+
+import { _sha256_hex } from './blind_review.js';
+
+/** A judge's preference between the two PRESENTED candidates — order-relative. */
+export type PairwiseVerdict = 'first' | 'second' | 'tie';
+
+/**
+ * The pluggable judge. `first`/`second` are the candidate bodies in PRESENTED
+ * order, so a judge implementation cannot tell which arm it is looking at
+ * unless the bodies themselves leak it.
+ */
+export type PairwiseJudge = (ctx: { readonly id: string }, first: string, second: string) => PairwiseVerdict;
+
+/** How a pair resolved once both orders were seen. */
+export type PairResolution = 'a' | 'b' | 'tie' | 'inconsistent';
+
+export interface SwapObservation {
+    readonly id: string;
+    /** Verdict with A presented first. */
+    readonly forward: PairwiseVerdict;
+    /** Verdict with B presented first. */
+    readonly reverse: PairwiseVerdict;
+    readonly resolution: PairResolution;
+}
+
+/**
+ * Judge one (a, b) pair in BOTH orders.
+ *
+ * A verdict that does not survive the swap resolves to `inconsistent`, never to
+ * a winner — a flagged tie would still be a tie in somebody's denominator,
+ * while `inconsistent` is its own bucket with its own reported rate.
+ */
+export function judgeBothOrders(id: string, a: string, b: string, judge: PairwiseJudge): SwapObservation {
+    const ctx = { id };
+    const forward = judge(ctx, a, b);
+    const reverse = judge(ctx, b, a);
+    // Normalise each verdict to the CANDIDATE it names, not the position.
+    const wf: PairResolution = forward === 'first' ? 'a' : forward === 'second' ? 'b' : 'tie';
+    const wr: PairResolution = reverse === 'first' ? 'b' : reverse === 'second' ? 'a' : 'tie';
+    const resolution: PairResolution = wf === wr ? wf : 'inconsistent';
+    return { id, forward, reverse, resolution };
+}
+
+export interface JudgeConsistency {
+    readonly judge: string;
+    /** Pairs actually repeated in both orders. */
+    readonly sampled: number;
+    /** Both orders named the same candidate, or both said tie. */
+    readonly consistent: number;
+    /** The two orders disagreed. */
+    readonly inconsistent: number;
+    /**
+     * `consistent / sampled`. `null` at zero pairs — a rate over an empty
+     * denominator is not 1.0, and reporting it as 1.0 would make an unrun swap
+     * indistinguishable from a perfectly stable judge.
+     */
+    readonly position_consistency: number | null;
+    /**
+     * Share of DECISIVE verdicts (`first`/`second`, both orders pooled) that
+     * named whichever candidate was shown FIRST. ~0.5 is unbiased; near 1.0 is
+     * primacy, near 0.0 recency. `null` when nothing decisive was said.
+     *
+     * This is the direction an inconsistency rate cannot show, and the reason a
+     * judge that flips at random is a different problem from one that always
+     * prefers the top of the list.
+     */
+    readonly first_position_rate: number | null;
+    /** Decisive verdicts counted for `first_position_rate`. */
+    readonly decisive_verdicts: number;
+}
+
+/** Per-judge metric over that judge's own swapped observations. */
+export function positionConsistency(judge: string, obs: readonly SwapObservation[]): JudgeConsistency {
+    let consistent = 0;
+    let inconsistent = 0;
+    let firstWins = 0;
+    let decisive = 0;
+    for (const o of obs) {
+        if (o.resolution === 'inconsistent') {
+            inconsistent += 1;
+        } else {
+            consistent += 1;
+        }
+        for (const v of [o.forward, o.reverse]) {
+            if (v === 'first') {
+                decisive += 1;
+                firstWins += 1;
+            } else if (v === 'second') {
+                decisive += 1;
+            }
+        }
+    }
+    const sampled = obs.length;
+    return {
+        judge,
+        sampled,
+        consistent,
+        inconsistent,
+        position_consistency: sampled > 0 ? consistent / sampled : null,
+        first_position_rate: decisive > 0 ? firstWins / decisive : null,
+        decisive_verdicts: decisive,
+    };
+}
+
+/**
+ * Deterministic sample of `ids` at `rate`, seeded by `seed`.
+ *
+ * The step says "SAMPLED pairwise judgments", because the swap doubles judge
+ * calls and doubling every one of them to measure the judge is a cost the
+ * measurement does not need. Deterministic so a published consistency figure is
+ * reproducible from the report's own seed — a sample nobody can re-draw is a
+ * number nobody can check.
+ *
+ * `rate <= 0` samples nothing; `rate >= 1` samples everything.
+ */
+export function sampleForSwap(ids: readonly string[], rate: number, seed: string): string[] {
+    if (!(rate > 0)) {
+        return [];
+    }
+    if (rate >= 1) {
+        return [...ids];
+    }
+    return ids.filter((id) => _unitHash(`${seed} ${id}`) < rate);
+}
+
+/**
+ * Stable [0,1) hash of a string.
+ *
+ * Reuses `blind_review._sha256_hex` rather than carrying a local one. That
+ * export exists for exactly this reason — `blind_review.ts:38-44` records that
+ * two hashes of one input are two answers to one question and the second is the
+ * one nobody updates. A hand-rolled FNV-1a was tried here first and its low bits
+ * were skewed enough that a rate of 0.25 sampled half the corpus; the test that
+ * caught it is kept.
+ */
+function _unitHash(s: string): number {
+    return parseInt(_sha256_hex(s).slice(0, 8), 16) / 4294967296;
+}
+
+/**
+ * One report line per judge, for emission BESIDE a verdict.
+ *
+ * `n/a` rather than a number when nothing was sampled: the step's verify is
+ * that the metric is reported, and a report that silently prints nothing for an
+ * unrun swap satisfies it only by looking like it ran.
+ */
+export function renderPositionConsistency(rows: readonly JudgeConsistency[]): string {
+    if (rows.length === 0) {
+        return 'position consistency: no judge sampled — order-swap NOT RUN.';
+    }
+    const pct = (v: number | null): string => (v === null ? 'n/a' : `${(v * 100).toFixed(0)}%`);
+    return rows
+        .map(
+            (r) =>
+                `position consistency · ${r.judge}: ${pct(r.position_consistency)} ` +
+                `(${String(r.consistent)}/${String(r.sampled)} pairs survived the swap; ` +
+                `first-position rate ${pct(r.first_position_rate)} over ${String(r.decisive_verdicts)} decisive verdicts)`,
+        )
+        .join('\n');
+}