@event4u/agent-config 14.12.0 → 14.14.0

package/src/scripts/ai_council/consensus_round.ts

@@ -0,0 +1,226 @@
+// The two-pass consensus round.
+//
+// Split out of `orchestrator.ts` in the change that made pass 1 conditional
+// (Phase 1B, inline findings). `orchestrator.ts` sits ~750 lines past the
+// 1500-line source-size ceiling, so the budget gate refuses new lines there —
+// correctly: the fix for a bloated file is not a shorter comment, it is the
+// concern moving out. This is the concern that moved, and it is the one the
+// change actually edits.
+//
+// `orchestrator.ts` re-exports `run_consensus_scoring`, so every existing
+// importer is untouched. The import of `consult` below closes a module cycle
+// with that file; it is call-time, not init-time — nothing here runs at module
+// load — which is the shape ESM resolves without a partial-initialization
+// hazard.
+import type { CouncilResponse, ExternalAIClient } from './clients.js';
+import { DEFAULT_MAX_TOKENS } from './clients.js';
+import type { Finding } from './consensus.js';
+import {
+    ConsensusBucket,
+    FindingScore,
+    aggregate_scores,
+    anonymize_findings,
+    bucket_by_threshold,
+    parse_findings_outcome,
+    parse_scores_response,
+} from './consensus.js';
+import type { RecordedExtractionOutcome } from './inline_findings.js';
+import { takeInlineFindings } from './inline_findings.js';
+import { consult } from './orchestrator.js';
+import type { RunConsensusScoringOptions } from './orchestrator_results.js';
+import { ConsensusResult, CouncilQuestion } from './orchestrator_results.js';
+import { build_extraction_user_prompt, build_scoring_user_prompt } from './prompts.js';
+
+/** Python `str.strip()` parity, as everywhere else in this package. */
+const _pyStrip = (s: string): string => s.replace(/^\s+|\s+$/g, '');
+
+/**
+ * Two-pass consensus round (Phase 4 / F3).
+ *
+ * Pass 1 — extraction: each member re-emits its own deliberation as
+ * a JSON array of `{id, text}` findings. Pass 2 — scoring: each
+ * member sees the *other* members' findings under anonymous labels
+ * and rates them 1-10 + agree/disagree + reason.
+ *
+ * The cost budget is shared across both passes; the daily ledger
+ * receives both. Errors in one member's extraction or scoring tag
+ * that member but never abort the round.
+ */
+export function run_consensus_scoring(
+    members: ExternalAIClient[],
+    deliberation_responses: CouncilResponse[],
+    opts: RunConsensusScoringOptions = {},
+): ConsensusResult {
+    const budget = opts.budget ?? null;
+    const table = opts.table ?? null;
+    const on_overrun = opts.on_overrun ?? null;
+    const project = opts.project ?? null;
+    const original_ask = opts.original_ask ?? '';
+    const max_tokens = opts.max_tokens ?? DEFAULT_MAX_TOKENS;
+    const strong_threshold = opts.strong_threshold ?? 0.7;
+    const minority_threshold = opts.minority_threshold ?? 0.4;
+    const inline_extractions = opts.inline_extractions ?? null;
+
+    if (members.length === 0 || deliberation_responses.length === 0) {
+        return new ConsensusResult({
+            bucket: new ConsensusBucket(),
+            findings: [],
+            scores: [],
+            metadata: new Map(),
+            extraction_responses: [],
+            scoring_responses: [],
+        });
+    }
+
+    // ── Pass 1: extraction ──────────────────────────────────────────
+    const member_by_name = new Map<string, ExternalAIClient>();
+    for (const m of members) {
+        member_by_name.set(m.name, m);
+    }
+    const extraction_responses: CouncilResponse[] = [];
+    const all_findings: Finding[] = [];
+    // Per-member extraction outcome — step 2.2. Recorded rather than derived from the
+    // findings count, because "zero findings" and "could not be read" are different facts
+    // and the count cannot tell them apart. `parsed-after-reask` is kept distinct from
+    // `parsed`: a member that needed a second ask is a signal about the prompt or the
+    // member, and collapsing it into `parsed` would hide the only evidence that the
+    // re-ask is doing anything.
+    const parse_outcomes = new Map<string, RecordedExtractionOutcome>();
+    for (const resp of deliberation_responses) {
+        const member = member_by_name.get(resp.provider);
+        if (member === undefined || resp.error || !_pyStrip(resp.text)) {
+            continue;
+        }
+        const source = `${member.name}:${member.model}`;
+
+        // Phase 1B replaces the second call, it does not delete it: no harvested
+        // block → fall through to the extraction call over untouched text.
+        if (takeInlineFindings(inline_extractions, source, parse_outcomes, all_findings)) {
+            continue;
+        }
+
+        const question = new CouncilQuestion({
+            mode: 'prompt',
+            user_prompt: build_extraction_user_prompt(resp.text),
+            max_tokens,
+        });
+        const extracted = consult([member], question, budget, {
+            table,
+            on_overrun,
+            project,
+            original_ask,
+        });
+        extraction_responses.push(...extracted);
+        if (extracted.length === 0 || extracted[0]!.error) {
+            continue;
+        }
+        let ex = parse_findings_outcome(extracted[0]!.text, { source });
+
+        // Step 2.2 — ONE bounded re-ask on `parse_failed`, and one only.
+        //
+        // A member that said something no parser could read used to be indistinguishable
+        // from a member that found nothing: `parse_findings_response` returned `[]` for
+        // both, and this loop pushed the empty array with no branch. So an unparseable
+        // answer counted as a clean zero-findings review.
+        //
+        // ONE re-ask, not a loop. A re-ask is a paid call, and an unbounded retry turns
+        // one unparseable answer into open-ended spend — the failure mode the N=3
+        // validation budget exists for, reached here by a different road. `empty` is NOT
+        // re-asked: a member that said nothing has nothing to restate, and re-asking it
+        // buys a second silence.
+        if (ex.outcome === 'parse_failed') {
+            const retry_q = new CouncilQuestion({
+                mode: 'prompt',
+                user_prompt:
+                    build_extraction_user_prompt(resp.text) +
+                    '\n\nYour previous answer could not be parsed. Reply with ONLY a JSON ' +
+                    'array of {"id": string, "text": string} objects, no prose before or ' +
+                    'after it. An empty array [] is a valid answer meaning you found nothing.',
+                max_tokens,
+            });
+            const retried = consult([member], retry_q, budget, {
+                table,
+                on_overrun,
+                project,
+                original_ask,
+            });
+            extraction_responses.push(...retried);
+            if (retried.length > 0 && !retried[0]!.error) {
+                ex = parse_findings_outcome(retried[0]!.text, { source });
+            }
+            parse_outcomes.set(source, ex.outcome === 'parsed' ? 'parsed-after-reask' : 'parse_failed');
+        } else {
+            parse_outcomes.set(source, ex.outcome);
+        }
+        all_findings.push(...ex.findings);
+    }
+
+    if (all_findings.length === 0) {
+        return new ConsensusResult({
+            bucket: new ConsensusBucket(),
+            findings: [],
+            scores: [],
+            metadata: new Map(),
+            parse_outcomes,
+            extraction_responses,
+            scoring_responses: [],
+        });
+    }
+
+    // ── Pass 2: scoring (each member rates the OTHERS' findings) ────
+    const scoring_responses: CouncilResponse[] = [];
+    const all_scores: FindingScore[] = [];
+    for (const member of members) {
+        const scorer = `${member.name}:${member.model}`;
+        const others = all_findings.filter((f) => f.source !== scorer);
+        if (others.length === 0) {
+            continue;
+        }
+        const anon = anonymize_findings(others);
+        const label_to_id = new Map<string, string>();
+        const anon_text = new Map<string, string>();
+        for (const [label, f] of anon) {
+            label_to_id.set(label, f.id);
+            anon_text.set(label, f.text);
+        }
+        const question = new CouncilQuestion({
+            mode: 'prompt',
+            user_prompt: build_scoring_user_prompt(anon_text),
+            max_tokens,
+        });
+        const scored = consult([member], question, budget, {
+            table,
+            on_overrun,
+            project,
+            original_ask,
+        });
+        scoring_responses.push(...scored);
+        if (scored.length === 0 || scored[0]!.error) {
+            continue;
+        }
+        for (const s of parse_scores_response(scored[0]!.text, { scorer })) {
+            const real_id = label_to_id.get(s.finding_id);
+            if (real_id === undefined) {
+                continue;
+            }
+            all_scores.push(
+                new FindingScore(real_id, s.scorer, s.score, s.agree, s.reason),
+            );
+        }
+    }
+
+    const metadata = aggregate_scores(all_findings, all_scores);
+    const bucket = bucket_by_threshold(all_findings, metadata, {
+        strong: strong_threshold,
+        minority: minority_threshold,
+    });
+    return new ConsensusResult({
+        bucket,
+        findings: all_findings,
+        scores: all_scores,
+        metadata,
+        parse_outcomes,
+        extraction_responses,
+        scoring_responses,
+    });
+}

package/src/scripts/ai_council/cross_exam.ts

@@ -0,0 +1,128 @@
+/**
+ * Targeted cross-examination — step 8.1.
+ *
+ * `road-to-inbox-harvest-2026-08-e-council-topology-evidence` step 8.1: *"Select
+ * a disputed finding, a conflicting pair, or a correct-looking minority claim,
+ * and ask focused rebuttal questions"*, verified by *"the cross-exam prompt
+ * names the exact disputed claim"*.
+ *
+ * ## "Names the exact claim" is a byte-level obligation, not a summary
+ *
+ * The failure this prevents is a composer that paraphrases: *"one reviewer
+ * questioned the index approach"* is a description of a dispute and gives the
+ * cross-examined model nothing to rebut. The prompt therefore carries the
+ * claim's **verbatim text**, unmodified and untruncated, and
+ * {@link crossExamNamesClaim} is the predicate that decides it — a substring
+ * check on the original string, which is the only form of "exact" that survives
+ * a later edit to the prompt's wording.
+ *
+ * ## The claim is untrusted, and is fenced
+ *
+ * A disputed claim is model output. It reaches the prompt through
+ * `wrapUntrustedBlocks` under a nonce, exactly as `build_peer_review_user_prompt`
+ * already does (`prompts.ts:932-942`) — a claim that could close its own fence
+ * could inject instructions into the cross-exam. Fencing does not modify the
+ * payload, which is what lets it coexist with the verbatim obligation above.
+ *
+ * ## Selection is deterministic and ordered, and the order is argued
+ *
+ * `conflicting-pair` outranks `disputed-finding` outranks `minority-claim`. A
+ * conflicting pair is the only kind where **at least one side is definitely
+ * wrong**, so the rebuttal has the highest information density; a lone disputed
+ * finding may resolve to "both are partly right"; a correct-looking minority
+ * claim is last because the majority may simply be right and the call buys the
+ * least. Ties break on the claim id, so selection is a pure function of input.
+ *
+ * Pure and offline: composes a prompt, never dispatches one.
+ */
+import { wrapUntrustedBlocks } from '../_lib/untrusted_content.js';
+
+/** What kind of thing is being cross-examined. Priority order, strongest first. */
+export const CROSS_EXAM_KINDS = ['conflicting-pair', 'disputed-finding', 'minority-claim'] as const;
+export type CrossExamKind = (typeof CROSS_EXAM_KINDS)[number];
+
+/** One candidate for cross-examination. */
+export interface DisputedClaim {
+    readonly id: string;
+    readonly kind: CrossExamKind;
+    /** Neutral label of whoever asserted it (`Response-A`), never a provider name. */
+    readonly assertedBy: string;
+    /** THE CLAIM, VERBATIM. Never summarised, never truncated. */
+    readonly claim: string;
+    /** Neutral label of whoever disputes it, when there is one. */
+    readonly disputedBy?: string;
+    /** The opposing text, verbatim, when there is one. */
+    readonly counterClaim?: string;
+}
+
+/**
+ * The focused questions. Fixed and shared across kinds on purpose: a per-kind
+ * question set is a second thing to keep in sync with the neutrality contract,
+ * and every question below is answerable about any of the three kinds.
+ */
+export const CROSS_EXAM_QUESTIONS: readonly string[] = [
+    'Is the claim above true as stated? Answer yes, no, or "true only under a condition" and name the condition.',
+    'What specific evidence would settle it? Name a file, a measurement, or a test — not a further opinion.',
+    'If it is false, what is the smallest correction that makes it true?',
+    'What would you have to believe for the opposing position to be correct?',
+];
+
+const PREAMBLE = `You are cross-examining ONE specific claim from a council deliberation.
+Do not review the artefact again, do not restate the debate, and do not
+comment on any claim other than the one quoted below. Your entire reply
+answers the numbered questions about that claim.`;
+
+/**
+ * Pick the claim to cross-examine, or `null`.
+ *
+ * Deterministic: kind priority first, then claim id.
+ */
+export function selectCrossExamTarget(candidates: readonly DisputedClaim[]): DisputedClaim | null {
+    if (candidates.length === 0) return null;
+    const ranked = [...candidates].sort((a, b) => {
+        const ka = CROSS_EXAM_KINDS.indexOf(a.kind);
+        const kb = CROSS_EXAM_KINDS.indexOf(b.kind);
+        return ka - kb || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+    });
+    return ranked[0] as DisputedClaim;
+}
+
+/** Compose the cross-exam prompt. The claim text is carried verbatim. */
+export function buildCrossExamPrompt(target: DisputedClaim, opts: { readonly nonce?: string } = {}): string {
+    const blocks = [{ heading: `### The claim under examination (asserted by ${target.assertedBy})`, content: target.claim }];
+    if (target.counterClaim !== undefined) {
+        blocks.push({
+            heading: `### The opposing position (asserted by ${target.disputedBy ?? 'another reviewer'})`,
+            content: target.counterClaim,
+        });
+    }
+    const fenced = wrapUntrustedBlocks(blocks, opts.nonce === undefined ? {} : { nonce: opts.nonce });
+    const questions = CROSS_EXAM_QUESTIONS.map((q, i) => `${String(i + 1)}. ${q}`).join('\n');
+    return [
+        PREAMBLE,
+        '',
+        `Kind: ${target.kind} · claim id: ${target.id}`,
+        '',
+        fenced,
+        '',
+        '---',
+        '',
+        'Answer these, in order:',
+        '',
+        questions,
+        '',
+    ].join('\n');
+}
+
+/**
+ * Does this prompt name the exact claim?
+ *
+ * A plain substring check on the ORIGINAL string. Deliberately not a similarity
+ * score: "exact" that tolerates a threshold is not exact, and the paraphrase is
+ * the failure mode. Both sides of a pair must be present when a pair was given.
+ */
+export function crossExamNamesClaim(prompt: string, target: DisputedClaim): boolean {
+    if (!prompt.includes(target.claim)) return false;
+    if (target.counterClaim !== undefined && !prompt.includes(target.counterClaim)) return false;
+    return true;
+}