@event4u/agent-config 2.12.0 → 2.14.0

package/scripts/ai_council/consensus.py

@@ -0,0 +1,329 @@
+"""Consensus scoring for the analysis lens (Phase 4 / F3).
+
+After the final deliberation round, members score each other's
+findings. The renderer ranks findings by consensus and surfaces a
+"Minority Views" section for sub-threshold items so they remain
+audit-trail signal rather than silent drop.
+
+Schema (Opus's machine-readable contract):
+
+    Finding            — `{id: str, source: str, text: str}`
+    FindingScore       — `{finding_id: str, scorer: str, score: 1..10,
+                          agree: bool, reason: str}`
+    ConsensusMetadata  — per-finding aggregate:
+                         `{finding_id, consensus_strength: 0..1,
+                           dissent_count, scorers, mean_score}`
+
+Threshold bucketing (Phase 4 Step 3):
+
+    consensus_strength > strong   → Strong Consensus
+    minority < strength <= strong → Findings (default body)
+    strength <= minority          → Minority Views
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+
+_JSON_BLOCK = re.compile(r"```(?:json)?\s*(\[.*?\])\s*```", re.DOTALL)
+_BARE_ARRAY = re.compile(r"(\[\s*\{.*?\}\s*\])", re.DOTALL)
+
+# Defaults mirror the roadmap (Phase 4 Step 4). The .agent-settings.yml
+# block overrides them at run time.
+DEFAULT_STRONG_THRESHOLD: float = 0.7
+DEFAULT_MINORITY_THRESHOLD: float = 0.4
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One finding extracted from a member's deliberation output."""
+
+    id: str
+    source: str  # provider/model that authored the finding
+    text: str
+
+
+@dataclass(frozen=True)
+class FindingScore:
+    """One scorer's vote on one finding."""
+
+    finding_id: str
+    scorer: str
+    score: int  # 1..10
+    agree: bool
+    reason: str
+
+
+def evidence_quality(mean_score: float) -> str:
+    """Classify mean score into a single-letter evidence-quality bucket.
+
+    H (high)   — mean ≥ 8.0; member agreement ran high.
+    M (medium) — 6.0 ≤ mean < 8.0; majority support, mixed conviction.
+    L (low)    — mean < 6.0 or no scorers; weak or contested.
+
+    Used by Phase 9 to surface a quick "how much did members back this"
+    signal next to the raw consensus_strength number.
+    """
+    if mean_score >= 8.0:
+        return "H"
+    if mean_score >= 6.0:
+        return "M"
+    return "L"
+
+
+@dataclass(frozen=True)
+class ConsensusMetadata:
+    """Aggregate consensus stats for a single finding.
+
+    Phase 9 adds ``concur_count``, ``dissent_reasons`` (per-scorer
+    one-line rationales for disagreement), and ``evidence_quality``
+    (H/M/L bucket of the mean score) so the renderer can emit
+    "N/M members concur; X dissented citing …; mean evidence-quality H"
+    without needing the underlying FindingScore list.
+    """
+
+    finding_id: str
+    consensus_strength: float  # 0..1
+    dissent_count: int
+    scorers: tuple[str, ...]
+    mean_score: float
+    concur_count: int = 0
+    dissent_reasons: tuple[tuple[str, str], ...] = ()  # (scorer, reason)
+    evidence_quality: str = "L"
+
+
+@dataclass(frozen=True)
+class ConsensusBucket:
+    """Threshold-bucketed findings ready for renderer sectioning."""
+
+    strong: list[tuple[Finding, ConsensusMetadata]] = field(default_factory=list)
+    findings: list[tuple[Finding, ConsensusMetadata]] = field(default_factory=list)
+    minority: list[tuple[Finding, ConsensusMetadata]] = field(default_factory=list)
+
+
+def aggregate_scores(
+    findings: Iterable[Finding],
+    scores: Iterable[FindingScore],
+) -> dict[str, ConsensusMetadata]:
+    """Aggregate per-finding scores into ConsensusMetadata.
+
+    `consensus_strength` = mean(score) / 10 * agreement_rate.
+
+    A finding's *own author* is never expected to score it; we drop
+    self-scores defensively to keep the aggregate honest. Missing
+    findings get zero scorers (strength=0, dissent_count=0).
+    """
+    by_id: dict[str, list[FindingScore]] = {f.id: [] for f in findings}
+    sources: dict[str, str] = {f.id: f.source for f in findings}
+    for s in scores:
+        if s.finding_id not in by_id:
+            continue
+        if s.scorer == sources[s.finding_id]:
+            continue  # ignore self-scores
+        by_id[s.finding_id].append(s)
+    out: dict[str, ConsensusMetadata] = {}
+    for fid, fs in by_id.items():
+        if not fs:
+            out[fid] = ConsensusMetadata(
+                finding_id=fid, consensus_strength=0.0,
+                dissent_count=0, scorers=(), mean_score=0.0,
+                concur_count=0, dissent_reasons=(), evidence_quality="L",
+            )
+            continue
+        mean = sum(s.score for s in fs) / len(fs)
+        agree_rate = sum(1 for s in fs if s.agree) / len(fs)
+        strength = (mean / 10.0) * agree_rate
+        dissent = sum(1 for s in fs if not s.agree)
+        concur = sum(1 for s in fs if s.agree)
+        scorers = tuple(s.scorer for s in fs)
+        # Phase 9 — collect (scorer, reason) pairs for dissenters only,
+        # in scoring order, so the renderer surfaces who pushed back
+        # and why without re-walking the FindingScore list.
+        dissent_reasons = tuple(
+            (s.scorer, s.reason) for s in fs if not s.agree
+        )
+        mean_rounded = round(mean, 2)
+        out[fid] = ConsensusMetadata(
+            finding_id=fid, consensus_strength=round(strength, 3),
+            dissent_count=dissent, scorers=scorers,
+            mean_score=mean_rounded,
+            concur_count=concur, dissent_reasons=dissent_reasons,
+            evidence_quality=evidence_quality(mean_rounded),
+        )
+    return out
+
+
+def bucket_by_threshold(
+    findings: Iterable[Finding],
+    metadata: dict[str, ConsensusMetadata],
+    *,
+    strong: float = DEFAULT_STRONG_THRESHOLD,
+    minority: float = DEFAULT_MINORITY_THRESHOLD,
+) -> ConsensusBucket:
+    """Split findings into Strong / Findings / Minority buckets.
+
+    `strong` and `minority` are the thresholds from
+    `.agent-settings.yml::ai_council.consensus_threshold_*`. Findings
+    with no metadata (no scorers) fall into the Minority bucket — they
+    were uncontested but unsupported.
+    """
+    if not 0.0 <= minority <= strong <= 1.0:
+        raise ValueError(
+            f"Threshold ordering broken: 0 <= {minority} <= {strong} <= 1 required.",
+        )
+    bucket = ConsensusBucket()
+    for f in findings:
+        m = metadata.get(f.id)
+        if m is None:
+            m = ConsensusMetadata(
+                finding_id=f.id, consensus_strength=0.0,
+                dissent_count=0, scorers=(), mean_score=0.0,
+                concur_count=0, dissent_reasons=(), evidence_quality="L",
+            )
+        if m.consensus_strength > strong:
+            bucket.strong.append((f, m))
+        elif m.consensus_strength > minority:
+            bucket.findings.append((f, m))
+        else:
+            bucket.minority.append((f, m))
+    # Strongest first inside each bucket.
+    for lst in (bucket.strong, bucket.findings, bucket.minority):
+        lst.sort(key=lambda pair: pair[1].consensus_strength, reverse=True)
+    return bucket
+
+
+def parse_findings_response(text: str, *, source: str) -> list[Finding]:
+    """Parse a member's structured-findings response into Finding objects.
+
+    Accepts either a fenced ```json``` block or a bare JSON array. Each
+    item must be `{id: str, text: str}` (the `source` is set from the
+    `source` arg so we can attribute findings to their author). Items
+    missing required keys are skipped silently — extraction is best-
+    effort, never raises.
+    """
+    array = _extract_json_array(text)
+    if not array:
+        return []
+    try:
+        parsed = json.loads(array)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(parsed, list):
+        return []
+    out: list[Finding] = []
+    for item in parsed:
+        if not isinstance(item, dict):
+            continue
+        fid = item.get("id")
+        txt = item.get("text")
+        if not fid or not txt:
+            continue
+        out.append(Finding(id=str(fid), source=source, text=str(txt).strip()))
+    return out
+
+
+def parse_scores_response(text: str, *, scorer: str) -> list[FindingScore]:
+    """Parse a member's scoring response into FindingScore objects.
+
+    Each item must be `{finding_id, score, agree, reason}`. Scores are
+    clamped to 1..10; non-numeric scores or out-of-range values cause
+    the item to be skipped (defensive — never poison aggregates).
+    """
+    array = _extract_json_array(text)
+    if not array:
+        return []
+    try:
+        parsed = json.loads(array)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(parsed, list):
+        return []
+    out: list[FindingScore] = []
+    for item in parsed:
+        if not isinstance(item, dict):
+            continue
+        fid = item.get("finding_id") or item.get("id")
+        score = item.get("score")
+        if not fid or not isinstance(score, (int, float)):
+            continue
+        score_int = int(score)
+        if not 1 <= score_int <= 10:
+            continue
+        out.append(FindingScore(
+            finding_id=str(fid), scorer=scorer, score=score_int,
+            agree=bool(item.get("agree", True)),
+            reason=str(item.get("reason", "")).strip(),
+        ))
+    return out
+
+
+def _extract_json_array(text: str) -> str:
+    """Best-effort JSON-array extraction from a model response."""
+    if not text:
+        return ""
+    fenced = _JSON_BLOCK.search(text)
+    if fenced:
+        return fenced.group(1)
+    bare = _BARE_ARRAY.search(text)
+    if bare:
+        return bare.group(1)
+    return ""
+
+
+def anonymize_findings(findings: list[Finding]) -> dict[str, Finding]:
+    """Return `{anon_label: Finding}` map so scorers see neutral labels.
+
+    Labels are `Finding-A`, `Finding-B`, … in input order. The author
+    mapping must be kept out of the prompt — keep it server-side only.
+    """
+    out: dict[str, Finding] = {}
+    for idx, f in enumerate(findings):
+        label = f"Finding-{chr(ord('A') + idx)}"
+        out[label] = f
+    return out
+
+
+def anonymize_responses(
+    responses: Iterable[tuple[str, str]],
+    *,
+    persona_labels: dict[str, str] | None = None,
+) -> tuple[dict[str, str], dict[str, str]]:
+    """Anonymize deliberation responses for the peer-review round (Phase 5).
+
+    `responses` is an iterable of ``(source, text)`` pairs where ``source``
+    is the canonical `provider:model` identifier. Returns:
+
+      - ``anon_text``: ``{Response-A: <body>}`` map fed into the prompt.
+      - ``label_to_source``: ``{Response-A: provider:model}`` map kept
+        server-side so the orchestrator can de-anonymize at synthesis time.
+
+    Empty / whitespace-only texts are skipped — they leak nothing and
+    would clutter the prompt. Input order is preserved so determinism
+    holds for tests (Iron-Law neutrality §peer-review: anonymization
+    strips identity, not order; deterministic A/B labels avoid
+    accidental cross-run reidentification when the same artefact is
+    re-run).
+
+    Phase 6 Step 3a wires `persona_labels` so advisor-mode runs render
+    as ``Response A (Contrarian)`` while provider identity stays
+    stripped. ``persona_labels`` maps ``source`` → ``persona`` (e.g.
+    ``"anthropic:claude-opus-4-1" -> "Contrarian"``); sources missing
+    from the map render as bare ``Response A``. Plain-member runs pass
+    ``persona_labels=None`` and behave exactly like today.
+    """
+    anon_text: dict[str, str] = {}
+    label_to_source: dict[str, str] = {}
+    idx = 0
+    for source, text in responses:
+        if not text or not text.strip():
+            continue
+        base = f"Response-{chr(ord('A') + idx)}"
+        persona = (persona_labels or {}).get(source)
+        label = f"{base} ({persona})" if persona else base
+        anon_text[label] = text.strip()
+        label_to_source[label] = source
+        idx += 1
+    return anon_text, label_to_source

package/scripts/ai_council/events_log.py

@@ -0,0 +1,137 @@
+"""Persistent council events log (step-8 phase 3).
+
+Single-function module that appends one JSON line per council event to
+``<project_root>/agents/council-events.log``. Schema v1 carries the
+minimum needed to answer the "why did the council skip / block this?"
+question at retro time without leaking prompt content.
+
+Privacy floor:
+    ``original_ask`` is never written verbatim — the caller passes the
+    raw string, and :func:`append_event` writes ``sha256(value)[:12]``
+    as ``original_ask_hash``. Mirrors the privacy floor in
+    ``agents/low-impact-decisions.md``.
+
+Kill-switch:
+    ``AGENT_CONFIG_NO_EVENTS_LOG=1`` short-circuits :func:`append_event`
+    to a no-op. Mirrors Step 7's ``AGENT_CONFIG_LEGACY_ANCHOR=1``
+    pattern. Tested via env-var override; the agent never reads or
+    parses the log itself.
+
+See: ``agents/roadmaps/step-8-quota-necessity-transparency.md`` (D3,
+D5) and ``docs/contracts/ai-council-config.md``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+SCHEMA_VERSION = 1
+
+EventAction = Literal["proceed", "skip_necessity", "block_quota"]
+
+_VALID_ACTIONS: frozenset[str] = frozenset(
+    {"proceed", "skip_necessity", "block_quota"},
+)
+
+#: Environment-variable kill-switch. Truthy values disable all writes;
+#: the function silently returns. Designed for CI / sandboxed runs and
+#: privacy-conscious power users.
+_KILL_SWITCH_ENV = "AGENT_CONFIG_NO_EVENTS_LOG"
+
+#: Default log path, resolved relative to the package root (two levels
+#: above ``scripts/ai_council/``). Callers can override via
+#: ``log_path=`` for tests.
+_DEFAULT_LOG_PATH = (
+    Path(__file__).resolve().parents[2] / "agents" / "council-events.log"
+)
+
+
+def _hash_original_ask(original_ask: str) -> str:
+    """Return sha256(original_ask)[:12] — the privacy-floor hash.
+
+    Empty / missing input maps to a stable sentinel so the schema field
+    is always populated.
+    """
+    if not original_ask:
+        return "0" * 12
+    return hashlib.sha256(
+        original_ask.encode("utf-8", errors="replace"),
+    ).hexdigest()[:12]
+
+
+def _kill_switch_active() -> bool:
+    value = os.environ.get(_KILL_SWITCH_ENV, "")
+    return value not in ("", "0", "false", "False")
+
+
+def append_event(
+    event: dict[str, Any], *, log_path: Path | None = None,
+) -> bool:
+    """Append a single JSON event line to the council events log.
+
+    Args:
+        event: Mapping with the v1 schema fields. Required keys:
+            ``lens``, ``invocation``, ``action``, ``verdict``,
+            ``provider_caps``, ``original_ask``. The function injects
+            ``schema_version``, ``ts_utc``, and replaces
+            ``original_ask`` with ``original_ask_hash``. Unknown keys
+            pass through verbatim — callers should not abuse this for
+            free-form payloads (privacy floor).
+        log_path: Override for tests. Defaults to
+            ``<project_root>/agents/council-events.log``.
+
+    Returns:
+        ``True`` when a line was written; ``False`` when the kill-switch
+        suppressed the write. Never raises on missing parent dir — the
+        function creates it on demand.
+
+    Raises:
+        ValueError: ``action`` not in :data:`_VALID_ACTIONS`.
+    """
+    if _kill_switch_active():
+        return False
+
+    action = event.get("action")
+    if action not in _VALID_ACTIONS:
+        raise ValueError(
+            f"events_log: action={action!r} not in "
+            f"{sorted(_VALID_ACTIONS)}.",
+        )
+
+    raw_ask = event.pop("original_ask", "") if "original_ask" in event else ""
+    record = {
+        "schema_version": SCHEMA_VERSION,
+        "ts_utc": datetime.now(timezone.utc).isoformat(
+            timespec="seconds",
+        ).replace("+00:00", "Z"),
+        "lens": event.get("lens", ""),
+        "invocation": event.get("invocation", ""),
+        "action": action,
+        "verdict": event.get("verdict", ""),
+        "provider_caps": event.get("provider_caps", {}),
+        "original_ask_hash": _hash_original_ask(raw_ask),
+    }
+    # Pass-through for any caller-supplied diagnostic fields that are
+    # not in the schema-v1 reserved set (e.g. `category`, `rationale`).
+    # The schema-v1 fields above always win on collision.
+    reserved = set(record) | {"original_ask"}
+    for k, v in event.items():
+        if k not in reserved:
+            record[k] = v
+
+    target = Path(log_path) if log_path is not None else _DEFAULT_LOG_PATH
+    target.parent.mkdir(parents=True, exist_ok=True)
+    line = json.dumps(record, ensure_ascii=False, separators=(",", ":"))
+    with target.open("a", encoding="utf-8") as fh:
+        fh.write(line + "\n")
+    return True
+
+
+def default_log_path() -> Path:
+    """Return the canonical events-log path (callers / tests)."""
+    return _DEFAULT_LOG_PATH