@event4u/agent-config 2.13.0 → 2.14.0

package/scripts/ai_council/replay.py

@@ -0,0 +1,155 @@
+"""Decision-replay artefact for council sessions (Phase 9).
+
+Produces a per-session ``decision-replay.md`` that surfaces the audit
+trail GPT review of PR #148 called out as missing: for each top
+finding, the consensus_strength, agreeing-members with their key
+argument, dissenting-members with their counter-argument, the
+evidence-quality verdict, and a final synthesis verdict line.
+
+The artefact is a pure projection of the consensus data plus the
+per-member deliberation texts — no extra model calls. Schema is
+documented in ``docs/contracts/ai-council-config.md`` under
+"Decision-replay schema".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable, Sequence
+
+from scripts.ai_council.clients import CouncilResponse
+from scripts.ai_council.consensus import (
+    ConsensusMetadata,
+    Finding,
+    FindingScore,
+)
+
+
+@dataclass(frozen=True)
+class DecisionReplayInputs:
+    """Bundle accepted by :func:`render_decision_replay`.
+
+    ``include_member_arguments`` toggles the redacted-vs-full output.
+    When ``False`` the artefact emits consensus + dissent COUNT only —
+    no per-member arguments — for sharing without leaking which model
+    framed which point.
+    """
+
+    findings: Sequence[Finding]
+    scores: Sequence[FindingScore]
+    metadata: dict[str, ConsensusMetadata]
+    deliberation: Sequence[CouncilResponse]  # last-round per-member texts
+    original_ask: str = ""
+    include_member_arguments: bool = True
+
+
+def _verdict(strength: float) -> str:
+    """Single-word verdict band for a consensus_strength."""
+    if strength > 0.7:
+        return "Strong"
+    if strength > 0.4:
+        return "Moderate"
+    return "Weak"
+
+
+def _scorer_argument(
+    scorer: str,
+    member_texts: dict[str, str],
+    score: FindingScore | None,
+) -> str:
+    """Return the one-line key argument for ``scorer`` on a finding.
+
+    Prefers the scorer's ``reason`` field (rich, contextual) and falls
+    back to the truncated deliberation snippet so the audit trail never
+    surfaces an empty argument.
+    """
+    if score and score.reason:
+        flat = " ".join(score.reason.split())
+        if len(flat) > 200:
+            flat = flat[:199].rstrip() + "…"
+        return flat
+    snippet = member_texts.get(scorer, "")
+    flat = " ".join(snippet.split())
+    if not flat:
+        return "no argument captured"
+    if len(flat) > 200:
+        flat = flat[:199].rstrip() + "…"
+    return flat
+
+
+def _scores_for_finding(
+    fid: str, scores: Iterable[FindingScore],
+) -> dict[str, FindingScore]:
+    return {s.scorer: s for s in scores if s.finding_id == fid}
+
+
+def render_decision_replay(inputs: DecisionReplayInputs) -> str:
+    """Render the ``decision-replay.md`` body.
+
+    Sections (in order): a leading H1 plus the original ask blockquote,
+    one ``## <finding-id> — <truncated text>`` block per finding (ranked
+    by consensus_strength desc), and a trailing footer with the toggle
+    state so consumers can tell at a glance whether arguments were
+    redacted.
+    """
+    member_texts = {f"{r.provider}:{r.model}": r.text or "" for r in inputs.deliberation}
+    ranked = sorted(
+        inputs.findings,
+        key=lambda f: inputs.metadata.get(
+            f.id,
+            ConsensusMetadata(
+                finding_id=f.id, consensus_strength=0.0, dissent_count=0,
+                scorers=(), mean_score=0.0,
+            ),
+        ).consensus_strength,
+        reverse=True,
+    )
+    lines: list[str] = ["# Decision Replay\n"]
+    if inputs.original_ask.strip():
+        ask = " ".join(inputs.original_ask.split())
+        if len(ask) > 400:
+            ask = ask[:399].rstrip() + "…"
+        lines.append(f"> {ask}\n")
+    if not ranked:
+        lines.append("*No findings were extracted for this session.*\n")
+        return "\n".join(lines).rstrip() + "\n"
+    for f in ranked:
+        m = inputs.metadata.get(f.id)
+        if m is None:
+            m = ConsensusMetadata(
+                finding_id=f.id, consensus_strength=0.0, dissent_count=0,
+                scorers=(), mean_score=0.0,
+            )
+        title = " ".join(f.text.split())
+        if len(title) > 120:
+            title = title[:119].rstrip() + "…"
+        verdict = _verdict(m.consensus_strength)
+        lines.append(f"## {f.id} — {title}\n")
+        lines.append(
+            f"- **Consensus**: {verdict} ({m.consensus_strength:.2f})\n"
+            f"- **Evidence quality**: {m.evidence_quality} "
+            f"(mean {m.mean_score:.1f}/10)\n"
+            f"- **Agreement**: {m.concur_count}/"
+            f"{m.concur_count + m.dissent_count} members concur, "
+            f"{m.dissent_count} dissent\n",
+        )
+        if inputs.include_member_arguments:
+            score_map = _scores_for_finding(f.id, inputs.scores)
+            agreeing = [s for s in m.scorers if score_map.get(s) and score_map[s].agree]
+            dissent = [pair for pair in m.dissent_reasons]
+            if agreeing:
+                lines.append("**Agreeing members**:")
+                for scorer in agreeing:
+                    arg = _scorer_argument(scorer, member_texts, score_map.get(scorer))
+                    lines.append(f"- _{scorer}_ — {arg}")
+                lines.append("")
+            if dissent:
+                lines.append("**Dissenting members**:")
+                for scorer, reason in dissent:
+                    arg = _scorer_argument(scorer, member_texts, score_map.get(scorer))
+                    lines.append(f"- _{scorer}_ — {arg}")
+                lines.append("")
+        lines.append(f"**Synthesis verdict**: {verdict} consensus — {f.source} sourced.\n")
+    mode_label = "full" if inputs.include_member_arguments else "redacted (counts only)"
+    lines.append(f"---\n\n_artefact mode: {mode_label}_\n")
+    return "\n".join(lines).rstrip() + "\n"

package/scripts/ai_council/session.py

@@ -71,14 +71,32 @@ def _utc_timestamp() -> str:
 
 
 def _serialise_response(r: CouncilResponse) -> dict[str, object]:
-    return {
+    """Project a `CouncilResponse` into the manifest schema.
+
+    Phase 5 / Step 1 — surface ``transport``, ``billable``,
+    ``subscription_label``, ``cost_usd``, and ``tokens_estimated`` so
+    the audit trail can distinguish flat-rate CLI calls from billable
+    api / community-CLI calls. When ``tokens_estimated`` is true the
+    token counts are kept (heuristic) but flagged so consumers can
+    null or disclaim them.
+    """
+    meta = r.metadata or {}
+    payload: dict[str, object] = {
         "provider": r.provider,
         "model": r.model,
         "input_tokens": r.input_tokens,
         "output_tokens": r.output_tokens,
         "latency_ms": r.latency_ms,
         "error": r.error,
+        "transport": meta.get("transport", "api"),
+        "billable": bool(meta.get("billable", True)),
+        "tokens_estimated": bool(meta.get("tokens_estimated", False)),
     }
+    if meta.get("subscription_label"):
+        payload["subscription_label"] = meta["subscription_label"]
+    if "cost_usd" in meta:
+        payload["cost_usd"] = meta["cost_usd"]
+    return payload
 
 
 def _load_retention_days(settings_path: Path | None = None) -> int:

package/scripts/ai_council/shadow_dispatch.py

@@ -0,0 +1,235 @@
+"""Shadow-mode dispatch for low-impact solo-member decisions (step-9 P10).
+
+When ``low_impact.dispatch: single`` is active, a Bernoulli-sampled subset
+of decisions is shadowed through the full council so disagreement between
+the solo verdict and the council verdict can be measured. The shadow log
+lives at ``agents/council-shadow-log.jsonl`` and is subject to the same
+privacy floor as the low-impact corpus: redactor-refused entries are
+dropped, not softened.
+
+The flip from ``single`` back to ``full`` is a user decision; this module
+emits data and an SLO banner, nothing else.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Iterable
+
+from scripts.ai_council.bundler import redact
+
+SHADOW_LOG_PATH = Path("agents/council-shadow-log.jsonl")
+
+SLO_THRESHOLD_WARN = 0.05
+SLO_THRESHOLD_BREACH = 0.08
+
+
+@dataclass(frozen=True)
+class ShadowDecision:
+    timestamp: str
+    query_hash: str
+    solo_verdict: str
+    full_verdict: str
+    agreed: bool
+    #: Step-9 P13 — True when the confidence gate auto-escalated this
+    #: decision to the full council. Distinguishes "silent disagreement"
+    #: (escalated=False, agreed=False) from "gate-caught" (escalated=True)
+    #: in the SLO banner.
+    escalated: bool = False
+    escalation_reason: str = "ok"
+
+
+def should_shadow(
+    sample_rate: float,
+    *,
+    rng: random.Random | None = None,
+) -> bool:
+    rate = max(0.0, min(1.0, sample_rate))
+    r = rng if rng is not None else random
+    return r.random() < rate
+
+
+def _hash_query(query: str) -> str:
+    redacted = redact(query)
+    return hashlib.sha256(redacted.encode("utf-8")).hexdigest()[:16]
+
+
+def _privacy_dropped(redacted: str) -> bool:
+    stripped = redacted.strip()
+    if not stripped:
+        return True
+    return stripped.startswith("[redacted")
+
+
+def record_shadow_decision(
+    log_path: Path,
+    *,
+    query: str,
+    solo_verdict: str,
+    full_verdict: str,
+    escalated: bool = False,
+    escalation_reason: str = "ok",
+) -> ShadowDecision | None:
+    """Append one JSONL row. Returns ``None`` when redaction would drop
+    the entry (privacy floor — do not soften).
+
+    ``escalated`` / ``escalation_reason`` come from the confidence
+    gate (step-9 P13). When True, ``solo_verdict`` is the rejected
+    solo response and ``full_verdict`` is the council's verdict that
+    actually answered the user.
+    """
+    redacted_q = redact(query)
+    if _privacy_dropped(redacted_q):
+        return None
+
+    decision = ShadowDecision(
+        timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        query_hash=_hash_query(query),
+        solo_verdict=solo_verdict,
+        full_verdict=full_verdict,
+        agreed=(solo_verdict == full_verdict),
+        escalated=escalated,
+        escalation_reason=escalation_reason,
+    )
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    with log_path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps({
+            "timestamp": decision.timestamp,
+            "query_hash": decision.query_hash,
+            "solo_verdict": decision.solo_verdict,
+            "full_verdict": decision.full_verdict,
+            "agreed": decision.agreed,
+            "escalated": decision.escalated,
+            "escalation_reason": decision.escalation_reason,
+        }) + "\n")
+    return decision
+
+
+def _iter_log(log_path: Path) -> Iterable[dict]:
+    if not log_path.exists():
+        return
+    with log_path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+
+def compute_disagreement_rate(
+    log_path: Path,
+    *,
+    window_days: int = 7,
+    now: datetime | None = None,
+) -> tuple[float, int]:
+    """``(disagreement_rate, sample_count)`` over the rolling window.
+
+    Counts a row as "disagreed" when ``agreed=False`` regardless of
+    the escalation flag — a gate-caught split is still a sign that
+    solo mode was wrong on that decision. :func:`compute_escalation_rate`
+    breaks the same window down by ``escalated=True`` for the banner.
+    """
+    cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=window_days)
+    total = 0
+    disagreed = 0
+    for row in _iter_log(log_path):
+        raw_ts = row.get("timestamp", "")
+        try:
+            ts = datetime.fromisoformat(raw_ts.replace("Z", "+00:00"))
+        except ValueError:
+            continue
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=timezone.utc)
+        if ts < cutoff:
+            continue
+        total += 1
+        if not row.get("agreed", True):
+            disagreed += 1
+    if total == 0:
+        return 0.0, 0
+    return disagreed / total, total
+
+
+def compute_escalation_rate(
+    log_path: Path,
+    *,
+    window_days: int = 7,
+    now: datetime | None = None,
+) -> tuple[float, int]:
+    """``(escalation_rate, sample_count)`` — fraction with ``escalated=True``.
+
+    Step-9 P13 — separates gate-caught escalations from silent
+    disagreement so the banner can name the dominant failure mode.
+    """
+    cutoff = (now or datetime.now(timezone.utc)) - timedelta(days=window_days)
+    total = 0
+    escalated = 0
+    for row in _iter_log(log_path):
+        raw_ts = row.get("timestamp", "")
+        try:
+            ts = datetime.fromisoformat(raw_ts.replace("Z", "+00:00"))
+        except ValueError:
+            continue
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=timezone.utc)
+        if ts < cutoff:
+            continue
+        total += 1
+        if row.get("escalated", False):
+            escalated += 1
+    if total == 0:
+        return 0.0, 0
+    return escalated / total, total
+
+
+def slo_status(rate: float) -> str:
+    if rate < SLO_THRESHOLD_WARN:
+        return "OK"
+    if rate < SLO_THRESHOLD_BREACH:
+        return "WARN"
+    return "BREACH"
+
+
+def slo_banner(
+    rate: float,
+    sample_count: int,
+    *,
+    escalation_rate: float | None = None,
+) -> str:
+    """One-line SLO banner. ``escalation_rate`` is appended when given.
+
+    Step-9 P13 — escalation tail surfaces the share of decisions the
+    confidence gate caught before they reached the user.
+    """
+    pct = rate * 100
+    status = slo_status(rate)
+    if sample_count == 0:
+        return "[shadow SLO] no samples yet"
+    if status == "OK":
+        base = (
+            f"[shadow SLO] OK · {pct:.1f}% disagreement over "
+            f"{sample_count} samples (<5%)"
+        )
+    elif status == "WARN":
+        base = (
+            f"[shadow SLO] WARN · {pct:.1f}% disagreement over "
+            f"{sample_count} samples (5–8% — consider reverting to "
+            f"low_impact.dispatch: full)"
+        )
+    else:
+        base = (
+            f"[shadow SLO] BREACH · {pct:.1f}% disagreement over "
+            f"{sample_count} samples (>8% — revert to "
+            f"low_impact.dispatch: full)"
+        )
+    if escalation_rate is not None:
+        base += f" · {escalation_rate * 100:.1f}% auto-escalated"
+    return base

package/scripts/ai_council/solo_dispatch.py

@@ -0,0 +1,226 @@
+"""Solo-member dispatch — step-9 P9 (U2).
+
+Picks the first enabled, auth-valid member from
+``routing.solo_member_fallback_chain`` so low-impact decisions can
+optionally route to a single member instead of the full council. The
+selection is intentionally side-effect-free: callers own logging,
+dispatch, and the all-invalid → full-council fallback.
+
+Iron Law: a None selection from :func:`select_solo_member` is the
+caller's signal to fall back to the full council with a WARN log —
+NEVER to fail the decision. The dispatcher must never break a
+user's flow because a CLI was offline.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Mapping
+
+from scripts.ai_council.config import MemberConfig, RoutingConfig
+from scripts.ai_council.confidence_gate import (
+    EscalationDecision,
+    should_escalate,
+)
+
+#: TTL for cached auth-probe results. Lazy probe per session; bumped
+#: forward whenever a probe is re-run.
+_AUTH_CACHE_TTL_SECONDS = 15 * 60
+
+#: Env var that forces every solo-dispatch path back to full council
+#: for the current invocation. Honored by :func:`select_solo_member`
+#: and surfaced through :func:`force_full_council`.
+FORCE_FULL_ENV = "AGENT_CONFIG_FORCE_FULL_COUNCIL"
+
+
+@dataclass
+class AuthCacheEntry:
+    """One auth-probe result with the expiry it was cached against."""
+
+    valid: bool
+    expires_at: float
+
+
+@dataclass
+class AuthCache:
+    """In-memory cache for auth-probe verdicts (per-process)."""
+
+    entries: dict[str, AuthCacheEntry] = field(default_factory=dict)
+
+    def get(self, name: str, *, now: float) -> bool | None:
+        entry = self.entries.get(name)
+        if entry is None or entry.expires_at <= now:
+            return None
+        return entry.valid
+
+    def set(self, name: str, *, valid: bool, now: float) -> None:
+        self.entries[name] = AuthCacheEntry(
+            valid=valid, expires_at=now + _AUTH_CACHE_TTL_SECONDS,
+        )
+
+
+def force_full_council(env: Mapping[str, str] | None = None) -> bool:
+    """Return True iff the env-var override is set to ``1``.
+
+    Truthy values other than ``1`` are intentionally rejected — the
+    override is a hard one-bit switch, not a free-form bool.
+    """
+    src = env if env is not None else os.environ
+    return src.get(FORCE_FULL_ENV, "") == "1"
+
+
+def select_solo_member(
+    routing: RoutingConfig,
+    members: Mapping[str, MemberConfig],
+    *,
+    auth_cache: AuthCache,
+    probe: Callable[[str, float], bool],
+    now: float | None = None,
+    env: Mapping[str, str] | None = None,
+) -> str | None:
+    """Return the first chain entry whose member is enabled + auth-valid.
+
+    Walks ``routing.solo_member_fallback_chain`` in order. For each
+    entry: skip when the member is missing or disabled; consult the
+    auth cache; on miss probe lazily with the configured timeout and
+    cache the result. Returns the provider name of the first valid
+    member, or ``None`` when every chain entry is unavailable.
+
+    ``probe(name, timeout_s) -> bool`` is the caller-supplied auth
+    check. It MUST honor ``timeout_s`` and return False on timeout
+    so the dispatcher cannot stall on a wedged CLI.
+
+    Env-var override (``AGENT_CONFIG_FORCE_FULL_COUNCIL=1``) short-
+    circuits to None, treating the whole chain as unavailable. The
+    caller still owns the WARN log + full-council escalation.
+    """
+    if force_full_council(env):
+        return None
+    if now is None:
+        now = time.monotonic()
+    timeout_s = routing.auth_check_timeout_seconds
+    for name in routing.solo_member_fallback_chain:
+        member = members.get(name)
+        if member is None or not member.enabled:
+            continue
+        cached = auth_cache.get(name, now=now)
+        if cached is False:
+            continue
+        if cached is True:
+            return name
+        try:
+            valid = bool(probe(name, timeout_s))
+        except Exception:
+            # Probe blew up — treat as auth-invalid so the chain
+            # walks to the next entry. Don't swallow silently in
+            # production: callers should log probe failures.
+            valid = False
+        auth_cache.set(name, valid=valid, now=now)
+        if valid:
+            return name
+    return None
+
+
+@dataclass(frozen=True)
+class SoloDispatchResult:
+    """Outcome of :func:`dispatch_with_escalation`.
+
+    ``verdict`` is the final answer text returned to the caller.
+    ``escalated`` is True when the solo response was rejected by the
+    confidence gate and the full council ran. ``solo_member`` /
+    ``solo_response`` are populated even on escalation so the shadow
+    log can record both sides without re-running the solo step.
+    """
+
+    verdict: str
+    escalated: bool
+    escalation_reason: str  # 'low_confidence' | 'split' | 'refusal' | 'short_response' | 'ok' | 'no_solo_member'
+    solo_member: str | None
+    solo_response: str | None
+    solo_confidence: float | None
+
+
+def dispatch_with_escalation(
+    routing: RoutingConfig,
+    members: Mapping[str, MemberConfig],
+    *,
+    auth_cache: AuthCache,
+    probe: Callable[[str, float], bool],
+    run_solo: Callable[[str], str],
+    run_full: Callable[[], str],
+    confidence_floor: float,
+    now: float | None = None,
+    env: Mapping[str, str] | None = None,
+) -> SoloDispatchResult:
+    """Solo-dispatch with auto-escalation on low-confidence / split / refusal.
+
+    Step-9 P13 — defense-in-depth on top of shadow-mode SLO.
+
+    Flow:
+
+    1. ``select_solo_member`` picks the chain entry.
+    2. None → escalate immediately (``no_solo_member``).
+    3. ``run_solo`` is invoked; response is scored via
+       :func:`scripts.ai_council.confidence_gate.should_escalate`.
+    4. Verdict ``escalate=True`` → ``run_full`` is invoked and that
+       verdict is returned; the solo response stays on the result
+       for shadow logging.
+    5. ``escalate=False`` → solo verdict is returned as-is.
+
+    ``run_solo(name) -> str`` and ``run_full() -> str`` are caller-
+    supplied; this module owns no LLM transport. Callers MUST raise
+    on transport errors — escalation is for *content* low-confidence,
+    not infrastructure failures (those bubble up to the orchestrator's
+    own retry / fallback policy).
+    """
+    name = select_solo_member(
+        routing,
+        members,
+        auth_cache=auth_cache,
+        probe=probe,
+        now=now,
+        env=env,
+    )
+    if name is None:
+        return SoloDispatchResult(
+            verdict=run_full(),
+            escalated=True,
+            escalation_reason="no_solo_member",
+            solo_member=None,
+            solo_response=None,
+            solo_confidence=None,
+        )
+    solo = run_solo(name)
+    decision: EscalationDecision = should_escalate(solo, floor=confidence_floor)
+    if decision.escalate:
+        return SoloDispatchResult(
+            verdict=run_full(),
+            escalated=True,
+            escalation_reason=decision.reason,
+            solo_member=name,
+            solo_response=solo,
+            solo_confidence=decision.confidence,
+        )
+    return SoloDispatchResult(
+        verdict=solo,
+        escalated=False,
+        escalation_reason="ok",
+        solo_member=name,
+        solo_response=solo,
+        solo_confidence=decision.confidence,
+    )
+
+
+__all__ = [
+    "AUTH_CACHE_TTL_SECONDS",
+    "AuthCache",
+    "AuthCacheEntry",
+    "FORCE_FULL_ENV",
+    "SoloDispatchResult",
+    "dispatch_with_escalation",
+    "force_full_council",
+    "select_solo_member",
+]
+AUTH_CACHE_TTL_SECONDS = _AUTH_CACHE_TTL_SECONDS