@event4u/agent-config 1.15.0 → 1.16.0

package/scripts/ai_council/bundler.py

@@ -0,0 +1,261 @@
+"""Context bundling for council consultations.
+
+Takes a raw artefact (free-form prompt, roadmap path, diff range, or
+file set) and produces a `CouncilContext` — a redacted, size-bounded
+text bundle plus a manifest describing exactly what was included.
+
+Hard rules:
+- Redaction is fail-closed. If a redaction pattern fires, the line is
+  scrubbed *before* the bundle is built.
+- Size guard is fail-loud. > MAX_BUNDLE_BYTES → raises BundleTooLarge,
+  never silently truncates (would mislead council members).
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+MAX_BUNDLE_BYTES = 50 * 1024  # 50 KB hard ceiling; user must narrow scope on hit.
+
+
+class BundleTooLarge(RuntimeError):
+    """Raised when the assembled bundle exceeds MAX_BUNDLE_BYTES."""
+
+
+@dataclass
+class CouncilContext:
+    mode: str  # one of: prompt, roadmap, diff, files
+    text: str
+    manifest: list[str] = field(default_factory=list)
+    excluded: list[str] = field(default_factory=list)
+
+
+# ── redaction patterns ───────────────────────────────────────────────────
+# Each pattern is matched line-wise; matching lines are replaced with the
+# placeholder. Order matters — the most specific pattern goes first.
+
+_REDACTION_LINE_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r".*~?/?\.config/agent-config/[^/\s]+\.key.*"),
+     "[redacted: agent-config key path]"),
+    (re.compile(r"^\s*Authorization:\s.*", re.IGNORECASE),
+     "[redacted: Authorization header]"),
+    (re.compile(r"(?i).*(api[_-]?key|secret|token|password)\s*[:=].*"),
+     "[redacted: secret-like assignment]"),
+    (re.compile(r"sk-ant-[A-Za-z0-9_\-]{8,}"), "[redacted: anthropic-key-like token]"),
+    (re.compile(r"sk-[A-Za-z0-9_\-]{20,}"), "[redacted: openai-key-like token]"),
+]
+
+
+def redact(text: str) -> str:
+    """Apply redaction patterns to a multi-line text buffer."""
+    out: list[str] = []
+    for line in text.splitlines():
+        replaced = line
+        for pattern, placeholder in _REDACTION_LINE_PATTERNS:
+            if pattern.search(replaced):
+                replaced = placeholder
+                break
+        out.append(replaced)
+    return "\n".join(out)
+
+
+def _enforce_size(text: str, mode: str) -> str:
+    encoded = text.encode("utf-8")
+    if len(encoded) > MAX_BUNDLE_BYTES:
+        raise BundleTooLarge(
+            f"Bundle for {mode!r} mode is {len(encoded)} bytes "
+            f"(> {MAX_BUNDLE_BYTES} hard ceiling). "
+            "Narrow the scope (smaller diff, fewer files, shorter prompt)."
+        )
+    return text
+
+
+def bundle_prompt(text: str) -> CouncilContext:
+    redacted = redact(text)
+    return CouncilContext(
+        mode="prompt",
+        text=_enforce_size(redacted, "prompt"),
+        manifest=["<inline prompt>"],
+    )
+
+
+def bundle_roadmap(path: str | Path) -> CouncilContext:
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Roadmap not found: {p}")
+    raw = p.read_text(encoding="utf-8")
+    redacted = redact(raw)
+    return CouncilContext(
+        mode="roadmap",
+        text=_enforce_size(redacted, "roadmap"),
+        manifest=[str(p)],
+        excluded=["<linked contracts/skills not included by default>"],
+    )
+
+
+def bundle_diff(base_ref: str, head_ref: str = "HEAD", cwd: str | Path | None = None) -> CouncilContext:
+    cmd = ["git", "diff", f"{base_ref}..{head_ref}"]
+    try:
+        proc = subprocess.run(
+            cmd, cwd=cwd, check=True, capture_output=True, text=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        raise RuntimeError(
+            f"git diff {base_ref}..{head_ref} failed: {exc.stderr.strip()}"
+        ) from exc
+    redacted = redact(proc.stdout)
+    return CouncilContext(
+        mode="diff",
+        text=_enforce_size(redacted, "diff"),
+        manifest=[f"git diff {base_ref}..{head_ref}"],
+    )
+
+
+# ── smart diff context (D4) ─────────────────────────────────────────────────
+# Language-agnostic signature detection. Order matters — most specific first.
+
+_SIGNATURE_PATTERNS: list[re.Pattern[str]] = [
+    re.compile(r"^\s*(?:async\s+)?def\s+\w+\s*\("),  # Python
+    re.compile(r"^\s*class\s+\w+\b"),  # Python / PHP / JS class
+    re.compile(r"^\s*(?:public|protected|private|static|abstract|final)\s+(?:static\s+)?function\s+\w+"),  # PHP method
+    re.compile(r"^\s*function\s+\w+\s*\("),  # PHP free function / JS
+    re.compile(r"^\s*export\s+(?:default\s+)?(?:async\s+)?function\s+\w+"),  # TS/JS export fn
+    re.compile(r"^\s*export\s+(?:default\s+)?class\s+\w+"),  # TS/JS export class
+    re.compile(r"^\s*(?:export\s+)?(?:const|let)\s+\w+\s*=\s*(?:async\s+)?\("),  # TS arrow fn
+    re.compile(r"^\s*(?:public|private|protected)\s+\w+\s*\("),  # TS method
+]
+
+_HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+_DIFF_FILE = re.compile(r"^\+\+\+ b/(.+)$")
+
+
+def _parse_diff_hunks(diff_text: str) -> list[tuple[str, int]]:
+    """Return [(file_path, new_start_line), ...] per hunk in input order."""
+    out: list[tuple[str, int]] = []
+    current_file: str | None = None
+    for line in diff_text.splitlines():
+        m = _DIFF_FILE.match(line)
+        if m:
+            current_file = m.group(1)
+            continue
+        h = _HUNK_HEADER.match(line)
+        if h and current_file and current_file != "/dev/null":
+            out.append((current_file, int(h.group(1))))
+    return out
+
+
+def _enclosing_signature(
+    file_text: str, target_line: int,
+) -> tuple[int, str] | None:
+    """Walk backwards from `target_line` (1-based) to nearest signature."""
+    lines = file_text.splitlines()
+    start = min(target_line - 1, len(lines) - 1)
+    for idx in range(start, -1, -1):
+        line = lines[idx]
+        for pat in _SIGNATURE_PATTERNS:
+            if pat.match(line):
+                return (idx + 1, line.rstrip())
+    return None
+
+
+def bundle_diff_with_context(
+    base_ref: str,
+    head_ref: str = "HEAD",
+    cwd: str | Path | None = None,
+    *,
+    max_context_bytes: int = 8 * 1024,
+) -> CouncilContext:
+    """Bundle a diff plus the nearest enclosing signatures for each hunk.
+
+    Appends a `## Surrounding signatures` section after the raw diff.
+    Signatures are detected by regex across PY / PHP / JS / TS. Reads
+    files from the working tree (correct when `head_ref` == HEAD); if
+    a touched file is missing on disk it is silently dropped from the
+    context section (the diff itself still shows the change).
+
+    Hard cap: `max_context_bytes` for the signature section. Combined
+    output still goes through `_enforce_size`, so the `BundleTooLarge`
+    behaviour is unchanged.
+    """
+    base = bundle_diff(base_ref, head_ref, cwd=cwd)
+    hunks = _parse_diff_hunks(base.text)
+    if not hunks:
+        return base
+
+    root = Path(cwd) if cwd else Path(".")
+    seen: set[tuple[str, int]] = set()  # (file, signature_line)
+    by_file: dict[str, list[tuple[int, str]]] = {}
+
+    for file_path, new_start in hunks:
+        target = root / file_path
+        try:
+            file_text = target.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        sig = _enclosing_signature(file_text, new_start)
+        if sig is None:
+            continue
+        key = (file_path, sig[0])
+        if key in seen:
+            continue
+        seen.add(key)
+        by_file.setdefault(file_path, []).append(sig)
+
+    if not by_file:
+        return base
+
+    out_lines: list[str] = ["", "## Surrounding signatures", ""]
+    truncated = False
+    used = 0
+    for file_path, sigs in by_file.items():
+        header = f"### {file_path}"
+        sig_block = "\n".join(f"    L{ln}: {text}" for ln, text in sorted(sigs))
+        chunk = f"{header}\n\n{sig_block}\n\n"
+        if used + len(chunk.encode("utf-8")) > max_context_bytes:
+            truncated = True
+            break
+        out_lines.append(header)
+        out_lines.append("")
+        out_lines.append(sig_block)
+        out_lines.append("")
+        used += len(chunk.encode("utf-8"))
+
+    if truncated:
+        out_lines.append(f"[truncated: signature section capped at {max_context_bytes} bytes]")
+
+    combined = base.text + "\n" + "\n".join(out_lines)
+    redacted = redact(combined)
+    return CouncilContext(
+        mode="diff",
+        text=_enforce_size(redacted, "diff"),
+        manifest=base.manifest + [f"+ surrounding signatures for {len(by_file)} file(s)"],
+    )
+
+
+def bundle_files(paths: list[str | Path]) -> CouncilContext:
+    parts: list[str] = []
+    manifest: list[str] = []
+    excluded: list[str] = []
+    for raw_path in paths:
+        p = Path(raw_path)
+        if not p.exists():
+            excluded.append(f"{p} (not found)")
+            continue
+        try:
+            content = p.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError) as exc:
+            excluded.append(f"{p} ({type(exc).__name__})")
+            continue
+        parts.append(f"### {p}\n\n{content}\n")
+        manifest.append(str(p))
+    bundled = "\n".join(parts)
+    redacted = redact(bundled)
+    return CouncilContext(
+        mode="files",
+        text=_enforce_size(redacted, "files"),
+        manifest=manifest,
+        excluded=excluded,
+    )

package/scripts/ai_council/clients.py

@@ -0,0 +1,381 @@
+"""External-AI clients for the council.
+
+Mirrors the contract from `scripts/skill_trigger_eval.py`:
+- Tokens come exclusively from ~/.config/agent-config/<provider>.key.
+- File mode must be exactly 0o600. Drift is a hard abort.
+- No environment-variable fallback. No keychain fallback.
+- Real SDKs (`anthropic`, `openai`) are *soft* dependencies — the
+  module imports cleanly without them; only `ask()` requires them.
+
+Tests inject mock clients via the `client=` constructor argument and
+never hit the real API.
+
+Mode contract (Phase 2b):
+- `billable=True` clients (AnthropicClient, OpenAIClient) participate
+  in the cost gate — projected USD spend is checked before each call.
+- `billable=False` clients (ManualClient, future PlaywrightClient)
+  skip the cost gate entirely. Spend = $0 to us; provider-side rate
+  limits are the user's concern.
+"""
+
+from __future__ import annotations
+
+import stat
+import sys
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TextIO
+
+ANTHROPIC_KEY_PATH = Path.home() / ".config" / "agent-config" / "anthropic.key"
+OPENAI_KEY_PATH = Path.home() / ".config" / "agent-config" / "openai.key"
+
+DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-5"
+DEFAULT_OPENAI_MODEL = "gpt-4o"
+
+
+class KeyGateError(RuntimeError):
+    """Raised when a provider key file violates the 0600 contract."""
+
+
+@dataclass
+class CouncilResponse:
+    """Normalised output from a single council member."""
+
+    provider: str
+    model: str
+    text: str
+    input_tokens: int = 0
+    output_tokens: int = 0
+    latency_ms: int = 0
+    error: str | None = None
+    metadata: dict[str, object] = field(default_factory=dict)
+
+
+def _load_key(path: Path, prefix: str, install_script: str) -> str:
+    """Shared 0600-gated key loader. Refuses anything outside the contract."""
+    if not path.exists():
+        raise KeyGateError(
+            f"Key not found at {path}.\n"
+            f"    Install it with: bash {install_script}"
+        )
+    st = path.stat()
+    mode = stat.S_IMODE(st.st_mode)
+    if mode != 0o600:
+        raise KeyGateError(
+            f"Unsafe permissions on {path}: got {oct(mode)}, expected 0o600.\n"
+            f"    Fix:  chmod 600 {path}"
+        )
+    key = path.read_text(encoding="utf-8").strip()
+    if not key:
+        raise KeyGateError(f"{path} is empty.")
+    if not key.startswith(prefix):
+        raise KeyGateError(
+            f"{path} does not look like a {prefix!r}-prefixed key."
+        )
+    return key
+
+
+def load_anthropic_key(path: Path = ANTHROPIC_KEY_PATH) -> str:
+    return _load_key(path, "sk-ant-", "scripts/install_anthropic_key.sh")
+
+
+def load_openai_key(path: Path = OPENAI_KEY_PATH) -> str:
+    return _load_key(path, "sk-", "scripts/install_openai_key.sh")
+
+
+class ExternalAIClient(ABC):
+    """Abstract base for council members."""
+
+    name: str = ""
+    model: str = ""
+    billable: bool = True  # API-mode subclasses spend money; manual/playwright don't.
+
+    @abstractmethod
+    def ask(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        max_tokens: int = 1024,
+    ) -> CouncilResponse:
+        """Send one independent query. Must never raise on network/API
+        failure — return a `CouncilResponse` with `error` set instead.
+        Other members should not be blocked by one failure."""
+
+
+class AnthropicClient(ExternalAIClient):
+    name = "anthropic"
+    billable = True
+
+    def __init__(
+        self,
+        model: str = DEFAULT_ANTHROPIC_MODEL,
+        client: object = None,
+        api_key: str | None = None,
+    ):
+        self.model = model
+        if client is not None:
+            self._client = client
+            return
+        if api_key is None:
+            raise RuntimeError(
+                "AnthropicClient requires explicit api_key or injected client. "
+                "Use load_anthropic_key() — no env-var fallback."
+            )
+        try:
+            import anthropic  # type: ignore[import-not-found]
+        except ImportError as exc:  # pragma: no cover - exercised only with real SDK
+            raise RuntimeError(
+                "anthropic package not installed. `pip install anthropic`."
+            ) from exc
+        self._client = anthropic.Anthropic(api_key=api_key)
+
+    def ask(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024) -> CouncilResponse:
+        t0 = time.monotonic()
+        try:
+            response = self._client.messages.create(
+                model=self.model,
+                max_tokens=max_tokens,
+                system=system_prompt,
+                messages=[{"role": "user", "content": user_prompt}],
+            )
+        except Exception as exc:  # noqa: BLE001 - normalise all SDK errors
+            return CouncilResponse(
+                provider=self.name, model=self.model, text="",
+                latency_ms=int((time.monotonic() - t0) * 1000),
+                error=f"{type(exc).__name__}: {exc}",
+            )
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        text = ""
+        content = getattr(response, "content", None)
+        if content:
+            text = getattr(content[0], "text", "") or ""
+        usage = getattr(response, "usage", None)
+        return CouncilResponse(
+            provider=self.name, model=self.model, text=text,
+            input_tokens=getattr(usage, "input_tokens", 0) if usage else 0,
+            output_tokens=getattr(usage, "output_tokens", 0) if usage else 0,
+            latency_ms=latency_ms,
+        )
+
+
+class OpenAIClient(ExternalAIClient):
+    name = "openai"
+    billable = True
+
+    def __init__(
+        self,
+        model: str = DEFAULT_OPENAI_MODEL,
+        client: object = None,
+        api_key: str | None = None,
+    ):
+        self.model = model
+        if client is not None:
+            self._client = client
+            return
+        if api_key is None:
+            raise RuntimeError(
+                "OpenAIClient requires explicit api_key or injected client. "
+                "Use load_openai_key() — no env-var fallback."
+            )
+        try:
+            import openai  # type: ignore[import-not-found]
+        except ImportError as exc:  # pragma: no cover - exercised only with real SDK
+            raise RuntimeError(
+                "openai package not installed. `pip install openai`."
+            ) from exc
+        self._client = openai.OpenAI(api_key=api_key)
+
+    def ask(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024) -> CouncilResponse:
+        t0 = time.monotonic()
+        try:
+            response = self._client.chat.completions.create(
+                model=self.model,
+                max_tokens=max_tokens,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt},
+                ],
+            )
+        except Exception as exc:  # noqa: BLE001 - normalise all SDK errors
+            return CouncilResponse(
+                provider=self.name, model=self.model, text="",
+                latency_ms=int((time.monotonic() - t0) * 1000),
+                error=f"{type(exc).__name__}: {exc}",
+            )
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        text = ""
+        choices = getattr(response, "choices", None)
+        if choices:
+            msg = getattr(choices[0], "message", None)
+            text = getattr(msg, "content", "") if msg else ""
+        usage = getattr(response, "usage", None)
+        return CouncilResponse(
+            provider=self.name, model=self.model, text=text or "",
+            input_tokens=getattr(usage, "prompt_tokens", 0) if usage else 0,
+            output_tokens=getattr(usage, "completion_tokens", 0) if usage else 0,
+            latency_ms=latency_ms,
+        )
+
+
+# ── Manual mode (Phase 2b) ───────────────────────────────────────────
+
+
+MANUAL_END_MARKER = "END"  # line containing only this terminates a paste block.
+
+
+def _read_until_marker(stream: TextIO, marker: str) -> str:
+    """Read lines from `stream` until a line equal to `marker` (after strip).
+
+    Returns the joined body without the marker line. EOF before the
+    marker is treated as end-of-input — the body collected so far is
+    returned; callers decide whether that counts as abort.
+    """
+    body: list[str] = []
+    for raw in stream:
+        line = raw.rstrip("\n")
+        if line.strip() == marker:
+            break
+        body.append(line)
+    return "\n".join(body).strip()
+
+
+class ManualClient(ExternalAIClient):
+    """Copy-paste council member — user is the transport.
+
+    `ask()` renders the system prompt + artefact as one Markdown block,
+    prints it to `stdout`, and reads pasted replies from `stdin`. After
+    each pasted reply, surfaces a 1/2/3 menu (more · next · abort) per
+    `user-interaction`. Loops until the user picks 2 or 3.
+
+    Spend is $0 — `billable=False` makes the orchestrator skip the cost
+    gate for this member regardless of the price table.
+
+    Tests inject `stdin` / `stdout` `TextIO` streams. Production usage
+    falls back to `sys.stdin` / `sys.stdout`.
+    """
+
+    billable = False
+
+    def __init__(
+        self,
+        *,
+        name: str = "manual",
+        model: str = "manual",
+        provider_label: str = "your LLM web UI",
+        stdin: TextIO | None = None,
+        stdout: TextIO | None = None,
+        end_marker: str = MANUAL_END_MARKER,
+    ):
+        self.name = name
+        self.model = model
+        self.provider_label = provider_label
+        self._stdin = stdin if stdin is not None else sys.stdin
+        self._stdout = stdout if stdout is not None else sys.stdout
+        self._end_marker = end_marker
+
+    def ask(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        max_tokens: int = 1024,  # noqa: ARG002 — accepted for ABC parity
+    ) -> CouncilResponse:
+        t0 = time.monotonic()
+        rounds: list[str] = []
+        block = self._render_block(system_prompt, user_prompt, follow_up=None)
+        self._emit(block)
+
+        try:
+            while True:
+                reply = _read_until_marker(self._stdin, self._end_marker)
+                rounds.append(reply)
+                choice = self._ask_menu(reply_chars=len(reply))
+
+                if choice == "2":  # done with this member
+                    break
+                if choice == "3":  # abort the council run
+                    return CouncilResponse(
+                        provider=self.name, model=self.model, text="",
+                        latency_ms=int((time.monotonic() - t0) * 1000),
+                        error="manual_aborted",
+                        metadata={"rounds": len(rounds), "manual": True},
+                    )
+                # choice == "1": collect follow-up, re-emit context block.
+                follow_up = self._read_follow_up()
+                if not follow_up:
+                    break  # empty follow-up → treat as "done with this member"
+                rounds.append(f"[follow-up sent]\n{follow_up}")
+                block = self._render_block(system_prompt, user_prompt, follow_up=follow_up)
+                self._emit(block)
+        except Exception as exc:  # noqa: BLE001 — never break the council on a stdin glitch
+            return CouncilResponse(
+                provider=self.name, model=self.model, text="\n\n".join(rounds),
+                latency_ms=int((time.monotonic() - t0) * 1000),
+                error=f"{type(exc).__name__}: {exc}",
+                metadata={"rounds": len(rounds), "manual": True},
+            )
+
+        text = "\n\n---\n\n".join(rounds).strip()
+        return CouncilResponse(
+            provider=self.name, model=self.model, text=text,
+            latency_ms=int((time.monotonic() - t0) * 1000),
+            metadata={"rounds": len(rounds), "manual": True},
+        )
+
+    # ── helpers ──────────────────────────────────────────────────────
+
+    def _emit(self, text: str) -> None:
+        self._stdout.write(text)
+        self._stdout.write("\n")
+        self._stdout.flush()
+
+    def _render_block(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        follow_up: str | None,
+    ) -> str:
+        bar = "═" * 67
+        head = (
+            f"{bar}\n"
+            f"Manual council member: {self.provider_label}\n"
+            f"Paste this block into the web UI · then paste the reply below.\n"
+            f"{bar}"
+        )
+        if follow_up is not None:
+            body = (
+                f"[Follow-up — paste this into the SAME chat thread]\n\n"
+                f"{follow_up}"
+            )
+        else:
+            body = f"{system_prompt}\n\n---\n\n{user_prompt}"
+        tail = (
+            f"{bar}\n"
+            f"End your pasted reply with a line containing only: {self._end_marker}\n"
+            f"{bar}"
+        )
+        return f"{head}\n\n{body}\n\n{tail}"
+
+    def _ask_menu(self, *, reply_chars: int) -> str:
+        prompt = (
+            f"\nReply received ({reply_chars} chars). Now what?\n"
+            f"  1. More feedback for this member (continue this thread)\n"
+            f"  2. Done with this member, move to the next\n"
+            f"  3. Abort the council run\n\n"
+            f"Choose 1/2/3: "
+        )
+        self._stdout.write(prompt)
+        self._stdout.flush()
+        line = self._stdin.readline().strip()
+        if line in {"1", "2", "3"}:
+            return line
+        # unknown input → treat as "next" so we never block forever in tests / piped runs.
+        return "2"
+
+    def _read_follow_up(self) -> str:
+        self._emit(
+            f"\nType your follow-up question, end with a line containing only: {self._end_marker}"
+        )
+        return _read_until_marker(self._stdin, self._end_marker)