delimit-cli 4.4.0 → 4.5.0

package/gateway/ai/inbox_executor.py

@@ -0,0 +1,565 @@
+"""Inbox executor — LED-1134 Phase 2.
+
+Closes the email→action loop: consumes the inbox-drafts registry written
+by Phase 1 and dispatches autonomous actions when founder Ship-it
+replies have transitioned drafts from pending → approved.
+
+Constitutional reference: docs/inbox_executor_v1.md is the source of
+truth for the wire contract, state machine, allowlist, and non-delegable
+refusal list. Authorized via owner attestation 2026-04-26T02:49Z
+(scope=authority_class_expansion, evidence_ref=LED-1134).
+
+DESIGN INTENT (per the strategic + operational deliberations):
+
+1. Separate process from inbox_daemon — daemon parses untrusted email
+   (large attack surface); executor performs privileged actions (small
+   attack surface). 3-1 panel vote against in-process consolidation.
+
+2. Re-verify HMAC + TTL at execute time, not just at insert time.
+   A draft sitting in the DB for 23h59m must NOT execute when it's
+   24h+1m stale by the time we get to it.
+
+3. Atomic transition approved → executing BEFORE the side effect.
+   SQLite UPDATE with rowcount=1 wins; rowcount=0 means another
+   instance already took it. At-most-once.
+
+4. Crash mid-execute leaves the row at status=executing for human
+   reconciliation. NO auto-retry — that turns at-most-once into
+   at-least-once.
+
+5. Non-delegable refusal list (per CLAUDE.md "Non-Delegable Decisions"):
+   force_push_shared, ruleset_disable, account_switch, cross_account_ops,
+   irreversible_capital_commit, constitutional_rewrite,
+   authority_class_expansion, venture_kill, permission_escalation,
+   public_truth_claim. ANY of these refuse, log, email founder for
+   fresh attestation through different channel.
+
+6. Thermal cutout — pause if more than N actions in T seconds. v1
+   default: 10 actions / 15 minutes (Haiku's original 3-in-5min would
+   trip on legitimate batch sweeps).
+
+7. Allowlist of dispatch handlers — only github_comment is wired in
+   PR-A; others land progressively.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import shlex
+import subprocess
+import threading
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from ai.inbox_drafts import (
+    DraftRow,
+    DraftStatus,
+    list_drafts,
+    record_attempt,
+    transition,
+    verify_draft,
+)
+
+logger = logging.getLogger("delimit.inbox_executor")
+
+# State path for the daemon-thread control. Mirrors the inbox_daemon
+# convention so operators can find both files in the same place.
+STATE_PATH = Path.home() / ".delimit" / "inbox_executor_state.json"
+DEFAULT_POLL_INTERVAL_SECONDS = 30
+
+
+# ── Constitutional refusal list ──────────────────────────────────────
+
+
+# Mirrors ai.governance.NON_DELEGABLE_OPERATION_CLASSES. We hard-code the
+# set here so the executor can refuse without an import dependency that
+# could theoretically be tampered. Both lists must stay in sync; the spec
+# doc (docs/inbox_executor_v1.md) is the single source of truth.
+NON_DELEGABLE_REFUSAL_LIST = frozenset({
+    "force_push_shared",
+    "ruleset_disable",
+    "account_switch",
+    "cross_account_ops",
+    "irreversible_capital_commit",
+    "constitutional_rewrite",
+    "authority_class_expansion",
+    "venture_kill",
+    "permission_escalation",
+    "public_truth_claim",
+})
+
+
+# ── Thermal cutout ────────────────────────────────────────────────────
+
+
+@dataclass
+class ThermalState:
+    """Tracks recent action timestamps to detect bursts.
+
+    Default: 10 actions / 15 minutes. Above the threshold the executor
+    self-pauses for a cooldown. Per the deliberation: 3-in-5min would
+    trip on legitimate batch sweeps (founder clearing 5–8 morning
+    approvals at once); 10-in-15min is a real burst.
+    """
+
+    threshold_count: int = 10
+    threshold_seconds: int = 15 * 60
+    cooldown_seconds: int = 5 * 60
+    recent_action_times: List[int] = field(default_factory=list)
+    paused_until: int = 0
+
+    def record(self, now: Optional[int] = None) -> None:
+        if now is None:
+            now = int(time.time())
+        self.recent_action_times.append(now)
+        # Drop entries older than the window.
+        cutoff = now - self.threshold_seconds
+        self.recent_action_times = [t for t in self.recent_action_times if t >= cutoff]
+        if len(self.recent_action_times) > self.threshold_count:
+            self.paused_until = now + self.cooldown_seconds
+            logger.warning(
+                "thermal cutout tripped: %d actions in last %ds; pausing %ds",
+                len(self.recent_action_times),
+                self.threshold_seconds,
+                self.cooldown_seconds,
+            )
+
+    def is_paused(self, now: Optional[int] = None) -> bool:
+        if now is None:
+            now = int(time.time())
+        return now < self.paused_until
+
+
+# ── Dispatch handlers ────────────────────────────────────────────────
+
+
+# A dispatch handler is a callable: (DraftRow) -> (ok: bool, executed_url: Optional[str], reason: Optional[str])
+# Pure functions — no shared state — to keep the executor predictable.
+DispatchHandler = Callable[[DraftRow], Tuple[bool, Optional[str], Optional[str]]]
+
+
+def _dispatch_github_comment(row: DraftRow) -> Tuple[bool, Optional[str], Optional[str]]:
+    """Post a GitHub issue comment via gh CLI.
+
+    Payload schema: {"body": "..."}
+    Target schema:  {"repo": "owner/name", "issue": <int>}
+    Returns the resulting comment URL on success.
+    """
+    repo = row.target.get("repo")
+    issue = row.target.get("issue")
+    body = (row.payload or {}).get("body") if isinstance(row.payload, dict) else None
+    if not (repo and issue and body):
+        return False, None, "github_comment requires target.repo, target.issue, payload.body"
+
+    cmd = ["gh", "issue", "comment", str(issue), "--repo", repo, "--body", body]
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+    except FileNotFoundError:
+        return False, None, "gh CLI not found"
+    except subprocess.TimeoutExpired:
+        return False, None, "gh issue comment timed out"
+
+    if result.returncode != 0:
+        stderr = (result.stderr or "").strip()[:300]
+        return False, None, f"gh issue comment failed: {stderr}"
+
+    # gh prints the URL on stdout, e.g.
+    # "https://github.com/owner/repo/issues/123#issuecomment-..."
+    url = (result.stdout or "").strip().splitlines()[-1] if result.stdout else None
+    return True, url, None
+
+
+def _dispatch_unimplemented(row: DraftRow) -> Tuple[bool, Optional[str], Optional[str]]:
+    """Placeholder for kinds the executor knows about but hasn't wired yet.
+
+    PR-A ships only github_comment. The other allowlist kinds
+    (social_post, ledger_done, notify_routing_update,
+    deploy_publish_prevalidated_artifact) will be wired in subsequent PRs;
+    for now they refuse loudly so the founder isn't surprised when
+    "Ship it" doesn't fire on a kind we haven't built yet.
+    """
+    return False, None, f"dispatch handler for kind={row.draft_kind} not implemented in PR-A"
+
+
+# Dispatch table. Adding a new key here is itself an authority_class_expansion
+# event — the spec doc must be updated and a fresh attestation logged.
+DISPATCH_TABLE: Dict[str, DispatchHandler] = {
+    "github_comment": _dispatch_github_comment,
+    "social_post": _dispatch_unimplemented,
+    "ledger_done": _dispatch_unimplemented,
+    "notify_routing_update": _dispatch_unimplemented,
+    "deploy_publish_prevalidated_artifact": _dispatch_unimplemented,
+}
+
+
+# ── Failure-notification hook ────────────────────────────────────────
+
+
+# Decoupled so tests can patch it without spinning up SMTP.
+NotifyFn = Callable[[str, str], None]
+
+
+def _default_notify(subject: str, body: str) -> None:
+    """Email founder via delimit_notify with [ALERT] subject prefix.
+
+    Imported lazily so this module doesn't import the entire notify
+    surface at startup. Best-effort: failures are logged but don't
+    interrupt the executor's main loop.
+    """
+    try:
+        from ai.notify import send_notification
+        send_notification(
+            channel="email",
+            subject=subject,
+            message=body,
+            event_type="executor_alert",
+        )
+    except Exception:
+        logger.exception("failure-notification hook itself failed; logging only")
+
+
+# ── Core executor cycle ──────────────────────────────────────────────
+
+
+def _execute_one(
+    row: DraftRow,
+    *,
+    notify: NotifyFn,
+) -> Dict[str, Any]:
+    """Process one approved draft. Returns a result dict for diagnostics.
+
+    Order of operations (the at-most-once contract):
+
+    1. Re-verify HMAC + TTL (drafts may have been signed long ago).
+    2. Refuse non-delegable kinds.
+    3. Atomically transition approved → executing. Lose the race → no-op.
+    4. Run the dispatch handler (the actual side effect).
+    5. Transition executing → completed (with executed_url) OR
+       executing → completed_with_error (with last_error) + email founder.
+
+    A crash between steps 3 and 5 leaves the row stuck at status=executing
+    for human reconciliation — we never auto-retry from executing.
+    """
+    out: Dict[str, Any] = {"draft_id": row.draft_id, "kind": row.draft_kind}
+
+    # Step 1: re-verify
+    ok, reason = verify_draft(row.to_signed_dict())
+    record_attempt(row.draft_id, kind="verify", outcome=("ok" if ok else "failed"), reason=reason)
+    if not ok:
+        out["outcome"] = "verify_failed"
+        out["reason"] = reason
+        # The draft was approved earlier (HMAC was good then) but is no
+        # longer verifiable now (TTL elapsed, or signature mismatch from
+        # tampering). Mark it terminal; do NOT execute.
+        transition(
+            row.draft_id,
+            expected=DraftStatus.APPROVED.value,
+            new=DraftStatus.COMPLETED_WITH_ERROR.value,
+            last_error=f"verify failed at execute time: {reason}",
+            completed=True,
+        )
+        notify(
+            f"[ALERT] Inbox executor refused {row.draft_id}",
+            f"Draft kind={row.draft_kind} failed re-verify at execute time:\n\n{reason}\n\n"
+            f"Marked completed_with_error. No retry.",
+        )
+        return out
+
+    # Step 2: refusal list
+    if row.draft_kind in NON_DELEGABLE_REFUSAL_LIST:
+        out["outcome"] = "refused_non_delegable"
+        out["reason"] = f"{row.draft_kind} is non-delegable per STR-183"
+        transition(
+            row.draft_id,
+            expected=DraftStatus.APPROVED.value,
+            new=DraftStatus.TERMINAL_UNRECOVERABLE.value,
+            last_error="kind is on the non-delegable refusal list",
+            completed=True,
+        )
+        notify(
+            f"[ALERT] Inbox executor refused {row.draft_id}",
+            f"Draft kind={row.draft_kind} is on the non-delegable refusal list. "
+            f"This action requires fresh per-invocation founder attestation through "
+            f"a different channel (not email Ship-it).",
+        )
+        return out
+
+    # Step 3: take the row atomically. Lose the race → no-op.
+    took = transition(
+        row.draft_id,
+        expected=DraftStatus.APPROVED.value,
+        new=DraftStatus.EXECUTING.value,
+    )
+    if not took:
+        out["outcome"] = "lost_race"
+        return out
+
+    # Step 4: dispatch
+    handler = DISPATCH_TABLE.get(row.draft_kind, _dispatch_unimplemented)
+    ok, executed_url, reason = handler(row)
+    record_attempt(
+        row.draft_id,
+        kind="execute",
+        outcome=("ok" if ok else "failed"),
+        reason=reason,
+        executed_url=executed_url,
+    )
+
+    # Step 5: terminal transition
+    if ok:
+        transition(
+            row.draft_id,
+            expected=DraftStatus.EXECUTING.value,
+            new=DraftStatus.COMPLETED.value,
+            executed_url=executed_url,
+            completed=True,
+        )
+        out["outcome"] = "executed"
+        out["executed_url"] = executed_url
+    else:
+        transition(
+            row.draft_id,
+            expected=DraftStatus.EXECUTING.value,
+            new=DraftStatus.COMPLETED_WITH_ERROR.value,
+            last_error=reason,
+            completed=True,
+        )
+        notify(
+            f"[ALERT] Inbox executor failed {row.draft_id}",
+            f"Draft kind={row.draft_kind} failed during dispatch:\n\n{reason}\n\n"
+            f"Marked completed_with_error. No retry — please re-trigger manually if needed.",
+        )
+        out["outcome"] = "execute_failed"
+        out["reason"] = reason
+
+    return out
+
+
+def run_cycle(
+    *,
+    thermal: ThermalState,
+    batch_limit: int = 10,
+    notify: Optional[NotifyFn] = None,
+) -> Dict[str, Any]:
+    """One pass of the executor poll loop.
+
+    Picks up to `batch_limit` approved drafts and processes each. Updates
+    thermal state on every action. Returns a summary dict suitable for
+    logging or status tooling.
+
+    Designed to be safe to call from a 30s scheduler/timer outside this
+    process (cron / systemd / supervisor).
+    """
+    notify_fn = notify or _default_notify
+
+    if thermal.is_paused():
+        return {
+            "status": "paused",
+            "paused_until": thermal.paused_until,
+            "processed": 0,
+        }
+
+    approved = list_drafts(status=DraftStatus.APPROVED.value, limit=batch_limit)
+    if not approved:
+        return {"status": "idle", "processed": 0}
+
+    results: List[Dict[str, Any]] = []
+    for row in approved:
+        if thermal.is_paused():
+            results.append({
+                "draft_id": row.draft_id,
+                "kind": row.draft_kind,
+                "outcome": "deferred_thermal",
+            })
+            continue
+        try:
+            r = _execute_one(row, notify=notify_fn)
+        except Exception as e:
+            # Cardinal rule: never let one bad draft kill the loop.
+            # The row stays at whatever state we last transitioned it
+            # to — likely executing if we crashed inside dispatch —
+            # which surfaces it for human reconciliation.
+            logger.exception("execute_one raised for %s", row.draft_id)
+            r = {
+                "draft_id": row.draft_id,
+                "kind": row.draft_kind,
+                "outcome": "exception",
+                "reason": f"{type(e).__name__}: {e}",
+            }
+        results.append(r)
+        # Only count actual side-effect attempts toward thermal, not
+        # refusals or verify-failures (those don't reach an external
+        # service).
+        if r.get("outcome") in {"executed", "execute_failed"}:
+            thermal.record()
+
+    return {
+        "status": "ran",
+        "processed": len(results),
+        "results": results,
+    }
+
+
+# ── Daemon control surface ───────────────────────────────────────────
+
+
+@dataclass
+class _ExecutorState:
+    """Thread-safe state for the daemon's start/stop/status surface.
+
+    Mirrors inbox_daemon's pattern. Writes to STATE_PATH on every cycle
+    so an operator who can't import this module can still cat the file
+    and see what's happening.
+    """
+
+    running: bool = False
+    last_cycle_at: Optional[str] = None
+    total_cycles: int = 0
+    total_processed: int = 0
+    total_executed: int = 0
+    total_failed: int = 0
+    consecutive_failures: int = 0
+    stopped_reason: Optional[str] = None
+    poll_interval_seconds: int = DEFAULT_POLL_INTERVAL_SECONDS
+    thread: Optional[threading.Thread] = None
+    stop_event: Optional[threading.Event] = None
+    thermal: Optional[ThermalState] = None
+    _lock: threading.Lock = field(default_factory=threading.Lock)
+
+    def to_status_dict(self) -> Dict[str, Any]:
+        with self._lock:
+            return {
+                "running": self.running,
+                "last_cycle_at": self.last_cycle_at,
+                "total_cycles": self.total_cycles,
+                "total_processed": self.total_processed,
+                "total_executed": self.total_executed,
+                "total_failed": self.total_failed,
+                "consecutive_failures": self.consecutive_failures,
+                "stopped_reason": self.stopped_reason,
+                "poll_interval_seconds": self.poll_interval_seconds,
+                "thermal_paused_until": (
+                    self.thermal.paused_until if self.thermal else None
+                ),
+            }
+
+    def persist(self) -> None:
+        try:
+            STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
+            STATE_PATH.write_text(json.dumps(self.to_status_dict(), indent=2))
+        except Exception:
+            logger.exception("could not persist executor state")
+
+
+_state = _ExecutorState()
+
+
+def _daemon_loop(state: _ExecutorState, stop_event: threading.Event) -> None:
+    """Run forever (until stop_event), invoking run_cycle each tick.
+
+    Records cycle count + outcome stats on the shared state and writes
+    a status file every cycle so external tools can monitor progress.
+    """
+    while not stop_event.is_set():
+        try:
+            result = run_cycle(thermal=state.thermal)
+            with state._lock:
+                state.total_cycles += 1
+                state.last_cycle_at = datetime.now(timezone.utc).isoformat()
+                if result.get("status") == "ran":
+                    state.total_processed += result.get("processed", 0)
+                    for r in result.get("results", []):
+                        if r.get("outcome") == "executed":
+                            state.total_executed += 1
+                        elif r.get("outcome") == "execute_failed":
+                            state.total_failed += 1
+                state.consecutive_failures = 0
+        except Exception:
+            with state._lock:
+                state.consecutive_failures += 1
+                state.last_cycle_at = datetime.now(timezone.utc).isoformat()
+            logger.exception("inbox_executor cycle raised")
+        state.persist()
+        # Sleep with early-exit on stop_event.
+        if stop_event.wait(timeout=state.poll_interval_seconds):
+            break
+    with state._lock:
+        state.running = False
+    state.persist()
+
+
+def start(
+    *,
+    poll_interval_seconds: int = DEFAULT_POLL_INTERVAL_SECONDS,
+    thermal_threshold_count: int = 10,
+    thermal_threshold_seconds: int = 15 * 60,
+    thermal_cooldown_seconds: int = 5 * 60,
+) -> Dict[str, Any]:
+    """Start the executor daemon thread.
+
+    Idempotent — calling start() on a running daemon returns the same
+    status without spawning a second thread. Mirrors the inbox_daemon
+    contract so the two control surfaces are operationally symmetric.
+    """
+    with _state._lock:
+        if _state.running:
+            return {**_state.to_status_dict(), "action": "already_running"}
+        _state.running = True
+        _state.stopped_reason = None
+        _state.poll_interval_seconds = poll_interval_seconds
+        _state.thermal = ThermalState(
+            threshold_count=thermal_threshold_count,
+            threshold_seconds=thermal_threshold_seconds,
+            cooldown_seconds=thermal_cooldown_seconds,
+        )
+        _state.stop_event = threading.Event()
+        _state.thread = threading.Thread(
+            target=_daemon_loop,
+            args=(_state, _state.stop_event),
+            name="inbox_executor",
+            daemon=True,
+        )
+        _state.thread.start()
+    _state.persist()
+    return {**_state.to_status_dict(), "action": "started"}
+
+
+def stop(reason: str = "manual") -> Dict[str, Any]:
+    """Stop the executor daemon. Idempotent."""
+    with _state._lock:
+        if not _state.running or not _state.stop_event:
+            return {**_state.to_status_dict(), "action": "already_stopped"}
+        _state.stop_event.set()
+        _state.stopped_reason = reason
+        thread = _state.thread
+    if thread:
+        thread.join(timeout=10.0)
+    _state.persist()
+    return {**_state.to_status_dict(), "action": "stopped"}
+
+
+def status() -> Dict[str, Any]:
+    """Return current daemon status — does not read SQLite."""
+    return _state.to_status_dict()
+
+
+def control(action: str = "status", **kwargs) -> Dict[str, Any]:
+    """Single entry-point matching the delimit_inbox_daemon pattern.
+
+    actions: 'start' (begin polling), 'stop' (halt), 'status' (show state).
+
+    kwargs are forwarded to start() (poll_interval_seconds, thermal
+    thresholds). Mostly used by tests to override the defaults.
+    """
+    action = (action or "status").lower().strip()
+    if action == "start":
+        return start(**kwargs)
+    if action == "stop":
+        return stop(**kwargs)
+    if action == "status":
+        return status()
+    return {"error": f"unknown action: {action!r}; use start|stop|status"}