arkaos 4.0.2 → 4.1.0

package/core/runtime/codex_cli.py

@@ -1,10 +1,34 @@
 """Codex CLI runtime adapter.
 
 OpenAI's Codex CLI. Supports sandboxed execution and file operations.
-More limited than Claude Code: no native hooks, no MCP servers.
+More limited than Claude Code: no native hooks, no ArkaOS agent dispatch.
+
+Headless invocation (verified against the installed binary,
+codex-cli 0.142.5, `codex exec --help`, 2026-07-04):
+
+    codex exec --json --skip-git-repo-check --sandbox read-only "<prompt>"
+
+``--json`` prints events to stdout as JSONL. Live-verified event shapes:
+
+    {"type":"item.completed","item":{"type":"agent_message","text":"..."}}
+    {"type":"turn.completed","usage":{"input_tokens":N,
+        "cached_input_tokens":N,"output_tokens":N,
+        "reasoning_output_tokens":N}}
+
+Older Codex CLI builds emitted ``{"msg":{"type":"agent_message",
+"message":"..."}}`` — parsed defensively too. When no JSONL parses at
+all, stdout is treated as raw text with a ``len(text) // 4`` token
+estimate (same fallback as the Gemini adapter). stdin is closed
+explicitly: with a prompt argument AND piped stdin, ``codex exec``
+appends stdin as a ``<stdin>`` block, which a hook/daemon caller must
+never trigger.
 """
 
+from __future__ import annotations
+
+import json
 import shutil
+import subprocess
 from pathlib import Path
 from os.path import expanduser
 from typing import TYPE_CHECKING
@@ -15,6 +39,14 @@ if TYPE_CHECKING:
     from core.runtime.llm_provider import LLMResponse
 
 
+# codex exec runs a full agent turn (reasoning + sandbox bootstrap) — the
+# live-verified trivial prompt took ~15s, so 60s (the Gemini budget) is
+# too tight a ceiling for real cognitive-layer prompts.
+_TIMEOUT_SECONDS = 120
+_TOKEN_ESTIMATE_DIVISOR = 4  # Rough chars-per-token heuristic.
+_STDERR_CLIP = 200
+
+
 class CodexCliAdapter(RuntimeAdapter):
     """Adapter for OpenAI's Codex CLI."""
 
@@ -32,6 +64,14 @@ class CodexCliAdapter(RuntimeAdapter):
             max_context_tokens=200_000,
         )
 
+    def capabilities(self) -> dict[str, bool]:
+        return {
+            "agent_dispatch": False,  # descriptor only — no ArkaOS agents
+            "headless": True,         # codex exec --json (real, PR-6)
+            "file_ops": True,         # native, sandboxed, interactive use
+            "hooks": False,
+        }
+
     def inject_context(self, layers: dict[str, str]) -> str:
         """Codex receives context via AGENTS.md instruction file."""
         parts = []
@@ -76,15 +116,6 @@ class CodexCliAdapter(RuntimeAdapter):
         raise NotImplementedError("Use Codex CLI's native content search")
 
     def headless_supported(self) -> bool:
-        # Auto-detect: headless is supported iff the `codex` binary is
-        # on PATH. When the operator installs Codex CLI later, this
-        # lights up without any code change (the headless_complete()
-        # method below already gates on shutil.which() too, so a missing
-        # binary will raise cleanly).
-        #
-        # Note: even when the binary is present, headless_complete()
-        # still raises until the invocation syntax is verified locally.
-        # See TODO(llm-agnostic) below for the verification checklist.
         return shutil.which("codex") is not None
 
     def headless_complete(
@@ -94,37 +125,174 @@ class CodexCliAdapter(RuntimeAdapter):
         max_tokens: int = 2000,
         system: str = "",
     ) -> "LLMResponse":
+        """One-shot completion via `codex exec --json`.
+
+        Verified against Codex CLI 0.142.5 (`codex exec --help` + a live
+        JSONL probe) on 2026-07-04. Sandbox is read-only and the git-repo
+        check is skipped so the call works from any cwd. Raises
+        LLMUnavailable on non-zero exit, timeout, or error-only streams.
+        """
+        from core.runtime.llm_provider import LLMUnavailable
+
         binary = shutil.which("codex")
         if binary is None:
             raise NotImplementedError(
                 "codex CLI not found on PATH — install Codex CLI to "
                 "enable headless completion."
             )
-        # TODO(llm-agnostic): Implement real headless completion.
-        #
-        # Status as of 2026-05-25 (PR60): Codex CLI still not verified
-        # in any ArkaOS dev environment. headless_supported() now
-        # auto-detects the binary on PATH so this lights up the moment
-        # someone installs it — but the actual subprocess call below
-        # still needs syntax verification before we can stop refusing.
-        #
-        # Verification checklist for whoever picks this up:
-        #   1. Install:   npm install -g @openai/codex-cli
-        #   2. Discover:  codex --help    (confirm non-interactive flag)
-        #   3. Pattern:   likely `codex exec "<prompt>"` or
-        #                 `codex --prompt "<prompt>" --format json`
-        #   4. Wire the subprocess call (mirror the Gemini adapter —
-        #      list-form args, 60s timeout, stderr clipped, JSON parse
-        #      with plain-text fallback, token estimate on miss).
-        #
-        # SubagentProvider cleanly falls back to anthropic-direct or
-        # stub when this raises, so the chain keeps working.
-        raise NotImplementedError(
-            "Codex CLI headless mode requires verified invocation syntax. "
-            "The `codex` binary is on PATH but ArkaOS has not validated "
-            "the non-interactive call shape locally. "
-            "Verification steps: `codex --help`, then update "
-            "core/runtime/codex_cli.py::headless_complete to call the "
-            "discovered subprocess shape. "
-            "SubagentProvider will cleanly fall back to anthropic-direct or stub."
+        effective_prompt = _merge_system_prompt(prompt, system)
+        cmd = [
+            binary, "exec", "--json", "--skip-git-repo-check",
+            "--sandbox", "read-only", effective_prompt,
+        ]
+        proc = _run_codex_cli(cmd)
+        if proc.returncode != 0:
+            stderr_tail = proc.stderr.strip()[:_STDERR_CLIP]
+            raise LLMUnavailable(
+                f"codex CLI exited {proc.returncode}: {stderr_tail}"
+            )
+        return _parse_codex_cli_output(proc.stdout)
+
+
+def _merge_system_prompt(prompt: str, system: str) -> str:
+    # codex exec takes a single prompt argument; prepend the system text
+    # when provided so downstream behaviour matches the other adapters.
+    if not system:
+        return prompt
+    return f"{system}\n\n---\n\n{prompt}"
+
+
+def _run_codex_cli(cmd: list[str]) -> subprocess.CompletedProcess:
+    from core.runtime.llm_provider import LLMUnavailable
+
+    try:
+        return subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=_TIMEOUT_SECONDS,
+            check=False,
+            # CRITICAL: with a prompt arg AND piped stdin, codex exec
+            # appends stdin as a <stdin> block — close it explicitly.
+            stdin=subprocess.DEVNULL,
         )
+    except subprocess.TimeoutExpired as exc:
+        raise LLMUnavailable(
+            f"codex CLI timed out after {_TIMEOUT_SECONDS}s"
+        ) from exc
+    except OSError as exc:
+        raise LLMUnavailable(f"codex CLI subprocess failed: {exc}") from exc
+
+
+def _agent_message_from_event(event: dict) -> str | None:
+    """Extract agent message text from a JSONL event (both shapes)."""
+    item = event.get("item")
+    if isinstance(item, dict) and item.get("type") == "agent_message":
+        return str(item.get("text") or "")
+    msg = event.get("msg")
+    if isinstance(msg, dict) and msg.get("type") == "agent_message":
+        return str(msg.get("message") or "")
+    return None
+
+
+def _usage_from_event(event: dict) -> dict | None:
+    """Extract the usage block from a turn.completed / token_count event."""
+    if event.get("type") == "turn.completed":
+        usage = event.get("usage")
+        if isinstance(usage, dict):
+            return usage
+    msg = event.get("msg")
+    if isinstance(msg, dict) and msg.get("type") == "token_count":
+        info = msg.get("info")
+        if isinstance(info, dict):
+            total = info.get("total_token_usage")
+            if isinstance(total, dict):
+                return total
+    return None
+
+
+def _error_from_event(event: dict) -> str | None:
+    if event.get("type") == "error":
+        return str(event.get("message") or "codex CLI error")
+    item = event.get("item")
+    if isinstance(item, dict) and item.get("type") == "error":
+        return str(item.get("message") or "codex CLI error")
+    return None
+
+
+def _parse_codex_cli_output(stdout: str) -> "LLMResponse":
+    from core.runtime.llm_provider import LLMResponse, LLMUnavailable
+
+    stripped = stdout.strip()
+    if not stripped:
+        return LLMResponse(
+            text="", tokens_in=0, tokens_out=0, cached_tokens=0, model=""
+        )
+
+    text: str | None = None
+    usage: dict | None = None
+    error: str | None = None
+    parsed_any = False
+    for line in stripped.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            event = json.loads(line)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if not isinstance(event, dict):
+            continue
+        parsed_any = True
+        message = _agent_message_from_event(event)
+        if message is not None:
+            text = message  # last agent message wins
+        found_usage = _usage_from_event(event)
+        if found_usage is not None:
+            usage = found_usage
+        found_error = _error_from_event(event)
+        if found_error is not None:
+            error = found_error
+
+    if not parsed_any:
+        # Non-JSONL fallback: treat stdout as raw text, estimate tokens.
+        return _response_from_plain_text(stripped)
+    if text is None:
+        # Error items can coexist with a successful agent message (e.g.
+        # skills-budget warnings) — only fail when NO message arrived.
+        raise LLMUnavailable(
+            f"codex CLI returned no agent message"
+            f"{': ' + error[:_STDERR_CLIP] if error else ''}"
+        )
+
+    tokens_in, tokens_out, cached = _token_counts(usage, text)
+    return LLMResponse(
+        text=text,
+        tokens_in=tokens_in,
+        tokens_out=tokens_out,
+        cached_tokens=cached,
+        model="",  # the JSONL stream does not report the model id
+    )
+
+
+def _response_from_plain_text(text: str) -> "LLMResponse":
+    from core.runtime.llm_provider import LLMResponse
+
+    estimate = max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR)
+    return LLMResponse(
+        text=text, tokens_in=0, tokens_out=estimate, cached_tokens=0, model=""
+    )
+
+
+def _token_counts(usage: dict | None, text: str) -> tuple[int, int, int]:
+    if not isinstance(usage, dict):
+        return 0, max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR), 0
+    try:
+        tokens_in = int(usage.get("input_tokens") or 0)
+        tokens_out = int(usage.get("output_tokens") or 0)
+        cached = int(usage.get("cached_input_tokens") or 0)
+    except (TypeError, ValueError):
+        return 0, max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR), 0
+    if tokens_in == 0 and tokens_out == 0:
+        return 0, max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR), cached
+    return tokens_in, tokens_out, cached

package/core/runtime/cost_governor.py

@@ -0,0 +1,184 @@
+"""CostGovernor — enforceable, opt-in budget caps over LLM cost telemetry.
+
+Extends ADR-011 ("token budgets are informational, not restrictive")
+WITHOUT revoking it: when no cap is configured in `~/.arkaos/config.json`
+the governor always allows (advisory stance preserved). Operators opt in
+via:
+
+    {"budget": {"hardCapUsd": 20.0, "dailyCapUsd": 50.0, "hardDeny": false}}
+
+- `budget.hardCapUsd`  — per-session cap (matched on `session_id`)
+- `budget.dailyCapUsd` — cap over today's total spend
+- `budget.hardDeny`    — when true, an exceeded cap DENIES (hook exit 2);
+                          default false = WARN-only (never blocks)
+
+Spend is read from `core.runtime.llm_cost_telemetry`: daily totals via
+`summarise(period="today")`, session totals by summing that session's
+rows (the summary's `by_session` list is top-10-capped, so a direct sum
+is the correct per-session aggregate). Missing telemetry, missing config
+or unparsable values NEVER block — the governor fails open.
+
+CLI:  python -m core.runtime.cost_governor <session_id> [--json]
+Exit: 0 allow / 3 deny.
+
+ADR: docs/adr/2026-07-04-cost-governor.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+from core.runtime.llm_cost_telemetry import read_entries, summarise
+
+
+DEFAULT_CONFIG_PATH = Path.home() / ".arkaos" / "config.json"
+
+
+@dataclass(frozen=True)
+class GovernorDecision:
+    """Outcome of one budget check."""
+
+    allow: bool
+    reason: str
+    spent_usd: float
+    cap_usd: float | None
+    hard_deny: bool = False
+
+    @property
+    def exceeded(self) -> bool:
+        return self.reason.endswith("cap-exceeded")
+
+    def to_warning(self) -> str:
+        """WARN string for hook additionalContext/stderr; "" when under cap."""
+        if not self.exceeded or self.cap_usd is None:
+            return ""
+        return (
+            f"[arka:warn] budget cap exceeded "
+            f"(${self.spent_usd:.2f} of ${self.cap_usd:.2f}) — {self.reason}"
+        )
+
+
+# ─── Config ───────────────────────────────────────────────────────────
+
+
+def _read_budget_config(config_path: Path | None) -> dict[str, Any]:
+    path = config_path or DEFAULT_CONFIG_PATH
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    budget = data.get("budget")
+    return budget if isinstance(budget, dict) else {}
+
+
+def _as_cap(raw: Any) -> float | None:
+    """Parse a cap value; absent/null/non-positive/invalid → no cap."""
+    if raw is None:
+        return None
+    try:
+        value = float(raw)
+    except (TypeError, ValueError):
+        return None
+    return value if value > 0 else None
+
+
+# ─── Spend aggregation (fails open) ───────────────────────────────────
+
+
+def _session_spend(session_id: str, telemetry_path: Path | None) -> float:
+    total = 0.0
+    for entry in read_entries(telemetry_path):
+        if str(entry.get("session_id") or "") != session_id:
+            continue
+        cost = entry.get("estimated_cost_usd")
+        if cost is None:
+            continue
+        try:
+            total += float(cost)
+        except (TypeError, ValueError):
+            continue
+    return total
+
+
+def _today_spend(telemetry_path: Path | None) -> float:
+    summary = summarise(period="today", path=telemetry_path)
+    return float(summary.total_cost_usd or 0.0)
+
+
+# ─── Public check ─────────────────────────────────────────────────────
+
+
+def check(
+    session_id: str,
+    config_path: Path | None = None,
+    telemetry_path: Path | None = None,
+) -> GovernorDecision:
+    """Evaluate the configured caps against recorded spend.
+
+    No caps configured → allow ("no-cap"). Telemetry unreadable →
+    allow ("telemetry-unavailable") — never block on missing data.
+    Exceeded cap → allow with warning unless `budget.hardDeny` is true.
+    """
+    budget = _read_budget_config(config_path)
+    session_cap = _as_cap(budget.get("hardCapUsd"))
+    daily_cap = _as_cap(budget.get("dailyCapUsd"))
+    hard_deny = bool(budget.get("hardDeny", False))
+    if session_cap is None and daily_cap is None:
+        return GovernorDecision(True, "no-cap", 0.0, None)
+
+    try:
+        session_spent = _session_spend(str(session_id or ""), telemetry_path)
+        today_spent = _today_spend(telemetry_path)
+    except Exception:  # noqa: BLE001 — fail open, per ADR
+        return GovernorDecision(True, "telemetry-unavailable", 0.0, None)
+
+    if session_cap is not None and session_spent > session_cap:
+        return GovernorDecision(
+            allow=not hard_deny, reason="session-cap-exceeded",
+            spent_usd=round(session_spent, 6), cap_usd=session_cap,
+            hard_deny=hard_deny,
+        )
+    if daily_cap is not None and today_spent > daily_cap:
+        return GovernorDecision(
+            allow=not hard_deny, reason="daily-cap-exceeded",
+            spent_usd=round(today_spent, 6), cap_usd=daily_cap,
+            hard_deny=hard_deny,
+        )
+    return GovernorDecision(
+        allow=True, reason="under-cap",
+        spent_usd=round(session_spent, 6),
+        cap_usd=session_cap if session_cap is not None else daily_cap,
+        hard_deny=hard_deny,
+    )
+
+
+# ─── CLI ──────────────────────────────────────────────────────────────
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m core.runtime.cost_governor",
+        description="Check session/daily budget caps. Exit 0 allow, 3 deny.",
+    )
+    parser.add_argument("session_id", help="session id to check")
+    parser.add_argument("--json", action="store_true", dest="as_json")
+    args = parser.parse_args(argv)
+    decision = check(args.session_id)
+    if args.as_json:
+        print(json.dumps(asdict(decision)))
+    else:
+        cap = f"${decision.cap_usd:.2f}" if decision.cap_usd is not None else "none"
+        verdict = "ALLOW" if decision.allow else "DENY"
+        print(
+            f"{verdict} {decision.reason} "
+            f"(spent ${decision.spent_usd:.2f}, cap {cap})"
+        )
+    return 0 if decision.allow else 3
+
+
+if __name__ == "__main__":  # pragma: no cover — thin CLI shim
+    raise SystemExit(main())

package/core/runtime/cursor.py

@@ -31,6 +31,14 @@ class CursorAdapter(RuntimeAdapter):
             max_context_tokens=200_000,
         )
 
+    def capabilities(self) -> dict[str, bool]:
+        return {
+            "agent_dispatch": False,  # single-agent mode only
+            "headless": False,        # no headless CLI as of 2026-07
+            "file_ops": True,         # IDE agent mode edits files
+            "hooks": False,           # no PreToolUse/Stop equivalent
+        }
+
     def inject_context(self, layers: dict[str, str]) -> str:
         """Cursor receives context via .cursorrules files."""
         parts = []

package/core/runtime/gemini_cli.py

@@ -53,6 +53,14 @@ class GeminiCliAdapter(RuntimeAdapter):
             max_context_tokens=1_000_000,
         )
 
+    def capabilities(self) -> dict[str, bool]:
+        return {
+            "agent_dispatch": False,  # descriptor only — no ArkaOS agents
+            "headless": True,         # gemini -p --output-format json
+            "file_ops": True,         # native, interactive use
+            "hooks": False,
+        }
+
     def inject_context(self, layers: dict[str, str]) -> str:
         """Gemini receives context via GEMINI.md instruction file."""
         parts = []

package/core/runtime/llm_provider.py

@@ -27,6 +27,7 @@ from typing import Protocol, runtime_checkable
 from core.runtime import registry
 from core.runtime.base import RuntimeAdapter
 from core.runtime.llm_cost_telemetry import record_cost
+from core.runtime.llm_retry import retry_completion
 from core.runtime.pricing import estimate_cost_usd
 
 
@@ -116,18 +117,27 @@ class SubagentProvider:
         system: str = "",
     ) -> LLMResponse:
         adapter = self._resolve_adapter()
-        try:
-            response = adapter.headless_complete(
-                prompt, max_tokens=max_tokens, system=system
-            )
-        except NotImplementedError as exc:
-            raise LLMUnavailable(str(exc)) from exc
-        except LLMUnavailable:
-            raise
-        except Exception as exc:  # noqa: BLE001
-            raise LLMUnavailable(
-                f"headless_complete failed: {exc.__class__.__name__}: {exc}"
-            ) from exc
+
+        def _attempt() -> LLMResponse:
+            try:
+                return adapter.headless_complete(
+                    prompt, max_tokens=max_tokens, system=system
+                )
+            except NotImplementedError as exc:
+                raise LLMUnavailable(str(exc)) from exc
+            except LLMUnavailable:
+                raise
+            except Exception as exc:  # noqa: BLE001
+                raise LLMUnavailable(
+                    f"headless_complete failed: {exc.__class__.__name__}: {exc}"
+                ) from exc
+
+        # Rate-limit-looking CLI failures (non-zero exit with 429/rate
+        # limit/overloaded stderr) surface as LLMUnavailable messages the
+        # default classifier recognises as retryable; everything else
+        # fails fast. Exhaustion re-raises LLMUnavailable (attempt count
+        # + last error) preserving the fallback-chain semantics.
+        response = retry_completion(_attempt, provider=self.name())
 
         _record(
             session_id=os.environ.get("ARKA_SESSION_ID", ""),
@@ -227,8 +237,17 @@ class AnthropicDirectProvider:
             )
         client = self._build_client()
         payload = self._build_anthropic_payload(prompt, system, max_tokens, model)
+        # Retry the raw SDK call (not a pre-wrapped LLMUnavailable) so the
+        # classifier can inspect status_code / response.headers for 429 +
+        # retry-after. Non-retryable SDK errors convert to LLMUnavailable
+        # below; exhaustion already raises LLMUnavailable inside the layer.
         try:
-            raw = client.messages.create(**payload)  # type: ignore[attr-defined]
+            raw = retry_completion(
+                lambda: client.messages.create(**payload),  # type: ignore[attr-defined]
+                provider=self.name(),
+            )
+        except LLMUnavailable:
+            raise
         except Exception as exc:  # noqa: BLE001
             raise LLMUnavailable(
                 f"anthropic.messages.create failed: {exc.__class__.__name__}: {exc}"