arkaos 2.22.0 → 2.25.0

package/core/runtime/claude_code.py

@@ -120,7 +120,7 @@ class ClaudeCodeAdapter(RuntimeAdapter):
         max_tokens: int = 2000,
         system: str = "",
     ) -> "LLMResponse":
-        from core.runtime.llm_provider import LLMResponse, LLMUnavailable
+        from core.runtime.llm_provider import LLMUnavailable
 
         binary = shutil.which("claude")
         if binary is None:
@@ -131,27 +131,28 @@ class ClaudeCodeAdapter(RuntimeAdapter):
         cmd = [binary, "-p", prompt, "--output-format", "json"]
         if system:
             cmd.extend(["--append-system-prompt", system])
-        try:
-            proc = subprocess.run(
-                cmd,
-                capture_output=True,
-                text=True,
-                timeout=60,
-                check=False,
-            )
-        except subprocess.TimeoutExpired as exc:
-            raise LLMUnavailable("claude CLI timed out after 60s") from exc
-        except OSError as exc:
-            raise LLMUnavailable(f"claude CLI subprocess failed: {exc}") from exc
-
+        proc = _run_claude_cli(cmd)
         if proc.returncode != 0:
             raise LLMUnavailable(
                 f"claude CLI exited {proc.returncode}: {proc.stderr.strip()[:200]}"
             )
-        return _parse_claude_json(proc.stdout)
+        return _parse_claude_cli_output(proc.stdout)
+
+
+def _run_claude_cli(cmd: list[str]) -> subprocess.CompletedProcess:
+    from core.runtime.llm_provider import LLMUnavailable
+
+    try:
+        return subprocess.run(
+            cmd, capture_output=True, text=True, timeout=60, check=False
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise LLMUnavailable("claude CLI timed out after 60s") from exc
+    except OSError as exc:
+        raise LLMUnavailable(f"claude CLI subprocess failed: {exc}") from exc
 
 
-def _parse_claude_json(stdout: str) -> "LLMResponse":
+def _parse_claude_cli_output(stdout: str) -> "LLMResponse":
     from core.runtime.llm_provider import LLMResponse
 
     payload = json.loads(stdout) if stdout.strip() else {}
@@ -170,3 +171,8 @@ def _parse_claude_json(stdout: str) -> "LLMResponse":
         cached_tokens=cache_read,
         model=model,
     )
+
+
+# Backward compatibility alias — tests and external importers that used
+# the old helper name continue to work without modification.
+_parse_claude_json = _parse_claude_cli_output

package/core/runtime/codex_cli.py

@@ -94,11 +94,28 @@ class CodexCliAdapter(RuntimeAdapter):
                 "codex CLI not found on PATH — install Codex CLI to "
                 "enable headless completion."
             )
-        # TODO(llm-agnostic): Verify Codex CLI headless invocation
-        # syntax (`codex exec "<prompt>"` was the working hypothesis
-        # but has not been confirmed for the current release). Until
-        # then, refuse rather than guess. Tracked in Task #12 report.
+        # TODO(llm-agnostic): Implement real headless completion.
+        #
+        # Status as of 2026-04-20: Codex CLI is NOT installed on the
+        # development machine, so actual invocation syntax could not
+        # be verified. Until a local install is available, refuse
+        # rather than ship guessed arguments.
+        #
+        # Verification checklist for whoever picks this up:
+        #   1. Install:   npm install -g @openai/codex-cli
+        #   2. Discover:  codex --help   (confirm non-interactive flag)
+        #   3. Pattern:   likely `codex exec "<prompt>"` or
+        #                 `codex --prompt "<prompt>" --format json`
+        #   4. Wire the subprocess call (mirror the Gemini adapter —
+        #      list-form args, 60s timeout, stderr clipped, JSON parse
+        #      with plain-text fallback, token estimate on miss).
+        #
+        # SubagentProvider cleanly falls back to anthropic-direct or
+        # stub when this raises, so the chain keeps working.
         raise NotImplementedError(
-            "Codex CLI headless completion not yet wired — verify CLI "
-            "syntax before enabling. See core/runtime/codex_cli.py TODO."
+            "Codex CLI headless mode requires local `codex` CLI. "
+            "Install: `npm install -g @openai/codex-cli` (verified 2026-04-20). "
+            "Verify syntax: `codex --help`. "
+            "See TODO(llm-agnostic) in this file. "
+            "SubagentProvider will cleanly fall back to anthropic-direct or stub."
         )

package/core/runtime/gemini_cli.py

@@ -1,9 +1,26 @@
 """Gemini CLI runtime adapter.
 
 Google's Gemini CLI. Uses GEMINI.md for instructions and activate_skill for skills.
+
+Headless invocation reference (verified against
+https://github.com/google-gemini/gemini-cli docs — Context7 query
+on 2026-04-20):
+
+    gemini -p "<prompt>" --output-format json
+
+The JSON payload contains a ``response`` key (the model's text) and a
+``stats`` block with ``totalTokenCount`` / token counts. On failure the
+payload includes an ``error`` block with diagnostic details. If JSON
+parsing fails we fall back to treating stdout as raw text and estimate
+tokens via a ``len(text) // 4`` heuristic — better than losing cost
+telemetry entirely.
 """
 
+from __future__ import annotations
+
+import json
 import shutil
+import subprocess
 from pathlib import Path
 from os.path import expanduser
 from typing import TYPE_CHECKING
@@ -14,6 +31,11 @@ if TYPE_CHECKING:
     from core.runtime.llm_provider import LLMResponse
 
 
+_TIMEOUT_SECONDS = 60
+_TOKEN_ESTIMATE_DIVISOR = 4  # Rough chars-per-token heuristic.
+_STDERR_CLIP = 200
+
+
 class GeminiCliAdapter(RuntimeAdapter):
     """Adapter for Google's Gemini CLI."""
 
@@ -73,10 +95,7 @@ class GeminiCliAdapter(RuntimeAdapter):
         raise NotImplementedError("Use Gemini CLI's native content search")
 
     def headless_supported(self) -> bool:
-        # Gemini CLI headless invocation syntax is not verified for the
-        # current release. Returning False lets SubagentProvider fall
-        # back gracefully rather than shell out blindly.
-        return False
+        return shutil.which("gemini") is not None
 
     def headless_complete(
         self,
@@ -85,17 +104,122 @@ class GeminiCliAdapter(RuntimeAdapter):
         max_tokens: int = 2000,
         system: str = "",
     ) -> "LLMResponse":
+        from core.runtime.llm_provider import LLMUnavailable
+
         binary = shutil.which("gemini")
         if binary is None:
             raise NotImplementedError(
                 "gemini CLI not found on PATH — install Gemini CLI to "
                 "enable headless completion."
             )
-        # TODO(llm-agnostic): Verify Gemini CLI's headless invocation
-        # (`gemini -p "<prompt>"` was the working hypothesis). Until
-        # confirmed for the shipped CLI version, refuse rather than
-        # guess. Tracked in Task #12 report.
-        raise NotImplementedError(
-            "Gemini CLI headless completion not yet wired — verify CLI "
-            "syntax before enabling. See core/runtime/gemini_cli.py TODO."
+        effective_prompt = _merge_system_prompt(prompt, system)
+        cmd = [binary, "-p", effective_prompt, "--output-format", "json"]
+        proc = _run_gemini_cli(cmd)
+        if proc.returncode != 0:
+            stderr_tail = proc.stderr.strip()[:_STDERR_CLIP]
+            raise LLMUnavailable(
+                f"gemini CLI exited {proc.returncode}: {stderr_tail}"
+            )
+        return _parse_gemini_cli_output(proc.stdout)
+
+
+def _merge_system_prompt(prompt: str, system: str) -> str:
+    # Gemini CLI's -p flag accepts a single prompt; prepend the system
+    # text when provided so downstream behaviour matches Claude Code.
+    if not system:
+        return prompt
+    return f"{system}\n\n---\n\n{prompt}"
+
+
+def _run_gemini_cli(cmd: list[str]) -> subprocess.CompletedProcess:
+    from core.runtime.llm_provider import LLMUnavailable
+
+    try:
+        return subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=_TIMEOUT_SECONDS,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise LLMUnavailable(
+            f"gemini CLI timed out after {_TIMEOUT_SECONDS}s"
+        ) from exc
+    except OSError as exc:
+        raise LLMUnavailable(f"gemini CLI subprocess failed: {exc}") from exc
+
+
+def _parse_gemini_cli_output(stdout: str) -> "LLMResponse":
+    from core.runtime.llm_provider import LLMResponse
+
+    stripped = stdout.strip()
+    if not stripped:
+        return LLMResponse(
+            text="", tokens_in=0, tokens_out=0, cached_tokens=0, model=""
+        )
+    payload = _safe_loads(stripped)
+    if payload is None:
+        # Non-JSON fallback: treat stdout as raw text, estimate tokens.
+        return _response_from_plain_text(stripped)
+    return _response_from_json_payload(payload)
+
+
+def _safe_loads(text: str) -> dict | None:
+    try:
+        data = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _response_from_plain_text(text: str) -> "LLMResponse":
+    from core.runtime.llm_provider import LLMResponse
+
+    estimate = max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR)
+    return LLMResponse(
+        text=text,
+        tokens_in=0,
+        tokens_out=estimate,
+        cached_tokens=0,
+        model="",
+    )
+
+
+def _response_from_json_payload(payload: dict) -> "LLMResponse":
+    from core.runtime.llm_provider import LLMResponse, LLMUnavailable
+
+    error = payload.get("error")
+    if isinstance(error, dict) and error:
+        message = str(error.get("message") or error).strip()[:_STDERR_CLIP]
+        raise LLMUnavailable(f"gemini CLI returned error: {message}")
+
+    text = str(payload.get("response") or payload.get("result") or "")
+    tokens_in, tokens_out = _extract_token_counts(payload, text)
+    model = str(payload.get("model") or "")
+    return LLMResponse(
+        text=text,
+        tokens_in=tokens_in,
+        tokens_out=tokens_out,
+        cached_tokens=0,
+        model=model,
+    )
+
+
+def _extract_token_counts(payload: dict, text: str) -> tuple[int, int]:
+    stats = payload.get("stats") or payload.get("usageMetadata") or {}
+    if isinstance(stats, dict):
+        tokens_in = int(stats.get("promptTokenCount") or stats.get("input_tokens") or 0)
+        tokens_out = int(
+            stats.get("candidatesTokenCount")
+            or stats.get("output_tokens")
+            or 0
         )
+        # Fall back to the rolled-up total when per-side counts are absent.
+        if tokens_in == 0 and tokens_out == 0:
+            total = int(stats.get("totalTokenCount") or 0)
+            if total > 0:
+                return 0, total
+        return tokens_in, tokens_out
+    # No stats block at all — estimate output from text length.
+    return 0, max(1, len(text) // _TOKEN_ESTIMATE_DIVISOR)

package/core/runtime/llm_provider.py

@@ -199,6 +199,19 @@ class AnthropicDirectProvider:
             }
         ]
 
+    def _build_anthropic_payload(
+        self, prompt: str, system: str, max_tokens: int, model: str
+    ) -> dict[str, object]:
+        payload: dict[str, object] = {
+            "model": model,
+            "max_tokens": max_tokens,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+        system_blocks = self._build_system_blocks(system)
+        if system_blocks:
+            payload["system"] = system_blocks
+        return payload
+
     def complete(
         self,
         prompt: str,
@@ -213,15 +226,7 @@ class AnthropicDirectProvider:
                 "cannot select a model."
             )
         client = self._build_client()
-        payload: dict[str, object] = {
-            "model": model,
-            "max_tokens": max_tokens,
-            "messages": [{"role": "user", "content": prompt}],
-        }
-        system_blocks = self._build_system_blocks(system)
-        if system_blocks:
-            payload["system"] = system_blocks
-
+        payload = self._build_anthropic_payload(prompt, system, max_tokens, model)
         try:
             raw = client.messages.create(**payload)  # type: ignore[attr-defined]
         except Exception as exc:  # noqa: BLE001

package/core/runtime/path_resolver.py

@@ -0,0 +1,202 @@
+"""Template-token path resolver for ArkaOS.
+
+Resolves user-specific paths from ``~/.arkaos/profile.json`` and environment
+variables, so source files can use neutral tokens like ``${VAULT_PATH}`` and
+``${ARKA_OS_REPOS}`` instead of hardcoded absolute paths.
+
+Token precedence (highest → lowest):
+  1. Environment variable (e.g. ``ARKAOS_VAULT_PATH``)
+  2. profile.json field (e.g. ``vaultPath``)
+  3. For ``${GIT_HOST}`` only: hardcoded default ``github.com``
+
+If ``~/.arkaos/profile.json`` is missing or unparseable and an
+unconfigured token is requested, ``ProfileMissingError`` is raised
+with a clear remediation message. Unknown tokens pass through
+unchanged so that prompt strings containing ``${SOME_BASH_VAR}`` for
+the agent are preserved.
+
+See ``core/specs/SPEC-paths-portability.md`` for the full contract.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from core.runtime.user_paths import user_data_root
+
+_PROFILE_FILENAME = "profile.json"
+_TOKEN_PATTERN = re.compile(r"\$\{([A-Z_]+)\}")
+
+_DEFAULT_PROJECT_ROOTS = ["~/Herd", "~/Work", "~/AIProjects"]
+_DEFAULT_REPOS_ROOT = "~/AIProjects"
+_DEFAULT_GIT_HOST = "github.com"
+
+_PROFILE_CACHE: "ProfileV3 | None" = None
+
+
+class ProfileMissingError(RuntimeError):
+    """Raised when ``~/.arkaos/profile.json`` cannot be loaded."""
+
+
+@dataclass(frozen=True)
+class ProfileV3:
+    """In-memory representation of profile.json v3."""
+
+    version: str
+    vault_path: str
+    repos_root: str
+    project_roots: list[str] = field(default_factory=list)
+    raw: dict = field(default_factory=dict)
+
+
+def profile_path() -> Path:
+    """Absolute path to ``~/.arkaos/profile.json``."""
+    return user_data_root() / _PROFILE_FILENAME
+
+
+def load_profile(*, refresh: bool = False) -> ProfileV3:
+    """Load and validate profile.json.
+
+    Cached per process; pass ``refresh=True`` to force a re-read.
+
+    Raises:
+        ProfileMissingError: file absent, unreadable, or unparseable JSON.
+    """
+    global _PROFILE_CACHE
+    if _PROFILE_CACHE is not None and not refresh:
+        return _PROFILE_CACHE
+
+    path = profile_path()
+    if not path.exists():
+        raise ProfileMissingError(
+            f"~/.arkaos/profile.json not found at {path}. "
+            "Run /arka setup to configure ArkaOS paths."
+        )
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ProfileMissingError(
+            f"~/.arkaos/profile.json could not be parsed ({exc}). "
+            "Run /arka setup to repair, or restore from a .bak file."
+        ) from exc
+
+    _PROFILE_CACHE = _project_from_raw(raw)
+    return _PROFILE_CACHE
+
+
+def reset_cache() -> None:
+    """Drop the cached profile (for tests)."""
+    global _PROFILE_CACHE
+    _PROFILE_CACHE = None
+
+
+def resolve(template: str) -> str:
+    """Substitute known ``${VAR}`` tokens in *template*.
+
+    Tokens recognised:
+      - ``${VAULT_PATH}``    → ``ARKAOS_VAULT_PATH`` env or ``profile.vaultPath``
+      - ``${ARKA_OS_REPOS}`` → ``ARKAOS_REPOS_ROOT`` env or ``profile.reposRoot``
+      - ``${PROJECT_ROOTS}`` → ``os.pathsep``-joined profile.projectRoots
+      - ``${GIT_HOST}``      → ``ARKAOS_GIT_HOST`` env, default ``github.com``
+      - ``${HOME}``          → ``os.path.expanduser("~")``
+
+    Unknown tokens pass through unchanged.
+    """
+    return _TOKEN_PATTERN.sub(_resolve_token_match, template)
+
+
+def resolve_dict(obj):
+    """Recursively resolve every string value in nested dicts/lists."""
+    if isinstance(obj, str):
+        return resolve(obj)
+    if isinstance(obj, dict):
+        return {k: resolve_dict(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [resolve_dict(item) for item in obj]
+    return obj
+
+
+def project_root_regex() -> re.Pattern[str]:
+    """Compile a regex matching paths under any configured project root.
+
+    Used by ``core/cognition/capture/collector.py`` to detect project paths
+    inside captured session digests across macOS, Linux and Windows.
+    """
+    profile = load_profile()
+    roots = [os.path.expanduser(r) for r in profile.project_roots] or [
+        os.path.expanduser(r) for r in _DEFAULT_PROJECT_ROOTS
+    ]
+    alternation = "|".join(re.escape(r.rstrip("/").rstrip("\\")) for r in roots)
+    return re.compile(rf"({alternation})[/\\]([^\s/\\]+)")
+
+
+def _resolve_token_match(match: re.Match[str]) -> str:
+    return _resolve_token(match.group(1), match.group(0))
+
+
+def _resolve_token(name: str, original: str) -> str:
+    if name == "HOME":
+        return os.path.expanduser("~")
+    if name == "GIT_HOST":
+        return _env_or("ARKAOS_GIT_HOST", _DEFAULT_GIT_HOST)
+    if name == "VAULT_PATH":
+        env_val = _env_or("ARKAOS_VAULT_PATH", _env_or("ARKAOS_VAULT", ""))
+        return env_val or load_profile().vault_path
+    if name == "ARKA_OS_REPOS":
+        env_val = _env_or("ARKAOS_REPOS_ROOT", "")
+        return env_val or load_profile().repos_root
+    if name == "PROJECT_ROOTS":
+        env_val = _env_or("ARKAOS_PROJECT_ROOTS", "")
+        if env_val:
+            return env_val
+        return os.pathsep.join(load_profile().project_roots)
+    return original
+
+
+def _env_or(name: str, fallback: str) -> str:
+    value = os.environ.get(name, "")
+    return value if value else fallback
+
+
+def _project_from_raw(raw: dict) -> ProfileV3:
+    vault_path = raw.get("vaultPath") or raw.get("vault_path") or ""
+    if not vault_path:
+        raise ProfileMissingError(
+            "profile.json has no vaultPath. Run /arka setup to configure it."
+        )
+    repos_root = (
+        raw.get("reposRoot")
+        or raw.get("repos_root")
+        or _DEFAULT_REPOS_ROOT
+    )
+    project_roots = list(raw.get("projectRoots") or [])
+    if not project_roots:
+        project_roots = _derive_project_roots(raw.get("projectsDir", ""))
+    return ProfileV3(
+        version=str(raw.get("version", "2")),
+        vault_path=vault_path,
+        repos_root=repos_root,
+        project_roots=project_roots,
+        raw=raw,
+    )
+
+
+def _derive_project_roots(projects_dir_text: str) -> list[str]:
+    """Best-effort parse of the legacy free-text ``projectsDir`` field.
+
+    Looks for absolute paths on macOS/Linux/Windows. Falls back to
+    ``_DEFAULT_PROJECT_ROOTS`` when no path can be extracted. This keeps
+    the ~20K legacy users functional until ``npx arkaos update`` rewrites
+    their profile.json with the new ``projectRoots`` field.
+    """
+    if not projects_dir_text:
+        return list(_DEFAULT_PROJECT_ROOTS)
+    posix = r"(?:/Users|/home)/\S+?/(?:Herd|Work|AIProjects|code|repos)"
+    windows = r"[A-Z]:\\Users\\[^\s\\]+\\(?:Herd|Work|AIProjects|code|repos)"
+    pattern = re.compile(rf"({posix}|{windows})")
+    found = pattern.findall(projects_dir_text)
+    return found or list(_DEFAULT_PROJECT_ROOTS)

package/core/shared/__init__.py

@@ -0,0 +1,6 @@
+"""Cross-cutting primitives shared by multiple ArkaOS core packages.
+
+Keep this package lean — only primitives that two or more sibling
+packages already duplicate belong here. It is NOT a dumping ground for
+utilities; each addition must delete a duplicate elsewhere.
+"""

package/core/shared/safe_session_id.py

@@ -0,0 +1,41 @@
+"""Shared session-id allowlist — path-traversal / injection guard.
+
+A session id is considered safe iff it matches ``[A-Za-z0-9._-]{1,128}``.
+Any other character (``/``, ``\\``, whitespace, control char, unicode,
+NUL, ``..``) rejects — callers MUST treat ``None`` as "do not use this
+id for any filesystem or shell path".
+
+Why this lives here: the exact same regex + helper was duplicated in 6
+modules (flow_enforcer, marker_cache, research_gate, kb_cache,
+auto_documentor, auto_doc_worker). A single source of truth prevents
+drift — if the allowlist ever tightens, it tightens everywhere.
+
+Historic aliases remain at each call site as module-level re-exports
+so external importers that did ``from core.workflow.flow_enforcer
+import SAFE_SESSION_ID_RE`` continue to work.
+"""
+
+from __future__ import annotations
+
+import re
+
+
+SAFE_SESSION_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,128}$")
+
+
+def safe_session_id(session_id: str) -> str | None:
+    """Validate ``session_id`` against the strict allowlist.
+
+    Returns the id unchanged when safe, or ``None`` when it contains
+    path separators, ``..`` traversal fragments, whitespace, unicode,
+    NUL bytes, or any character outside ``[A-Za-z0-9._-]``. Length is
+    capped at 128 characters to prevent pathological filesystem paths.
+
+    Callers MUST treat ``None`` as reject — never construct a path or
+    shell argument from the raw input when this returns ``None``.
+    """
+    if not session_id or not isinstance(session_id, str):
+        return None
+    if not SAFE_SESSION_ID_RE.match(session_id):
+        return None
+    return session_id