arkaos 2.25.0 → 2.31.0

package/core/cognition/retrieval.py

@@ -0,0 +1,383 @@
+"""Hooks-as-retrieval prototype for the ArkaOS Cognitive Layer.
+
+After each tool call, the PostToolUse hook calls into this module to
+extract likely entities from the tool output, find relevant notes in the
+Obsidian vault, and write a small JSON cache that the UserPromptSubmit
+hook injects into the next turn as an ``[arka:context]`` advisory.
+
+This is the prototype unblocked by Claude Code 2.1.122 (late-binding
+``ToolSearch`` + hook-block isolation, see KB note
+2026-04-29-claude-code-2-1-122-and-2-1-123). It uses filesystem grep
+(ripgrep when available, Python fallback otherwise) instead of running
+a separate MCP server with embeddings — a deliberate "smallest thing
+that could possibly work" choice per the 2026-05-13 Conclave ADR.
+
+Performance budget: ≤ 800 ms p95 per ``capture_context`` call. The
+PostToolUse hook enforces this with a hard timeout; the function itself
+caps entities (20), vault hits (5), and ripgrep wall time (1 s).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.runtime.user_paths import user_data_root
+
+_CACHE_DIRNAME = "context-cache"
+_DEFAULT_TTL_SECONDS = 600
+_REAPER_MAX_AGE_SECONDS = 24 * 60 * 60  # cache files older than 24h get pruned
+_MAX_ENTITIES = 20
+_MAX_VAULT_HITS = 5
+_RIPGREP_TIMEOUT_S = 1.0
+_PY_FALLBACK_MAX_FILES = 500
+
+# Common-word stoplist — kept tiny on purpose; we want to capture short
+# domain terms like "auth" or "DSL" but cut out the highest-frequency
+# English filler tokens that dominate any tool output.
+_STOPLIST = frozenset(
+    {
+        "The", "And", "But", "For", "Not", "With", "From", "This", "That",
+        "When", "Where", "Then", "Else", "True", "False", "None", "Null",
+        "Error", "Warning", "Info", "Debug", "Trace", "Yes", "No", "Run",
+        "Read", "Write", "Edit", "Delete", "Show", "Find", "List",
+    }
+)
+
+_FILE_PATH = re.compile(r"(?:^|\s)((?:[\w.-]+/)+[\w.-]+\.[a-zA-Z]{1,6})")
+_CAMEL_OR_PASCAL = re.compile(r"\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b")
+_AT_MENTION = re.compile(r"@([\w./_-]+)")
+_PROPER_NOUN = re.compile(r"\b([A-Z][a-zA-Z]{2,})\b")
+
+
+@dataclass
+class ContextHit:
+    """One vault note that matches one or more extracted entities."""
+
+    entity: str
+    vault_path: str
+    snippet: str
+    score: float = 1.0
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class ContextCache:
+    """The JSON payload persisted per session."""
+
+    session_id: str
+    captured_at: str
+    ttl_seconds: int
+    hits: list[ContextHit] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "session_id": self.session_id,
+            "captured_at": self.captured_at,
+            "ttl_seconds": self.ttl_seconds,
+            "hits": [h.to_dict() for h in self.hits],
+        }
+
+
+def extract_entities(text: str) -> list[str]:
+    """Pull file paths, identifiers, and proper nouns out of *text*.
+
+    Returns a deduplicated, capped list ready for vault search. Cheap
+    enough to call on every PostToolUse invocation.
+    """
+    if not text:
+        return []
+    seen: set[str] = set()
+    out: list[str] = []
+
+    def push(tok: str) -> None:
+        tok = tok.strip(" .,;:!?\"'()[]{}")
+        if not tok or tok in _STOPLIST or tok in seen:
+            return
+        if tok.isdigit():
+            return
+        seen.add(tok)
+        out.append(tok)
+        return
+
+    for m in _FILE_PATH.finditer(text):
+        push(m.group(1))
+    for m in _AT_MENTION.finditer(text):
+        push(m.group(1))
+    for m in _CAMEL_OR_PASCAL.finditer(text):
+        push(m.group(1))
+    for m in _PROPER_NOUN.finditer(text):
+        if len(out) >= _MAX_ENTITIES:
+            break
+        push(m.group(1))
+    return out[:_MAX_ENTITIES]
+
+
+def search_vault(entities: list[str], vault_path: str) -> list[ContextHit]:
+    """Find Obsidian notes that mention any of the extracted entities.
+
+    Uses ripgrep when available (sub-second across thousands of notes),
+    falls back to a pure-Python scan capped at ``_PY_FALLBACK_MAX_FILES``
+    files so the hook timeout stays safe on Windows or minimal Linux
+    installs without ripgrep.
+    """
+    vault = Path(vault_path).expanduser()
+    if not entities or not vault.is_dir():
+        return []
+    if _ripgrep_available():
+        hits = _search_ripgrep(entities, vault)
+    else:
+        hits = _search_python(entities, vault)
+    return hits[:_MAX_VAULT_HITS]
+
+
+def capture_context(
+    session_id: str,
+    tool_output: str,
+    vault_path: str,
+    cache_dir: Path | None = None,
+) -> ContextCache:
+    """Run extract + search and persist the cache. Returns the cache.
+
+    Idempotent across overlapping calls in the same second: a second
+    invocation overwrites the first. Empty hits still produce a file
+    so the UserPromptSubmit reader sees a fresh ``captured_at`` and
+    doesn't mistake a quiet turn for stale data.
+    """
+    entities = extract_entities(tool_output)
+    hits = search_vault(entities, vault_path) if entities else []
+    cache = ContextCache(
+        session_id=session_id,
+        captured_at=datetime.now(timezone.utc).isoformat(),
+        ttl_seconds=_DEFAULT_TTL_SECONDS,
+        hits=hits,
+    )
+    _write_cache(cache, cache_dir or _default_cache_dir())
+    return cache
+
+
+def read_context(
+    session_id: str,
+    ttl_seconds: int = _DEFAULT_TTL_SECONDS,
+    cache_dir: Path | None = None,
+) -> list[ContextHit]:
+    """Read cached hits for *session_id* if still within TTL."""
+    path = (cache_dir or _default_cache_dir()) / f"{session_id}.json"
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+    captured = data.get("captured_at", "")
+    if not _within_ttl(captured, ttl_seconds):
+        return []
+    return [ContextHit(**h) for h in data.get("hits", [])]
+
+
+def format_advisory(hits: list[ContextHit], max_chars: int = 1200) -> str:
+    """Render hits into a compact `[arka:context]` advisory string."""
+    if not hits:
+        return ""
+    lines = ["[arka:context] KB matches from last turn:"]
+    for h in hits:
+        line = f"- {h.entity} → {h.vault_path}: {h.snippet[:160]}"
+        if sum(len(l) for l in lines) + len(line) > max_chars:
+            break
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def _default_cache_dir() -> Path:
+    return user_data_root() / _CACHE_DIRNAME
+
+
+def _write_cache(cache: ContextCache, cache_dir: Path) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    path = cache_dir / f"{cache.session_id}.json"
+    tmp = cache_dir / f"{cache.session_id}.json.tmp"
+    tmp.write_text(json.dumps(cache.to_dict(), indent=2), encoding="utf-8")
+    os.replace(tmp, path)
+    _prune_stale_cache(cache_dir)
+
+
+def _prune_stale_cache(cache_dir: Path) -> int:
+    """Delete cache files older than 24h to bound disk growth.
+
+    Called from ``_write_cache`` so cleanup amortises across normal use
+    without requiring a separate cron job. Per Marta's PR4 follow-up
+    observation: the read-side TTL hides stale data but on-disk files
+    accumulate one-per-session forever. Returns the count of pruned files.
+    """
+    if not cache_dir.is_dir():
+        return 0
+    cutoff = datetime.now(timezone.utc).timestamp() - _REAPER_MAX_AGE_SECONDS
+    pruned = 0
+    for child in cache_dir.glob("*.json"):
+        try:
+            if child.stat().st_mtime < cutoff:
+                child.unlink()
+                pruned += 1
+        except OSError:
+            continue
+    return pruned
+
+
+def _within_ttl(captured_iso: str, ttl_seconds: int) -> bool:
+    try:
+        captured = datetime.fromisoformat(captured_iso)
+    except ValueError:
+        return False
+    now = datetime.now(timezone.utc)
+    if captured.tzinfo is None:
+        captured = captured.replace(tzinfo=timezone.utc)
+    return (now - captured).total_seconds() <= ttl_seconds
+
+
+def _ripgrep_available() -> bool:
+    try:
+        subprocess.run(
+            ["rg", "--version"], check=True, capture_output=True, timeout=1.0
+        )
+        return True
+    except (FileNotFoundError, subprocess.SubprocessError):
+        return False
+
+
+def _search_ripgrep(entities: list[str], vault: Path) -> list[ContextHit]:
+    pattern = "|".join(re.escape(e) for e in entities)
+    cmd = [
+        "rg", "--json", "--max-count", "1", "--smart-case",
+        "-tmd", "-e", pattern, str(vault),
+    ]
+    try:
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=_RIPGREP_TIMEOUT_S
+        )
+    except subprocess.SubprocessError:
+        return []
+    return _parse_ripgrep_json(result.stdout, vault, entities)
+
+
+def _parse_ripgrep_json(stdout: str, vault: Path, entities: list[str]) -> list[ContextHit]:
+    hits: list[ContextHit] = []
+    seen: set[str] = set()
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if event.get("type") != "match":
+            continue
+        data = event.get("data", {})
+        path = data.get("path", {}).get("text", "")
+        if not path or path in seen:
+            continue
+        seen.add(path)
+        text = data.get("lines", {}).get("text", "").strip()
+        matched_entity = _pick_matched_entity(text, entities)
+        rel = _relative_to_vault(path, vault)
+        hits.append(ContextHit(entity=matched_entity, vault_path=rel, snippet=text))
+    return hits
+
+
+def _search_python(entities: list[str], vault: Path) -> list[ContextHit]:
+    pattern = re.compile("|".join(re.escape(e) for e in entities), re.IGNORECASE)
+    files = list(vault.rglob("*.md"))[:_PY_FALLBACK_MAX_FILES]
+    hits: list[ContextHit] = []
+    for f in files:
+        try:
+            text = f.read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            continue
+        match = pattern.search(text)
+        if not match:
+            continue
+        snippet_start = max(0, match.start() - 40)
+        snippet_end = min(len(text), match.end() + 80)
+        snippet = text[snippet_start:snippet_end].replace("\n", " ").strip()
+        hits.append(
+            ContextHit(
+                entity=match.group(0),
+                vault_path=str(f.relative_to(vault)),
+                snippet=snippet,
+            )
+        )
+        if len(hits) >= _MAX_VAULT_HITS:
+            break
+    return hits
+
+
+def _pick_matched_entity(text: str, entities: list[str]) -> str:
+    lower = text.lower()
+    for e in entities:
+        if e.lower() in lower:
+            return e
+    return entities[0] if entities else ""
+
+
+def _relative_to_vault(absolute: str, vault: Path) -> str:
+    try:
+        return str(Path(absolute).relative_to(vault))
+    except ValueError:
+        return absolute
+
+
+def _vault_path_or_none() -> str | None:
+    """Resolve the configured vault path or return None when missing.
+
+    Hooks call this; they must never crash because profile.json is absent.
+    """
+    try:
+        from core.runtime.path_resolver import load_profile
+        return load_profile().vault_path
+    except Exception:
+        return None
+
+
+def _cli_capture(argv: list[str]) -> int:
+    """Stdin → capture_context. Used by the PostToolUse hook."""
+    import sys
+    if len(argv) < 2:
+        return 2
+    session_id = argv[1]
+    vault = _vault_path_or_none()
+    if not vault:
+        return 0  # silent skip when no profile
+    tool_output = sys.stdin.read()
+    capture_context(session_id, tool_output, vault)
+    return 0
+
+
+def _cli_inject(argv: list[str]) -> int:
+    """Print advisory for *session_id*. Used by the UserPromptSubmit hook."""
+    if len(argv) < 2:
+        return 2
+    session_id = argv[1]
+    advisory = format_advisory(read_context(session_id))
+    if advisory:
+        print(advisory)
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        return 2
+    cmd = argv[1]
+    if cmd == "capture":
+        return _cli_capture(argv[1:])
+    if cmd == "inject":
+        return _cli_inject(argv[1:])
+    return 2
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main(sys.argv))

package/core/cognition/scheduler/daemon.py

@@ -18,7 +18,16 @@ import yaml
 
 @dataclass
 class ScheduleConfig:
-    """Configuration for a single scheduled cognitive task."""
+    """Configuration for a single scheduled cognitive task.
+
+    Two execution modes (mutually exclusive):
+      - prompt_file (default): shell out to the active Claude CLI with the
+        rendered prompt as the user input. Backward-compat for legacy
+        dreaming.md / research.md schedules.
+      - python_module: invoke ``python -m <module> [args...]`` directly.
+        Used by Dreaming v2 (PR8) which is a Python engine, not a
+        prompt-only task.
+    """
 
     command: str
     prompt_file: str
@@ -27,6 +36,8 @@ class ScheduleConfig:
     retry_on_fail: bool = True
     max_retries: int = 2
     timeout_minutes: int = 60
+    python_module: str | None = None
+    module_args: list[str] = field(default_factory=list)
 
     @classmethod
     def load(cls, config_path: str) -> "list[ScheduleConfig]":
@@ -43,12 +54,14 @@ class ScheduleConfig:
             schedules.append(
                 cls(
                     command=cfg["command"],
-                    prompt_file=cfg["prompt_file"],
+                    prompt_file=cfg.get("prompt_file", ""),
                     run_time=time(hour, minute),
                     enabled=cfg.get("enabled", True),
                     retry_on_fail=cfg.get("retry_on_fail", True),
                     max_retries=cfg.get("max_retries", 2),
                     timeout_minutes=cfg.get("timeout_minutes", 60),
+                    python_module=cfg.get("python_module"),
+                    module_args=list(cfg.get("module_args") or []),
                 )
             )
         return schedules
@@ -140,7 +153,14 @@ class ArkaScheduler:
         )
 
     def _build_command(self, schedule: ScheduleConfig) -> list[str]:
-        """Build the Claude CLI invocation for a schedule."""
+        """Build the subprocess invocation for a schedule.
+
+        Dispatches on python_module first (PR8 Dreaming v2 path), falls
+        back to the legacy Claude-CLI-with-prompt path for unchanged
+        schedules.
+        """
+        if schedule.python_module:
+            return [sys.executable, "-m", schedule.python_module, *schedule.module_args]
         claude_bin = self._resolve_claude_binary()
         prompt_path = os.path.expanduser(schedule.prompt_file)
         prompt_content = Path(prompt_path).read_text(encoding="utf-8")

package/core/runtime/llm_provider.py

@@ -31,7 +31,7 @@ from core.runtime.pricing import estimate_cost_usd
 
 
 _DEFAULT_CONFIG_PATH = Path.home() / ".arkaos" / "config.json"
-_FALLBACK_ORDER: tuple[str, ...] = ("subagent", "anthropic-direct", "stub")
+_FALLBACK_ORDER: tuple[str, ...] = ("subagent", "ollama", "anthropic-direct", "stub")
 
 
 # ─── Public dataclass ─────────────────────────────────────────────────
@@ -305,8 +305,15 @@ class StubProvider:
 # ─── Factory + fallback chain ─────────────────────────────────────────
 
 
+def _import_ollama_provider() -> type:
+    """Lazy import to keep the Ollama provider optional at runtime."""
+    from core.runtime.ollama_provider import OllamaProvider
+    return OllamaProvider
+
+
 _PROVIDERS: dict[str, type] = {
     "subagent": SubagentProvider,
+    "ollama": _import_ollama_provider(),
     "anthropic-direct": AnthropicDirectProvider,
     "stub": StubProvider,
 }

package/core/runtime/ollama_provider.py

@@ -0,0 +1,144 @@
+"""Ollama provider — local LLM inference via the localhost API.
+
+One of several backends behind the `LLMProvider` Protocol in
+``core/runtime/llm_provider.py``. The cognitive layer never imports
+this class directly; it goes through ``get_llm_provider()`` which
+selects from a configurable chain (subagent → ollama → anthropic-direct
+→ stub by default; user-configurable in ``~/.arkaos/config.json``).
+
+Communicates with the local Ollama service at the standard
+``http://localhost:11434`` endpoint via stdlib ``urllib`` — no extra
+dependencies. Model picked from ``OLLAMA_MODEL`` env var, then
+``profile.json:cognitiveModel``, then a sensible default. Whichever
+the user has chosen sticks to a single chat completion call per
+``complete()`` invocation.
+
+See ``docs/adr/2026-05-13-cognitive-layer-pivot-to-hooks.md`` and the
+PR8 v2.30.0 commit for the architectural rationale.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from core.runtime.llm_provider import LLMResponse, LLMUnavailable
+
+
+_OLLAMA_HOST = "http://localhost:11434"
+_DEFAULT_MODEL = "gemma3:27b"  # fallback only — most users override via env / profile
+_GENERATE_TIMEOUT_S = 120  # nightly Dreaming run — large prompts allowed
+
+
+class OllamaProvider:
+    """Provider that hits the local Ollama daemon for chat completion."""
+
+    ENV_HOST = "OLLAMA_HOST"
+    ENV_MODEL = "OLLAMA_MODEL"
+
+    def __init__(self, model: str | None = None, host: str | None = None) -> None:
+        self._model_override = model
+        self._host = host or os.environ.get(self.ENV_HOST, _OLLAMA_HOST)
+
+    def name(self) -> str:
+        return "ollama"
+
+    def is_available(self) -> bool:
+        """Cheap reachability check against ``/api/tags``."""
+        url = f"{self._host.rstrip('/')}/api/tags"
+        try:
+            with urllib.request.urlopen(url, timeout=1.5) as response:
+                return response.status == 200
+        except (urllib.error.URLError, TimeoutError, OSError):
+            return False
+
+    def complete(
+        self,
+        prompt: str,
+        *,
+        max_tokens: int = 2000,
+        system: str = "",
+    ) -> LLMResponse:
+        model = self._resolve_model()
+        if not model:
+            raise LLMUnavailable("Ollama model not configured (set OLLAMA_MODEL or profile.cognitiveModel)")
+
+        payload = self._build_payload(model, prompt, system, max_tokens)
+        data = self._post_generate(payload)
+        return self._to_response(data, model)
+
+    def _resolve_model(self) -> str | None:
+        if self._model_override:
+            return self._model_override
+        env_value = os.environ.get(self.ENV_MODEL, "").strip()
+        if env_value:
+            return env_value
+        profile_model = _read_profile_model()
+        if profile_model:
+            return profile_model
+        return _DEFAULT_MODEL
+
+    def _build_payload(self, model: str, prompt: str, system: str, max_tokens: int) -> dict:
+        """Build /api/chat payload.
+
+        Using the chat endpoint (not /api/generate) means Ollama applies
+        the model's official chat template, which is critical for
+        instruction-tuned models like Qwen, Gemma, Mistral — without the
+        template most models burn through their token budget before
+        emitting visible content. Smoke-tested 2026-05-13 against
+        qwen3-coder:30b which returned the expected "hi" in 2 tokens.
+        """
+        messages: list[dict] = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+        return {
+            "model": model,
+            "messages": messages,
+            "stream": False,
+            "options": {"num_predict": max_tokens},
+        }
+
+    def _post_generate(self, payload: dict) -> dict:
+        url = f"{self._host.rstrip('/')}/api/chat"
+        body = json.dumps(payload).encode("utf-8")
+        request = urllib.request.Request(
+            url, data=body, headers={"Content-Type": "application/json"}
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=_GENERATE_TIMEOUT_S) as response:
+                raw = response.read().decode("utf-8")
+        except (urllib.error.URLError, TimeoutError, OSError) as exc:
+            raise LLMUnavailable(f"Ollama request failed: {exc}") from exc
+        try:
+            return json.loads(raw)
+        except json.JSONDecodeError as exc:
+            raise LLMUnavailable(f"Ollama returned invalid JSON: {exc}") from exc
+
+    def _to_response(self, data: dict, model: str) -> LLMResponse:
+        message = data.get("message", {}) or {}
+        text = (message.get("content") or data.get("response") or "").strip()
+        tokens_in = int(data.get("prompt_eval_count", 0) or 0)
+        tokens_out = int(data.get("eval_count", 0) or 0)
+        return LLMResponse(
+            text=text,
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            cached_tokens=0,  # Ollama does not surface a cache signal
+            model=model,
+        )
+
+
+def _read_profile_model() -> str | None:
+    path = Path.home() / ".arkaos" / "profile.json"
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    value = data.get("cognitiveModel")
+    if isinstance(value, str) and value.strip():
+        return value.strip()
+    return None