arkaos 2.22.1 → 2.28.0

package/core/cognition/retrieval.py

@@ -0,0 +1,383 @@
+"""Hooks-as-retrieval prototype for the ArkaOS Cognitive Layer.
+
+After each tool call, the PostToolUse hook calls into this module to
+extract likely entities from the tool output, find relevant notes in the
+Obsidian vault, and write a small JSON cache that the UserPromptSubmit
+hook injects into the next turn as an ``[arka:context]`` advisory.
+
+This is the prototype unblocked by Claude Code 2.1.122 (late-binding
+``ToolSearch`` + hook-block isolation, see KB note
+2026-04-29-claude-code-2-1-122-and-2-1-123). It uses filesystem grep
+(ripgrep when available, Python fallback otherwise) instead of running
+a separate MCP server with embeddings — a deliberate "smallest thing
+that could possibly work" choice per the 2026-05-13 Conclave ADR.
+
+Performance budget: ≤ 800 ms p95 per ``capture_context`` call. The
+PostToolUse hook enforces this with a hard timeout; the function itself
+caps entities (20), vault hits (5), and ripgrep wall time (1 s).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.runtime.user_paths import user_data_root
+
+_CACHE_DIRNAME = "context-cache"
+_DEFAULT_TTL_SECONDS = 600
+_REAPER_MAX_AGE_SECONDS = 24 * 60 * 60  # cache files older than 24h get pruned
+_MAX_ENTITIES = 20
+_MAX_VAULT_HITS = 5
+_RIPGREP_TIMEOUT_S = 1.0
+_PY_FALLBACK_MAX_FILES = 500
+
+# Common-word stoplist — kept tiny on purpose; we want to capture short
+# domain terms like "auth" or "DSL" but cut out the highest-frequency
+# English filler tokens that dominate any tool output.
+_STOPLIST = frozenset(
+    {
+        "The", "And", "But", "For", "Not", "With", "From", "This", "That",
+        "When", "Where", "Then", "Else", "True", "False", "None", "Null",
+        "Error", "Warning", "Info", "Debug", "Trace", "Yes", "No", "Run",
+        "Read", "Write", "Edit", "Delete", "Show", "Find", "List",
+    }
+)
+
+_FILE_PATH = re.compile(r"(?:^|\s)((?:[\w.-]+/)+[\w.-]+\.[a-zA-Z]{1,6})")
+_CAMEL_OR_PASCAL = re.compile(r"\b([A-Z][a-z]+(?:[A-Z][a-z]+)+)\b")
+_AT_MENTION = re.compile(r"@([\w./_-]+)")
+_PROPER_NOUN = re.compile(r"\b([A-Z][a-zA-Z]{2,})\b")
+
+
+@dataclass
+class ContextHit:
+    """One vault note that matches one or more extracted entities."""
+
+    entity: str
+    vault_path: str
+    snippet: str
+    score: float = 1.0
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class ContextCache:
+    """The JSON payload persisted per session."""
+
+    session_id: str
+    captured_at: str
+    ttl_seconds: int
+    hits: list[ContextHit] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "session_id": self.session_id,
+            "captured_at": self.captured_at,
+            "ttl_seconds": self.ttl_seconds,
+            "hits": [h.to_dict() for h in self.hits],
+        }
+
+
+def extract_entities(text: str) -> list[str]:
+    """Pull file paths, identifiers, and proper nouns out of *text*.
+
+    Returns a deduplicated, capped list ready for vault search. Cheap
+    enough to call on every PostToolUse invocation.
+    """
+    if not text:
+        return []
+    seen: set[str] = set()
+    out: list[str] = []
+
+    def push(tok: str) -> None:
+        tok = tok.strip(" .,;:!?\"'()[]{}")
+        if not tok or tok in _STOPLIST or tok in seen:
+            return
+        if tok.isdigit():
+            return
+        seen.add(tok)
+        out.append(tok)
+        return
+
+    for m in _FILE_PATH.finditer(text):
+        push(m.group(1))
+    for m in _AT_MENTION.finditer(text):
+        push(m.group(1))
+    for m in _CAMEL_OR_PASCAL.finditer(text):
+        push(m.group(1))
+    for m in _PROPER_NOUN.finditer(text):
+        if len(out) >= _MAX_ENTITIES:
+            break
+        push(m.group(1))
+    return out[:_MAX_ENTITIES]
+
+
+def search_vault(entities: list[str], vault_path: str) -> list[ContextHit]:
+    """Find Obsidian notes that mention any of the extracted entities.
+
+    Uses ripgrep when available (sub-second across thousands of notes),
+    falls back to a pure-Python scan capped at ``_PY_FALLBACK_MAX_FILES``
+    files so the hook timeout stays safe on Windows or minimal Linux
+    installs without ripgrep.
+    """
+    vault = Path(vault_path).expanduser()
+    if not entities or not vault.is_dir():
+        return []
+    if _ripgrep_available():
+        hits = _search_ripgrep(entities, vault)
+    else:
+        hits = _search_python(entities, vault)
+    return hits[:_MAX_VAULT_HITS]
+
+
+def capture_context(
+    session_id: str,
+    tool_output: str,
+    vault_path: str,
+    cache_dir: Path | None = None,
+) -> ContextCache:
+    """Run extract + search and persist the cache. Returns the cache.
+
+    Idempotent across overlapping calls in the same second: a second
+    invocation overwrites the first. Empty hits still produce a file
+    so the UserPromptSubmit reader sees a fresh ``captured_at`` and
+    doesn't mistake a quiet turn for stale data.
+    """
+    entities = extract_entities(tool_output)
+    hits = search_vault(entities, vault_path) if entities else []
+    cache = ContextCache(
+        session_id=session_id,
+        captured_at=datetime.now(timezone.utc).isoformat(),
+        ttl_seconds=_DEFAULT_TTL_SECONDS,
+        hits=hits,
+    )
+    _write_cache(cache, cache_dir or _default_cache_dir())
+    return cache
+
+
+def read_context(
+    session_id: str,
+    ttl_seconds: int = _DEFAULT_TTL_SECONDS,
+    cache_dir: Path | None = None,
+) -> list[ContextHit]:
+    """Read cached hits for *session_id* if still within TTL."""
+    path = (cache_dir or _default_cache_dir()) / f"{session_id}.json"
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return []
+    captured = data.get("captured_at", "")
+    if not _within_ttl(captured, ttl_seconds):
+        return []
+    return [ContextHit(**h) for h in data.get("hits", [])]
+
+
+def format_advisory(hits: list[ContextHit], max_chars: int = 1200) -> str:
+    """Render hits into a compact `[arka:context]` advisory string."""
+    if not hits:
+        return ""
+    lines = ["[arka:context] KB matches from last turn:"]
+    for h in hits:
+        line = f"- {h.entity} → {h.vault_path}: {h.snippet[:160]}"
+        if sum(len(l) for l in lines) + len(line) > max_chars:
+            break
+        lines.append(line)
+    return "\n".join(lines)
+
+
+def _default_cache_dir() -> Path:
+    return user_data_root() / _CACHE_DIRNAME
+
+
+def _write_cache(cache: ContextCache, cache_dir: Path) -> None:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    path = cache_dir / f"{cache.session_id}.json"
+    tmp = cache_dir / f"{cache.session_id}.json.tmp"
+    tmp.write_text(json.dumps(cache.to_dict(), indent=2), encoding="utf-8")
+    os.replace(tmp, path)
+    _prune_stale_cache(cache_dir)
+
+
+def _prune_stale_cache(cache_dir: Path) -> int:
+    """Delete cache files older than 24h to bound disk growth.
+
+    Called from ``_write_cache`` so cleanup amortises across normal use
+    without requiring a separate cron job. Per Marta's PR4 follow-up
+    observation: the read-side TTL hides stale data but on-disk files
+    accumulate one-per-session forever. Returns the count of pruned files.
+    """
+    if not cache_dir.is_dir():
+        return 0
+    cutoff = datetime.now(timezone.utc).timestamp() - _REAPER_MAX_AGE_SECONDS
+    pruned = 0
+    for child in cache_dir.glob("*.json"):
+        try:
+            if child.stat().st_mtime < cutoff:
+                child.unlink()
+                pruned += 1
+        except OSError:
+            continue
+    return pruned
+
+
+def _within_ttl(captured_iso: str, ttl_seconds: int) -> bool:
+    try:
+        captured = datetime.fromisoformat(captured_iso)
+    except ValueError:
+        return False
+    now = datetime.now(timezone.utc)
+    if captured.tzinfo is None:
+        captured = captured.replace(tzinfo=timezone.utc)
+    return (now - captured).total_seconds() <= ttl_seconds
+
+
+def _ripgrep_available() -> bool:
+    try:
+        subprocess.run(
+            ["rg", "--version"], check=True, capture_output=True, timeout=1.0
+        )
+        return True
+    except (FileNotFoundError, subprocess.SubprocessError):
+        return False
+
+
+def _search_ripgrep(entities: list[str], vault: Path) -> list[ContextHit]:
+    pattern = "|".join(re.escape(e) for e in entities)
+    cmd = [
+        "rg", "--json", "--max-count", "1", "--smart-case",
+        "-tmd", "-e", pattern, str(vault),
+    ]
+    try:
+        result = subprocess.run(
+            cmd, capture_output=True, text=True, timeout=_RIPGREP_TIMEOUT_S
+        )
+    except subprocess.SubprocessError:
+        return []
+    return _parse_ripgrep_json(result.stdout, vault, entities)
+
+
+def _parse_ripgrep_json(stdout: str, vault: Path, entities: list[str]) -> list[ContextHit]:
+    hits: list[ContextHit] = []
+    seen: set[str] = set()
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if event.get("type") != "match":
+            continue
+        data = event.get("data", {})
+        path = data.get("path", {}).get("text", "")
+        if not path or path in seen:
+            continue
+        seen.add(path)
+        text = data.get("lines", {}).get("text", "").strip()
+        matched_entity = _pick_matched_entity(text, entities)
+        rel = _relative_to_vault(path, vault)
+        hits.append(ContextHit(entity=matched_entity, vault_path=rel, snippet=text))
+    return hits
+
+
+def _search_python(entities: list[str], vault: Path) -> list[ContextHit]:
+    pattern = re.compile("|".join(re.escape(e) for e in entities), re.IGNORECASE)
+    files = list(vault.rglob("*.md"))[:_PY_FALLBACK_MAX_FILES]
+    hits: list[ContextHit] = []
+    for f in files:
+        try:
+            text = f.read_text(encoding="utf-8", errors="ignore")
+        except OSError:
+            continue
+        match = pattern.search(text)
+        if not match:
+            continue
+        snippet_start = max(0, match.start() - 40)
+        snippet_end = min(len(text), match.end() + 80)
+        snippet = text[snippet_start:snippet_end].replace("\n", " ").strip()
+        hits.append(
+            ContextHit(
+                entity=match.group(0),
+                vault_path=str(f.relative_to(vault)),
+                snippet=snippet,
+            )
+        )
+        if len(hits) >= _MAX_VAULT_HITS:
+            break
+    return hits
+
+
+def _pick_matched_entity(text: str, entities: list[str]) -> str:
+    lower = text.lower()
+    for e in entities:
+        if e.lower() in lower:
+            return e
+    return entities[0] if entities else ""
+
+
+def _relative_to_vault(absolute: str, vault: Path) -> str:
+    try:
+        return str(Path(absolute).relative_to(vault))
+    except ValueError:
+        return absolute
+
+
+def _vault_path_or_none() -> str | None:
+    """Resolve the configured vault path or return None when missing.
+
+    Hooks call this; they must never crash because profile.json is absent.
+    """
+    try:
+        from core.runtime.path_resolver import load_profile
+        return load_profile().vault_path
+    except Exception:
+        return None
+
+
+def _cli_capture(argv: list[str]) -> int:
+    """Stdin → capture_context. Used by the PostToolUse hook."""
+    import sys
+    if len(argv) < 2:
+        return 2
+    session_id = argv[1]
+    vault = _vault_path_or_none()
+    if not vault:
+        return 0  # silent skip when no profile
+    tool_output = sys.stdin.read()
+    capture_context(session_id, tool_output, vault)
+    return 0
+
+
+def _cli_inject(argv: list[str]) -> int:
+    """Print advisory for *session_id*. Used by the UserPromptSubmit hook."""
+    if len(argv) < 2:
+        return 2
+    session_id = argv[1]
+    advisory = format_advisory(read_context(session_id))
+    if advisory:
+        print(advisory)
+    return 0
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) < 2:
+        return 2
+    cmd = argv[1]
+    if cmd == "capture":
+        return _cli_capture(argv[1:])
+    if cmd == "inject":
+        return _cli_inject(argv[1:])
+    return 2
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main(sys.argv))

package/core/cognition/scheduler/daemon.py

@@ -144,6 +144,11 @@ class ArkaScheduler:
         claude_bin = self._resolve_claude_binary()
         prompt_path = os.path.expanduser(schedule.prompt_file)
         prompt_content = Path(prompt_path).read_text(encoding="utf-8")
+        try:
+            from core.runtime.path_resolver import resolve
+            prompt_content = resolve(prompt_content)
+        except Exception:
+            pass  # fall back to raw template if profile unavailable
         return [claude_bin, "-p", prompt_content, "--dangerously-skip-permissions"]
 
     # ------------------------------------------------------------------

package/core/obsidian/writer.py

@@ -130,14 +130,15 @@ class ObsidianWriter:
 
         if config_path.exists():
             try:
+                from core.runtime.path_resolver import resolve
                 config = json.loads(config_path.read_text())
-                vault = config.get("vault_path", "")
-                if vault and Path(vault).exists():
+                vault = resolve(config.get("vault_path", ""))
+                if vault and not vault.startswith("${") and Path(vault).exists():
                     return Path(vault)
-            except (json.JSONDecodeError, OSError):
+            except (json.JSONDecodeError, OSError, Exception):
                 pass
 
-        # 3. Environment variable
+        # 3. Environment variable (legacy fallback only — path_resolver handles ARKAOS_VAULT_PATH)
         env_vault = os.environ.get("ARKAOS_VAULT", "")
         if env_vault and Path(env_vault).exists():
             return Path(env_vault)

package/core/runtime/path_resolver.py

@@ -0,0 +1,202 @@
+"""Template-token path resolver for ArkaOS.
+
+Resolves user-specific paths from ``~/.arkaos/profile.json`` and environment
+variables, so source files can use neutral tokens like ``${VAULT_PATH}`` and
+``${ARKA_OS_REPOS}`` instead of hardcoded absolute paths.
+
+Token precedence (highest → lowest):
+  1. Environment variable (e.g. ``ARKAOS_VAULT_PATH``)
+  2. profile.json field (e.g. ``vaultPath``)
+  3. For ``${GIT_HOST}`` only: hardcoded default ``github.com``
+
+If ``~/.arkaos/profile.json`` is missing or unparseable and an
+unconfigured token is requested, ``ProfileMissingError`` is raised
+with a clear remediation message. Unknown tokens pass through
+unchanged so that prompt strings containing ``${SOME_BASH_VAR}`` for
+the agent are preserved.
+
+See ``core/specs/SPEC-paths-portability.md`` for the full contract.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from core.runtime.user_paths import user_data_root
+
+_PROFILE_FILENAME = "profile.json"
+_TOKEN_PATTERN = re.compile(r"\$\{([A-Z_]+)\}")
+
+_DEFAULT_PROJECT_ROOTS = ["~/Herd", "~/Work", "~/AIProjects"]
+_DEFAULT_REPOS_ROOT = "~/AIProjects"
+_DEFAULT_GIT_HOST = "github.com"
+
+_PROFILE_CACHE: "ProfileV3 | None" = None
+
+
+class ProfileMissingError(RuntimeError):
+    """Raised when ``~/.arkaos/profile.json`` cannot be loaded."""
+
+
+@dataclass(frozen=True)
+class ProfileV3:
+    """In-memory representation of profile.json v3."""
+
+    version: str
+    vault_path: str
+    repos_root: str
+    project_roots: list[str] = field(default_factory=list)
+    raw: dict = field(default_factory=dict)
+
+
+def profile_path() -> Path:
+    """Absolute path to ``~/.arkaos/profile.json``."""
+    return user_data_root() / _PROFILE_FILENAME
+
+
+def load_profile(*, refresh: bool = False) -> ProfileV3:
+    """Load and validate profile.json.
+
+    Cached per process; pass ``refresh=True`` to force a re-read.
+
+    Raises:
+        ProfileMissingError: file absent, unreadable, or unparseable JSON.
+    """
+    global _PROFILE_CACHE
+    if _PROFILE_CACHE is not None and not refresh:
+        return _PROFILE_CACHE
+
+    path = profile_path()
+    if not path.exists():
+        raise ProfileMissingError(
+            f"~/.arkaos/profile.json not found at {path}. "
+            "Run /arka setup to configure ArkaOS paths."
+        )
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ProfileMissingError(
+            f"~/.arkaos/profile.json could not be parsed ({exc}). "
+            "Run /arka setup to repair, or restore from a .bak file."
+        ) from exc
+
+    _PROFILE_CACHE = _project_from_raw(raw)
+    return _PROFILE_CACHE
+
+
+def reset_cache() -> None:
+    """Drop the cached profile (for tests)."""
+    global _PROFILE_CACHE
+    _PROFILE_CACHE = None
+
+
+def resolve(template: str) -> str:
+    """Substitute known ``${VAR}`` tokens in *template*.
+
+    Tokens recognised:
+      - ``${VAULT_PATH}``    → ``ARKAOS_VAULT_PATH`` env or ``profile.vaultPath``
+      - ``${ARKA_OS_REPOS}`` → ``ARKAOS_REPOS_ROOT`` env or ``profile.reposRoot``
+      - ``${PROJECT_ROOTS}`` → ``os.pathsep``-joined profile.projectRoots
+      - ``${GIT_HOST}``      → ``ARKAOS_GIT_HOST`` env, default ``github.com``
+      - ``${HOME}``          → ``os.path.expanduser("~")``
+
+    Unknown tokens pass through unchanged.
+    """
+    return _TOKEN_PATTERN.sub(_resolve_token_match, template)
+
+
+def resolve_dict(obj):
+    """Recursively resolve every string value in nested dicts/lists."""
+    if isinstance(obj, str):
+        return resolve(obj)
+    if isinstance(obj, dict):
+        return {k: resolve_dict(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [resolve_dict(item) for item in obj]
+    return obj
+
+
+def project_root_regex() -> re.Pattern[str]:
+    """Compile a regex matching paths under any configured project root.
+
+    Used by ``core/cognition/capture/collector.py`` to detect project paths
+    inside captured session digests across macOS, Linux and Windows.
+    """
+    profile = load_profile()
+    roots = [os.path.expanduser(r) for r in profile.project_roots] or [
+        os.path.expanduser(r) for r in _DEFAULT_PROJECT_ROOTS
+    ]
+    alternation = "|".join(re.escape(r.rstrip("/").rstrip("\\")) for r in roots)
+    return re.compile(rf"({alternation})[/\\]([^\s/\\]+)")
+
+
+def _resolve_token_match(match: re.Match[str]) -> str:
+    return _resolve_token(match.group(1), match.group(0))
+
+
+def _resolve_token(name: str, original: str) -> str:
+    if name == "HOME":
+        return os.path.expanduser("~")
+    if name == "GIT_HOST":
+        return _env_or("ARKAOS_GIT_HOST", _DEFAULT_GIT_HOST)
+    if name == "VAULT_PATH":
+        env_val = _env_or("ARKAOS_VAULT_PATH", _env_or("ARKAOS_VAULT", ""))
+        return env_val or load_profile().vault_path
+    if name == "ARKA_OS_REPOS":
+        env_val = _env_or("ARKAOS_REPOS_ROOT", "")
+        return env_val or load_profile().repos_root
+    if name == "PROJECT_ROOTS":
+        env_val = _env_or("ARKAOS_PROJECT_ROOTS", "")
+        if env_val:
+            return env_val
+        return os.pathsep.join(load_profile().project_roots)
+    return original
+
+
+def _env_or(name: str, fallback: str) -> str:
+    value = os.environ.get(name, "")
+    return value if value else fallback
+
+
+def _project_from_raw(raw: dict) -> ProfileV3:
+    vault_path = raw.get("vaultPath") or raw.get("vault_path") or ""
+    if not vault_path:
+        raise ProfileMissingError(
+            "profile.json has no vaultPath. Run /arka setup to configure it."
+        )
+    repos_root = (
+        raw.get("reposRoot")
+        or raw.get("repos_root")
+        or _DEFAULT_REPOS_ROOT
+    )
+    project_roots = list(raw.get("projectRoots") or [])
+    if not project_roots:
+        project_roots = _derive_project_roots(raw.get("projectsDir", ""))
+    return ProfileV3(
+        version=str(raw.get("version", "2")),
+        vault_path=vault_path,
+        repos_root=repos_root,
+        project_roots=project_roots,
+        raw=raw,
+    )
+
+
+def _derive_project_roots(projects_dir_text: str) -> list[str]:
+    """Best-effort parse of the legacy free-text ``projectsDir`` field.
+
+    Looks for absolute paths on macOS/Linux/Windows. Falls back to
+    ``_DEFAULT_PROJECT_ROOTS`` when no path can be extracted. This keeps
+    the ~20K legacy users functional until ``npx arkaos update`` rewrites
+    their profile.json with the new ``projectRoots`` field.
+    """
+    if not projects_dir_text:
+        return list(_DEFAULT_PROJECT_ROOTS)
+    posix = r"(?:/Users|/home)/\S+?/(?:Herd|Work|AIProjects|code|repos)"
+    windows = r"[A-Z]:\\Users\\[^\s\\]+\\(?:Herd|Work|AIProjects|code|repos)"
+    pattern = re.compile(rf"({posix}|{windows})")
+    found = pattern.findall(projects_dir_text)
+    return found or list(_DEFAULT_PROJECT_ROOTS)