arkaos 4.13.2 → 4.14.0

package/core/memory/semantic_store.py

@@ -0,0 +1,301 @@
+"""Session semantic memory store (F1-A2 — memory/learning reform).
+
+Per-turn transcript memory at ``~/.arkaos/session-memory.db``: every
+turn is summarised, sanitized, optionally embedded (multi-backend, see
+core/knowledge/embedding_backends.py) and stored. Retrieval happens in
+two honest modes, labeled exactly like core/knowledge/vector_store.py:
+
+- ``semantic``          — cosine over stored embeddings, computed OFF
+                          the hot path (the detached turn_capture worker
+                          precomputes neighbours into a session cache)
+- ``keyword-degraded``  — live LIKE query, ``score`` is None
+
+No vec0 dependency by design: the read path NEVER embeds, and the
+worker's brute-force cosine over the bounded scan window (most recent
+rows with embeddings) runs off-turn, so an extension adds failure modes
+without buying latency where it matters. Embeddings are only compared
+within the same backend+model+dims — cosine across vector spaces is a
+meaningless number dressed as similarity, so mismatched rows are skipped.
+
+The store is born self-healing (core/shared/sqlite_recovery.py).
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import sqlite3
+import uuid
+from contextlib import closing
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+DB_ENV = "ARKA_SESSION_MEMORY_DB"
+_SEMANTIC_SCAN_CAP = 2000  # most-recent embedded rows the worker scans
+_DEFAULT_RETENTION_DAYS = 90
+_DEFAULT_MAX_ROWS = 20_000
+
+
+class TurnRecord(BaseModel):
+    """One captured turn — embeddings always declare their provenance."""
+
+    id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+    ts: str = ""
+    session_id: str = ""
+    project_name: str = ""
+    cwd: str = ""
+    summary: str = ""
+    tools_used: list[str] = Field(default_factory=list)
+    file_paths: list[str] = Field(default_factory=list)
+    importance: float = 0.5
+    embedding: list[float] | None = None
+    embedding_backend: str = "none"
+    embedding_model: str = ""
+    dims: int = 0
+
+
+def default_db_path() -> Path:
+    env = os.environ.get(DB_ENV, "").strip()
+    if env:
+        return Path(env)
+    return Path.home() / ".arkaos" / "session-memory.db"
+
+
+_TAG_TOKEN_RE = re.compile(r"\[([^\]]*)\]")
+
+
+def neutralize_summary(text: str) -> str:
+    """Read-side neutralization before any context injection (OWASP LLM01):
+    collapse whitespace so a stored summary cannot forge a new line, and
+    defuse ``[tag]`` tokens into ``(tag)`` so it cannot impersonate
+    ``[SESSION-MEMORY]``/``[arka:*]`` markers. Write-time sanitization
+    only redacts client identifiers — it does not neutralize payloads."""
+    collapsed = " ".join(str(text).split())
+    return _TAG_TOKEN_RE.sub(r"(\1)", collapsed)
+
+
+def _cosine(a: list[float], b: list[float]) -> float:
+    # Callers guarantee equal dims (backend+model+dims guard upstream).
+    dot = sum(x * y for x, y in zip(a, b, strict=True))
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    return dot / norm if norm else 0.0
+
+
+class SessionMemoryStore:
+    """SQLite-backed turn memory; connections opened per operation."""
+
+    def __init__(self, db_path: str | Path | None = None) -> None:
+        self._db_path = Path(db_path) if db_path else default_db_path()
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+        from core.shared.sqlite_recovery import open_with_recovery
+        open_with_recovery(self._db_path, self._init_db)
+
+    def _conn(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        return conn
+
+    def _init_db(self) -> None:
+        # closing() closes the connection; the inner `conn` context
+        # commits the transaction — sqlite3's context manager alone
+        # commits but never closes (leaks in long-lived consumers).
+        with closing(self._conn()) as conn, conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS turns (
+                    id TEXT PRIMARY KEY,
+                    ts TEXT NOT NULL,
+                    session_id TEXT NOT NULL,
+                    project_name TEXT NOT NULL DEFAULT '',
+                    cwd TEXT NOT NULL DEFAULT '',
+                    summary TEXT NOT NULL DEFAULT '',
+                    tools_used TEXT NOT NULL DEFAULT '[]',
+                    file_paths TEXT NOT NULL DEFAULT '[]',
+                    importance REAL NOT NULL DEFAULT 0.5,
+                    embedding TEXT,
+                    embedding_backend TEXT NOT NULL DEFAULT 'none',
+                    embedding_model TEXT NOT NULL DEFAULT '',
+                    dims INTEGER NOT NULL DEFAULT 0
+                )
+            """)
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_turns_project_ts "
+                "ON turns (project_name, ts DESC)"
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_turns_session ON turns (session_id)"
+            )
+
+    def save(self, record: TurnRecord) -> None:
+        row = record.model_dump()
+        row["tools_used"] = json.dumps(record.tools_used)
+        row["file_paths"] = json.dumps(record.file_paths)
+        row["embedding"] = (
+            json.dumps(record.embedding) if record.embedding is not None else None
+        )
+        with closing(self._conn()) as conn, conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO turns (id, ts, session_id, project_name,"
+                " cwd, summary, tools_used, file_paths, importance, embedding,"
+                " embedding_backend, embedding_model, dims)"
+                " VALUES (:id, :ts, :session_id, :project_name, :cwd, :summary,"
+                " :tools_used, :file_paths, :importance, :embedding,"
+                " :embedding_backend, :embedding_model, :dims)",
+                row,
+            )
+
+    def _row_to_record(self, row: sqlite3.Row) -> TurnRecord:
+        data = dict(row)
+        data["tools_used"] = json.loads(data.get("tools_used") or "[]")
+        data["file_paths"] = json.loads(data.get("file_paths") or "[]")
+        raw_emb = data.get("embedding")
+        data["embedding"] = json.loads(raw_emb) if raw_emb else None
+        return TurnRecord(**data)
+
+    def recent(
+        self,
+        project_name: str | None = None,
+        limit: int = 20,
+        exclude_session: str | None = None,
+    ) -> list[TurnRecord]:
+        """Most recent turns, importance-weighted recency order."""
+        where, params = self._scope(project_name, exclude_session)
+        with closing(self._conn()) as conn:
+            rows = conn.execute(
+                f"SELECT * FROM turns {where} ORDER BY ts DESC LIMIT ?",
+                [*params, limit],
+            ).fetchall()
+        records = [self._row_to_record(r) for r in rows]
+        return sorted(records, key=lambda r: (r.importance, r.ts), reverse=True)
+
+    def _scope(
+        self, project_name: str | None, exclude_session: str | None
+    ) -> tuple[str, list]:
+        clauses, params = [], []
+        if project_name:
+            clauses.append("project_name = ?")
+            params.append(project_name)
+        if exclude_session:
+            clauses.append("session_id != ?")
+            params.append(exclude_session)
+        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+        return where, params
+
+    def keyword_search(
+        self, query: str, project_name: str | None = None, top_k: int = 3
+    ) -> list[dict]:
+        """Live substring match — labeled, never presented as semantic."""
+        words = [w for w in query.lower().split() if len(w) > 2][:5]
+        if not words:
+            return []
+        where, params = self._scope(project_name, None)
+        like = " OR ".join(["lower(summary) LIKE ?"] * len(words))
+        where = f"{where} AND ({like})" if where else f"WHERE {like}"
+        with closing(self._conn()) as conn:
+            rows = conn.execute(
+                f"SELECT * FROM turns {where} ORDER BY ts DESC LIMIT ?",
+                params + [f"%{w}%" for w in words] + [top_k],
+            ).fetchall()
+        return [
+            {**self._row_to_record(r).model_dump(exclude={"embedding"}),
+             "score": None, "retrieval": "keyword-degraded"}
+            for r in rows
+        ]
+
+    def semantic_neighbors(
+        self,
+        vector: list[float],
+        project_name: str | None = None,
+        top_k: int = 3,
+        exclude_session: str | None = None,
+        backend: str = "",
+        model: str = "",
+    ) -> list[dict]:
+        """Brute-force cosine over the bounded scan window (off-turn only).
+
+        Rows are SKIPPED unless backend, model AND dims all match the
+        query vector's provenance — cosine across different embedding
+        spaces yields a meaningless number dressed as similarity, the
+        exact silent lie the honesty labels exist to prevent.
+        """
+        where, params = self._scope(project_name, exclude_session)
+        prefix = f"{where} AND" if where else "WHERE"
+        with closing(self._conn()) as conn:
+            rows = conn.execute(
+                f"SELECT * FROM turns {prefix} embedding IS NOT NULL"
+                " ORDER BY ts DESC LIMIT ?",
+                [*params, _SEMANTIC_SCAN_CAP],
+            ).fetchall()
+        scored = []
+        for row in rows:
+            record = self._row_to_record(row)
+            if not record.embedding or record.dims != len(vector):
+                continue
+            if record.embedding_backend != backend or record.embedding_model != model:
+                continue
+            score = _cosine(vector, record.embedding)
+            scored.append(
+                {**record.model_dump(exclude={"embedding"}),
+                 "score": score, "retrieval": "semantic"}
+            )
+        scored.sort(key=lambda d: d["score"], reverse=True)
+        return scored[:top_k]
+
+    def backfill_candidates(self, limit: int = 10) -> list[TurnRecord]:
+        with closing(self._conn()) as conn:
+            rows = conn.execute(
+                "SELECT * FROM turns WHERE embedding IS NULL AND summary != ''"
+                " ORDER BY ts DESC LIMIT ?",
+                (limit,),
+            ).fetchall()
+        return [self._row_to_record(r) for r in rows]
+
+    def update_embedding(
+        self, turn_id: str, vector: list[float], backend: str, model: str
+    ) -> None:
+        with closing(self._conn()) as conn, conn:
+            conn.execute(
+                "UPDATE turns SET embedding = ?, embedding_backend = ?,"
+                " embedding_model = ?, dims = ? WHERE id = ?",
+                (json.dumps(vector), backend, model, len(vector), turn_id),
+            )
+
+    def prune(
+        self,
+        retention_days: int = _DEFAULT_RETENTION_DAYS,
+        max_rows: int = _DEFAULT_MAX_ROWS,
+    ) -> int:
+        cutoff = (
+            datetime.now(UTC) - timedelta(days=retention_days)
+        ).isoformat()
+        with closing(self._conn()) as conn, conn:
+            removed = conn.execute(
+                "DELETE FROM turns WHERE ts < ?", (cutoff,)
+            ).rowcount
+            removed += conn.execute(
+                "DELETE FROM turns WHERE id NOT IN"
+                " (SELECT id FROM turns ORDER BY ts DESC LIMIT ?)",
+                (max_rows,),
+            ).rowcount
+        return removed
+
+    def vacuum(self) -> None:
+        """Nightly-maintenance compaction (never on a hook hot path)."""
+        with closing(self._conn()) as conn:
+            conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+            conn.execute("VACUUM")
+
+    def stats(self) -> dict:
+        with closing(self._conn()) as conn:
+            total = conn.execute("SELECT COUNT(*) AS c FROM turns").fetchone()["c"]
+            backends = dict(
+                conn.execute(
+                    "SELECT embedding_backend, COUNT(*) FROM turns"
+                    " GROUP BY embedding_backend"
+                ).fetchall()
+            )
+        return {"total_turns": total, "by_embedding_backend": backends}

package/core/memory/turn_capture.py

@@ -0,0 +1,277 @@
+"""Detached turn-capture worker (F1-A2 — memory/learning reform).
+
+Enqueued by the Stop hook via ``Popen(start_new_session=True)`` — ALL
+embedding/summarisation cost is paid here, off-turn, never on a hook's
+hot path. Pipeline per turn:
+
+    transcript tail → summary → sanitize (MANDATORY — recipes precedent:
+    no sanitizer config ⇒ text is refused, metadata-only record) →
+    importance → embed (multi-backend, provenance declared) → store →
+    precompute cross-session semantic neighbours into the session cache
+    PRODUCED FOR the F1-A3 retrieval layer (no reader exists in this PR)
+    → amortised maintenance (embedding backfill + retention prune).
+
+CLI: ``python3 -m core.memory.turn_capture (<session_id> <transcript_path> [cwd] | maintenance)``
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import tempfile
+from datetime import UTC, datetime
+from pathlib import Path
+
+from core.knowledge.embedding_backends import EmbeddingResult, embed
+from core.memory.semantic_store import SessionMemoryStore, TurnRecord
+
+_TAIL_BYTES = 262_144  # bounded transcript read
+_SUMMARY_CHARS = 600
+_CACHE_DIRNAME = "context-cache"
+_NEIGHBOUR_K = 3
+_BACKFILL_BATCH = 10
+
+
+def _config_enabled() -> bool:
+    """``memory.sessionMemory`` (default True) + env kill-switch."""
+    if os.environ.get("ARKA_SESSION_MEMORY", "").strip() == "0":
+        return False
+    config_path = Path.home() / ".arkaos" / "config.json"
+    try:
+        data = json.loads(config_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return True
+    memory_cfg = data.get("memory") or {}
+    return bool(memory_cfg.get("sessionMemory", True))
+
+
+def _read_tail(transcript_path: str) -> str:
+    path = Path(transcript_path)
+    if not path.is_file():
+        return ""
+    size = path.stat().st_size
+    with path.open("rb") as fh:
+        fh.seek(max(0, size - _TAIL_BYTES))
+        return fh.read().decode("utf-8", "replace")
+
+
+def _last_assistant_text(transcript_path: str, raw: str) -> str:
+    try:
+        from core.workflow.flow_enforcer import _load_last_assistant_messages
+        msgs = _load_last_assistant_messages(transcript_path, 1, raw_text=raw)
+        return msgs[-1] if msgs else ""
+    except Exception:
+        return ""
+
+
+def _parse_tool_uses(raw: str) -> tuple[list[str], list[str]]:
+    """Collect tool names + touched file paths from the transcript tail."""
+    tools: list[str] = []
+    paths: list[str] = []
+    for line in raw.splitlines()[-400:]:
+        try:
+            entry = json.loads(line)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        content = (entry.get("message") or {}).get("content") or []
+        for block in content if isinstance(content, list) else []:
+            if not isinstance(block, dict) or block.get("type") != "tool_use":
+                continue
+            name = str(block.get("name") or "")
+            if name and name not in tools:
+                tools.append(name)
+            file_path = (block.get("input") or {}).get("file_path")
+            if isinstance(file_path, str) and file_path not in paths:
+                paths.append(file_path)
+    return tools[:20], paths[:20]
+
+
+def _sanitized_summary(text: str) -> tuple[str, bool]:
+    """Sanitize or refuse — unsanitized text is never persisted."""
+    if not text:
+        return "", True
+    try:
+        from core.evals.sanitizer import SanitizerConfigMissing, sanitize_text
+    except Exception:
+        return "", False
+    try:
+        clean, _counts = sanitize_text(text[:_SUMMARY_CHARS])
+        return clean, True
+    except SanitizerConfigMissing:
+        return "", False
+
+
+_MCP_TOOL_RE = re.compile(r"^mcp__[^_]((?!__).)*__")
+
+
+def _safe_tool_names(tools: list[str], sanitized_ok: bool) -> list[str]:
+    """Tool names are metadata but not neutral: ``mcp__<server>__<tool>``
+    carries an operator-chosen server segment that can name a client.
+    Sanitizer available → redact through it; unavailable → strip the
+    server segment entirely (fail closed, v2.18.0 precedent)."""
+    if sanitized_ok:
+        try:
+            from core.evals.sanitizer import sanitize_text
+            return [sanitize_text(name)[0] for name in tools]
+        except Exception:
+            pass
+    return [_MCP_TOOL_RE.sub("mcp__", name) for name in tools]
+
+
+def _importance(text: str, tools: list[str]) -> float:
+    score = 0.5
+    lowered = text.lower()
+    if re.search(r"\berror\b|\bfailed\b|\bexception\b|traceback", lowered):
+        score += 0.2
+    if re.search(r"\[arka:qg:|verdict|approved|rejected", lowered):
+        score += 0.15
+    if any(t in ("Write", "Edit", "NotebookEdit") for t in tools):
+        score += 0.1
+    return min(score, 1.0)
+
+
+def _cache_path(session_id: str) -> Path:
+    cache_dir = Path.home() / ".arkaos" / _CACHE_DIRNAME
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir / f"session-memory-{session_id}.json"
+
+
+def _precompute_cache(
+    store: SessionMemoryStore, record: TurnRecord, session_id: str
+) -> None:
+    """Rank cross-session neighbours NOW so the next prompt just reads."""
+    if record.embedding is not None:
+        items = store.semantic_neighbors(
+            record.embedding, record.project_name or None,
+            top_k=_NEIGHBOUR_K, exclude_session=session_id,
+            backend=record.embedding_backend, model=record.embedding_model,
+        )
+        retrieval = "semantic"
+    else:
+        items, retrieval = [], "keyword-degraded"
+    payload = {
+        "version": 1,
+        "session_id": session_id,
+        "computed_at": datetime.now(UTC).isoformat(),
+        "retrieval": retrieval,
+        "embedding_backend": record.embedding_backend,
+        "embedding_model": record.embedding_model,
+        "dims": record.dims,
+        "items": [
+            {"summary": i["summary"][:200], "project_name": i["project_name"],
+             "ts": i["ts"], "score": i["score"], "retrieval": i["retrieval"]}
+            for i in items
+        ],
+    }
+    target = _cache_path(session_id)
+    # Unique tmp per writer: overlapping workers for the same session must
+    # not share a tmp path, or write+replace stops being atomic.
+    fd, tmp_name = tempfile.mkstemp(
+        dir=target.parent, prefix=f"{target.name}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(json.dumps(payload, indent=2))
+        os.replace(tmp_name, target)
+    except OSError:
+        Path(tmp_name).unlink(missing_ok=True)
+
+
+def _maintenance(store: SessionMemoryStore) -> None:
+    """Amortised per run: small embedding backfill + retention prune."""
+    for candidate in store.backfill_candidates(_BACKFILL_BATCH):
+        result: EmbeddingResult = embed(candidate.summary)
+        if result.vector is not None:
+            store.update_embedding(
+                candidate.id, result.vector, result.backend, result.model
+            )
+    store.prune()
+
+
+def capture_turn(session_id: str, transcript_path: str, cwd: str = "") -> int:
+    if not _config_enabled() or not session_id:
+        return 0
+    raw = _read_tail(transcript_path)
+    if not raw:
+        return 0
+    text = _last_assistant_text(transcript_path, raw)
+    tools, paths = _parse_tool_uses(raw)
+    summary, sanitized_ok = _sanitized_summary(text)
+    tools = _safe_tool_names(tools, sanitized_ok)
+    if not sanitized_ok:
+        paths = []  # refuse anything textual when the sanitizer is absent
+    result: EmbeddingResult = embed(summary) if summary else EmbeddingResult()
+    record = TurnRecord(
+        ts=datetime.now(UTC).isoformat(),
+        session_id=session_id,
+        project_name=Path(cwd).name if cwd else "",
+        cwd=cwd,
+        summary=summary,
+        tools_used=tools,
+        file_paths=paths,
+        importance=_importance(text, tools),
+        embedding=result.vector,
+        embedding_backend=result.backend,
+        embedding_model=result.model,
+        dims=result.dims,
+    )
+    store = SessionMemoryStore()
+    store.save(record)
+    _precompute_cache(store, record, session_id)
+    _maintenance(store)
+    return 0
+
+
+_MAINTENANCE_MAX_BATCHES = 50
+
+
+def run_maintenance() -> int:
+    """Nightly job (F1-A4, scheduler python_module): full embedding
+    backfill in batches, retention prune, vacuum. Off-turn by design —
+    the per-turn worker only amortises a small slice of this."""
+    if not _config_enabled():
+        return 0
+    from core.memory.semantic_store import default_db_path
+
+    if not default_db_path().is_file():
+        return 0
+    store = SessionMemoryStore()
+    for _ in range(_MAINTENANCE_MAX_BATCHES):
+        candidates = store.backfill_candidates(_BACKFILL_BATCH)
+        if not candidates:
+            break
+        for candidate in candidates:
+            result = embed(candidate.summary)
+            if result.vector is None:
+                return 0  # backend degraded — retry next night, never spin
+            store.update_embedding(
+                candidate.id, result.vector, result.backend, result.model
+            )
+    store.prune()
+    store.vacuum()
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = argv if argv is not None else sys.argv[1:]
+    if args and args[0] == "maintenance":
+        try:
+            return run_maintenance()
+        except Exception:  # detached worker must die quietly
+            return 0
+    if len(args) < 2:
+        print(
+            "usage: python3 -m core.memory.turn_capture"
+            " (<session_id> <transcript_path> [cwd] | maintenance)"
+        )
+        return 2
+    try:
+        return capture_turn(args[0], args[1], args[2] if len(args) > 2 else "")
+    except Exception:  # detached worker must die quietly
+        return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())