@agentikos/omega-os 0.2.0 → 0.19.5

package/omega/Agentik_Engine/omega_engine/memory.py

@@ -0,0 +1,271 @@
+"""Semantic memory — search across every conversation, mission, audit.
+
+The system has lots of facts on disk: Telegram history, .done.json
+files, audit verdicts, event logs. Each is queryable in isolation, but
+no cross-source semantic search exists. An operator asking "what did we
+decide about authentication last month?" has no answer.
+
+This module fills the gap. It uses the existing HybridRetriever (FTS5
+BM25 + hashed cosine — no extra deps, no external embedding API) to
+index every memory-worthy artefact and serve a single search query.
+
+Sources indexed:
+
+  * Telegram messages — every inbound + outbound row in
+    ``telegram-history.db``
+  * Worker .done.json — every per-task summary + pending_actions +
+    errors
+  * Audit verdicts — every row in ``audits.db`` with the verdict
+    summary + findings
+  * Event log — every event with a non-trivial payload
+
+The persistent SQLite index lives at
+``Agentik_Runtime/memory/index.db``. ``omega memory reindex`` rebuilds
+from scratch; ``omega memory search "<query>"`` returns ranked snippets
+with source attribution.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class MemoryHit:
+    source: str             # "telegram" | "done_json" | "audit" | "event"
+    ref: str                # source-specific id (topic_id, task_id, run_id, ...)
+    text: str
+    score: float
+    timestamp: int
+
+
+def _retriever_path(omega_home: str | Path) -> Path:
+    home = Path(omega_home)
+    p = home / "Agentik_Runtime" / "memory" / "index.db"
+    p.parent.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def _get_retriever(omega_home: str | Path):
+    """Build (or load) the HybridRetriever for memory."""
+    from omega_engine.rag.hybrid import HybridRetriever
+    return HybridRetriever(db_path=str(_retriever_path(omega_home)))
+
+
+def _iter_telegram(omega_home: Path):
+    db = omega_home / "Agentik_Runtime" / "telegram-history.db"
+    if not db.exists():
+        return
+    conn = sqlite3.connect(str(db))
+    try:
+        for row in conn.execute(
+            "SELECT id, topic_id, role, text, timestamp FROM messages"
+        ):
+            mid, topic_id, role, text, ts = row
+            yield {
+                "id": f"tg-{mid}",
+                "source": "telegram",
+                "ref": f"topic={topic_id} role={role}",
+                "text": text or "",
+                "timestamp": int(ts or 0),
+            }
+    finally:
+        conn.close()
+
+
+def _iter_done_json(omega_home: Path):
+    sess = omega_home / "Agentik_Runtime" / "sessions"
+    if not sess.is_dir():
+        return
+    for path in sess.glob("*/.done.json"):
+        try:
+            data = json.loads(path.read_text())
+        except (json.JSONDecodeError, OSError):
+            continue
+        task_id = path.parent.name
+        parts = [
+            f"status={data.get('status', '?')}",
+            data.get("summary", ""),
+        ]
+        if data.get("pending_actions"):
+            parts.append("pending: " + "; ".join(data["pending_actions"]))
+        if data.get("errors"):
+            parts.append("errors: " + "; ".join(data["errors"]))
+        yield {
+            "id": f"done-{task_id}",
+            "source": "done_json",
+            "ref": task_id,
+            "text": "\n".join(p for p in parts if p),
+            "timestamp": int(path.stat().st_mtime),
+        }
+
+
+def _iter_events(omega_home: Path):
+    """Index the event log — every task.created with an intent + every
+    task.completed gets a row. This catches dispatcher missions that
+    didn't produce a .done.json (no children to spawn).
+    """
+    db = omega_home / "Agentik_Runtime" / "eventlog" / "omega.db"
+    if not db.exists():
+        return
+    conn = sqlite3.connect(str(db))
+    try:
+        for row in conn.execute(
+            "SELECT task_id, type, payload, ts FROM events "
+            "WHERE type IN ('task.created', 'task.completed', 'task.failed')"
+        ):
+            task_id, type_, payload, ts = row
+            try:
+                p = json.loads(payload or "{}")
+            except json.JSONDecodeError:
+                continue
+            # task.created carries intent for top-level oracles
+            text_parts = [f"{type_}"]
+            if p.get("role"):
+                text_parts.append(f"role={p['role']}")
+            if p.get("mission_id"):
+                text_parts.append(f"mission={p['mission_id']}")
+            spec = p.get("spec") or {}
+            if spec.get("intent"):
+                text_parts.append(f"intent: {spec['intent'][:300]}")
+            elif spec.get("task"):
+                text_parts.append(f"task: {spec['task'][:300]}")
+            if p.get("reason"):
+                text_parts.append(f"reason: {p['reason'][:200]}")
+            text = "\n".join(text_parts)
+            if len(text) < 30:        # skip uninformative events
+                continue
+            yield {
+                "id": f"evt-{task_id}-{type_}-{ts}",
+                "source": "event",
+                "ref": f"{task_id}/{type_}",
+                "text": text,
+                "timestamp": int(ts or 0),
+            }
+    finally:
+        conn.close()
+
+
+def _iter_audits(omega_home: Path):
+    db = omega_home / "Agentik_Runtime" / "audits.db"
+    if not db.exists():
+        return
+    conn = sqlite3.connect(str(db))
+    try:
+        for row in conn.execute(
+            "SELECT run_id, audit_id, score, verified, verdict_json, timestamp "
+            "FROM audit_runs"
+        ):
+            run_id, audit_id, score, verified, vj, ts = row
+            verdict = json.loads(vj or "{}")
+            findings = verdict.get("findings", [])
+            parts = [
+                f"{audit_id} score={score} verified={bool(verified)}",
+                verdict.get("summary", ""),
+                "findings: " + "; ".join(
+                    str(f.get("message") or f.get("claim", ""))[:80]
+                    for f in findings[:5]
+                ),
+            ]
+            yield {
+                "id": f"audit-{run_id}",
+                "source": "audit",
+                "ref": run_id,
+                "text": "\n".join(p for p in parts if p),
+                "timestamp": int(ts or 0),
+            }
+    finally:
+        conn.close()
+
+
+def _build_documents(source_iter):
+    """Wrap source iterators into Document instances."""
+    from omega_engine.rag.base import Document
+    out = []
+    for d in source_iter:
+        out.append(Document(
+            id=d["id"], text=d["text"] or "(empty)",
+            metadata={
+                "source": d["source"],
+                "ref": d["ref"],
+                "timestamp": d["timestamp"],
+            },
+        ))
+    return out
+
+
+def reindex(omega_home: str | Path) -> dict[str, int]:
+    """Rebuild the index from scratch. Returns counts per source."""
+    home = Path(omega_home)
+    # Drop the old index to avoid leftover documents.
+    idx = _retriever_path(home)
+    if idx.exists():
+        idx.unlink()
+    retriever = _get_retriever(home)
+
+    counts: dict[str, int] = {"telegram": 0, "done_json": 0,
+                              "audit": 0, "event": 0, "total": 0}
+    tg_docs = _build_documents(_iter_telegram(home))
+    if tg_docs:
+        retriever.index(tg_docs)
+    counts["telegram"] = len(tg_docs)
+
+    dj_docs = _build_documents(_iter_done_json(home))
+    if dj_docs:
+        retriever.index(dj_docs)
+    counts["done_json"] = len(dj_docs)
+
+    au_docs = _build_documents(_iter_audits(home))
+    if au_docs:
+        retriever.index(au_docs)
+    counts["audit"] = len(au_docs)
+
+    ev_docs = _build_documents(_iter_events(home))
+    if ev_docs:
+        retriever.index(ev_docs)
+    counts["event"] = len(ev_docs)
+
+    counts["total"] = (counts["telegram"] + counts["done_json"]
+                       + counts["audit"] + counts["event"])
+    return counts
+
+
+def search(
+    omega_home: str | Path, query: str, *, k: int = 10,
+) -> list[MemoryHit]:
+    """Return the top-k semantic matches across all sources."""
+    home = Path(omega_home)
+    retriever = _get_retriever(home)
+    result = retriever.retrieve(query, k=k)
+    hits: list[MemoryHit] = []
+    for doc in result.documents:
+        meta = doc.metadata or {}
+        hits.append(MemoryHit(
+            source=str(meta.get("source", "?")),
+            ref=str(meta.get("ref", doc.id)),
+            text=doc.text[:600],
+            score=float(meta.get("score", 0.0)),
+            timestamp=int(meta.get("timestamp", 0)),
+        ))
+    return hits
+
+
+def index_stats(omega_home: str | Path) -> dict:
+    """Quick stats for the doctor and CLI."""
+    home = Path(omega_home)
+    p = _retriever_path(home)
+    if not p.exists():
+        return {"indexed": False, "total": 0}
+    conn = sqlite3.connect(str(p))
+    try:
+        total = conn.execute("SELECT COUNT(*) FROM docs_meta").fetchone()[0]
+    except sqlite3.OperationalError:
+        total = 0
+    finally:
+        conn.close()
+    return {"indexed": True, "total": int(total),
+            "size_bytes": p.stat().st_size}