@simbimbo/memory-ocmemog 0.1.4

package/brain/runtime/memory/tool_catalog.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import json
+from typing import Dict, Any
+
+from brain.runtime.instrumentation import emit_event
+from brain.runtime import state_store
+from brain.runtime.memory import store
+
+
+def _ensure_table(conn) -> None:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS tool_catalog (
+            tool_id TEXT PRIMARY KEY,
+            description TEXT,
+            permission_class TEXT,
+            capability_tags TEXT,
+            first_seen TEXT DEFAULT (datetime('now')),
+            last_used TEXT
+        )
+        """
+    )
+
+
+def record_tool_metadata(metadata: Dict[str, Any]) -> None:
+    conn = store.connect()
+    _ensure_table(conn)
+    tool_id = metadata.get("tool_id")
+    description = metadata.get("description", "")
+    permission_class = metadata.get("permission_class", "restricted")
+    capability_tags = json.dumps(metadata.get("capability_tags", []) or [])
+    conn.execute(
+        """
+        INSERT INTO tool_catalog (tool_id, description, permission_class, capability_tags)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT(tool_id) DO UPDATE SET
+            description=excluded.description,
+            permission_class=excluded.permission_class,
+            capability_tags=excluded.capability_tags
+        """,
+        (tool_id, description, permission_class, capability_tags),
+    )
+    conn.commit()
+    conn.close()
+    emit_event(
+        state_store.reports_dir() / "brain_memory.log.jsonl",
+        "brain_memory_tool_catalog_update",
+        status="ok",
+        tool_id=tool_id,
+    )
+
+
+def record_tool_usage(tool_id: str) -> None:
+    conn = store.connect()
+    _ensure_table(conn)
+    conn.execute(
+        "UPDATE tool_catalog SET last_used=datetime('now') WHERE tool_id=?",
+        (tool_id,),
+    )
+    conn.commit()
+    conn.close()
+    emit_event(
+        state_store.reports_dir() / "brain_memory.log.jsonl",
+        "brain_memory_tool_catalog_update",
+        status="ok",
+        tool_id=tool_id,
+    )

package/brain/runtime/memory/unresolved_state.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from typing import Dict, List
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+TYPES = {
+    "unresolved_question",
+    "paused_task",
+    "interrupted_thread",
+    "pending_decision",
+    "incomplete_hypothesis",
+}
+
+
+def _connect() -> sqlite3.Connection:
+    path = state_store.data_dir() / "unresolved_state.db"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS unresolved_state (
+            state_id INTEGER PRIMARY KEY AUTOINCREMENT,
+            state_type TEXT NOT NULL,
+            reference TEXT,
+            summary TEXT,
+            created_at TEXT NOT NULL DEFAULT (datetime('now')),
+            resolved INTEGER NOT NULL DEFAULT 0
+        )
+        """
+    )
+    return conn
+
+
+def add_unresolved_state(state_type: str, reference: str, summary: str) -> int:
+    if state_type not in TYPES:
+        state_type = "unresolved_question"
+    conn = _connect()
+    conn.execute(
+        "INSERT INTO unresolved_state (state_type, reference, summary, created_at, resolved) VALUES (?, ?, ?, ?, 0)",
+        (state_type, reference, summary, time.strftime("%Y-%m-%d %H:%M:%S")),
+    )
+    conn.commit()
+    row = conn.execute("SELECT last_insert_rowid()").fetchone()
+    conn.close()
+    emit_event(LOGFILE, "brain_unresolved_state_added", status="ok", state_type=state_type)
+    return int(row[0]) if row else 0
+
+
+def list_unresolved_state(limit: int = 20) -> List[Dict[str, object]]:
+    conn = _connect()
+    rows = conn.execute(
+        "SELECT state_id, state_type, reference, summary, created_at FROM unresolved_state WHERE resolved=0 ORDER BY created_at DESC LIMIT ?",
+        (limit,),
+    ).fetchall()
+    conn.close()
+    return [dict(row) for row in rows]
+
+
+def resolve_unresolved_state(state_id: int) -> bool:
+    conn = _connect()
+    conn.execute("UPDATE unresolved_state SET resolved=1 WHERE state_id=?", (state_id,))
+    conn.commit()
+    conn.close()
+    emit_event(LOGFILE, "brain_unresolved_state_resolved", status="ok", state_id=state_id)
+    return True
+
+
+def list_unresolved_state_for_references(references: List[str], limit: int = 20) -> List[Dict[str, object]]:
+    refs = [str(ref).strip() for ref in references if str(ref).strip()]
+    if not refs:
+        return []
+    placeholders = ",".join("?" for _ in refs)
+    conn = _connect()
+    rows = conn.execute(
+        f"SELECT state_id, state_type, reference, summary, created_at FROM unresolved_state WHERE resolved=0 AND reference IN ({placeholders}) ORDER BY created_at DESC LIMIT ?",
+        (*refs, limit),
+    ).fetchall()
+    conn.close()
+    return [dict(row) for row in rows]
+
+
+def count_unresolved_state() -> int:
+    conn = _connect()
+    row = conn.execute("SELECT COUNT(*) FROM unresolved_state WHERE resolved=0").fetchone()
+    conn.close()
+    return int(row[0]) if row else 0

package/brain/runtime/memory/vector_index.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import json
+import math
+import threading
+from typing import Any, Dict, List, Iterable
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.memory import embedding_engine, store, memory_links
+from brain.runtime.security import redaction
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+EMBEDDING_TABLES: tuple[str, ...] = (
+    "knowledge",
+    "runbooks",
+    "lessons",
+    "directives",
+    "reflections",
+    "tasks",
+)
+_REBUILD_LOCK = threading.Lock()
+_WRITE_CHUNK_SIZE = 64
+
+
+def _ensure_vector_table(conn) -> None:
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS vector_embeddings (
+            id TEXT PRIMARY KEY,
+            source_type TEXT NOT NULL,
+            source_id TEXT NOT NULL,
+            embedding TEXT NOT NULL,
+            created_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )
+        """
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_vector_embeddings_source ON vector_embeddings (source_type, source_id)"
+    )
+
+
+def _cosine_similarity(a: List[float], b: List[float]) -> float:
+    if not a or not b:
+        return 0.0
+    size = min(len(a), len(b))
+    if size == 0:
+        return 0.0
+    a2 = a[:size]
+    b2 = b[:size]
+    dot = sum(x * y for x, y in zip(a2, b2))
+    mag_a = math.sqrt(sum(x * x for x in a2))
+    mag_b = math.sqrt(sum(x * x for x in b2))
+    if mag_a == 0.0 or mag_b == 0.0:
+        return 0.0
+    return dot / (mag_a * mag_b)
+
+
+def insert_memory(memory_id: int, content: str, confidence: float, *, source_type: str = "knowledge") -> None:
+    source_type = source_type if source_type in EMBEDDING_TABLES else "knowledge"
+    redacted_content, changed = redaction.redact_text(content)
+    embedding = embedding_engine.generate_embedding(redacted_content)
+    metadata_json = json.dumps({"redacted": changed, "source_type": source_type})
+
+    def _write() -> None:
+        conn = store.connect()
+        try:
+            _ensure_vector_table(conn)
+            conn.execute(
+                "INSERT INTO memory_index (source, confidence, metadata_json, content, schema_version) VALUES (?, ?, ?, ?, ?)",
+                (
+                    f"{source_type}:{memory_id}",
+                    confidence,
+                    metadata_json,
+                    redacted_content,
+                    store.SCHEMA_VERSION,
+                ),
+            )
+            if embedding:
+                emit_event(LOGFILE, "brain_memory_embedding_generated", status="ok", source_id=str(memory_id))
+                conn.execute(
+                    """
+                    INSERT INTO vector_embeddings (id, source_type, source_id, embedding)
+                    VALUES (?, ?, ?, ?)
+                    ON CONFLICT(id) DO UPDATE SET embedding=excluded.embedding
+                    """,
+                    (f"{source_type}:{memory_id}", source_type, str(memory_id), json.dumps(embedding)),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+
+    store.submit_write(_write, timeout=30.0)
+
+
+def _load_table_rows(table: str, *, limit: int | None = None, descending: bool = False) -> List[Dict[str, Any]]:
+    conn = store.connect()
+    try:
+        order = "DESC" if descending else "ASC"
+        if limit is None:
+            rows = conn.execute(
+                f"SELECT id, content, confidence, metadata_json FROM {table} ORDER BY id {order}",
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                f"SELECT id, content, confidence, metadata_json FROM {table} ORDER BY id {order} LIMIT ?",
+                (limit,),
+            ).fetchall()
+    finally:
+        conn.close()
+    return [dict(row) for row in rows]
+
+
+def _prepare_embedding_rows(rows: Iterable[Dict[str, Any]], *, table: str) -> List[Dict[str, Any]]:
+    prepared: List[Dict[str, Any]] = []
+    for row in rows:
+        content = str(row.get("content") or "")
+        redacted_content, changed = redaction.redact_text(content)
+        embedding = embedding_engine.generate_embedding(redacted_content)
+        if not embedding:
+            continue
+        try:
+            metadata = json.loads(row.get("metadata_json") or "{}")
+        except Exception:
+            metadata = {}
+        metadata["redacted"] = changed
+        prepared.append(
+            {
+                "id": int(row["id"]),
+                "content": redacted_content,
+                "confidence": float(row.get("confidence") or 0.0),
+                "metadata_json": json.dumps(metadata),
+                "embedding": json.dumps(embedding),
+                "source_type": table,
+            }
+        )
+    return prepared
+
+
+def _write_embedding_chunk(table: str, rows: List[Dict[str, Any]]) -> int:
+    if not rows:
+        return 0
+
+    def _write() -> int:
+        conn = store.connect()
+        try:
+            _ensure_vector_table(conn)
+            for row in rows:
+                conn.execute(
+                    f"UPDATE {table} SET content=?, metadata_json=? WHERE id=?",
+                    (row["content"], row["metadata_json"], row["id"]),
+                )
+                conn.execute(
+                    """
+                    INSERT INTO vector_embeddings (id, source_type, source_id, embedding)
+                    VALUES (?, ?, ?, ?)
+                    ON CONFLICT(id) DO UPDATE SET embedding=excluded.embedding
+                    """,
+                    (f"{table}:{row['id']}", table, str(row["id"]), row["embedding"]),
+                )
+            conn.commit()
+            return len(rows)
+        finally:
+            conn.close()
+
+    return int(store.submit_write(_write, timeout=60.0))
+
+
+def index_memory(limit: int = 100, *, tables: Iterable[str] | None = None) -> int:
+    emit_event(LOGFILE, "brain_memory_vector_index_start", status="ok")
+    count = 0
+    for table in (tables or EMBEDDING_TABLES):
+        if table not in EMBEDDING_TABLES:
+            continue
+        prepared = _prepare_embedding_rows(_load_table_rows(table, limit=limit, descending=True), table=table)
+        for offset in range(0, len(prepared), _WRITE_CHUNK_SIZE):
+            count += _write_embedding_chunk(table, prepared[offset: offset + _WRITE_CHUNK_SIZE])
+    emit_event(LOGFILE, "brain_memory_vector_index_complete", status="ok", indexed=count)
+    return count
+
+
+def rebuild_vector_index(*, tables: Iterable[str] | None = None) -> int:
+    emit_event(LOGFILE, "brain_memory_vector_rebuild_start", status="ok")
+    if not _REBUILD_LOCK.acquire(blocking=False):
+        emit_event(LOGFILE, "brain_memory_vector_rebuild_complete", status="skipped", reason="already_running")
+        return 0
+    count = 0
+    try:
+        requested_tables = [table for table in (tables or EMBEDDING_TABLES) if table in EMBEDDING_TABLES]
+
+        def _clear() -> None:
+            conn = store.connect()
+            try:
+                _ensure_vector_table(conn)
+                if requested_tables:
+                    conn.executemany(
+                        "DELETE FROM vector_embeddings WHERE source_type = ?",
+                        [(table,) for table in requested_tables],
+                    )
+                conn.commit()
+            finally:
+                conn.close()
+
+        store.submit_write(_clear, timeout=60.0)
+        for table in requested_tables:
+            prepared = _prepare_embedding_rows(_load_table_rows(table), table=table)
+            for offset in range(0, len(prepared), _WRITE_CHUNK_SIZE):
+                count += _write_embedding_chunk(table, prepared[offset: offset + _WRITE_CHUNK_SIZE])
+    finally:
+        _REBUILD_LOCK.release()
+    emit_event(LOGFILE, "brain_memory_vector_rebuild_complete", status="ok", indexed=count)
+    return count
+
+
+def search_memory(query: str, limit: int = 5) -> List[Dict[str, Any]]:
+    emit_event(LOGFILE, "brain_memory_vector_search_start", status="ok")
+    conn = store.connect()
+    _ensure_vector_table(conn)
+
+    query_embedding = embedding_engine.generate_embedding(query)
+    results: List[Dict[str, Any]] = []
+
+    if query_embedding:
+        rows = conn.execute("SELECT id, source_type, source_id, embedding FROM vector_embeddings").fetchall()
+        scored: List[Dict[str, Any]] = []
+        for row in rows:
+            try:
+                emb = json.loads(row["embedding"])
+                emb_list = [float(x) for x in emb]
+            except Exception:
+                continue
+            score = _cosine_similarity(query_embedding, emb_list)
+            scored.append(
+                {
+                    "entry_id": row["id"],
+                    "source_type": row["source_type"],
+                    "source_id": row["source_id"],
+                    "score": round(score, 6),
+                }
+            )
+        scored.sort(key=lambda item: item["score"], reverse=True)
+        results = scored[:limit]
+
+    if not results:
+        rows = conn.execute(
+            "SELECT id, source, content, confidence, metadata_json FROM memory_index WHERE content LIKE ? ORDER BY id DESC LIMIT ?",
+            (f"%{query}%", limit),
+        ).fetchall()
+        fallback_results: List[Dict[str, Any]] = []
+        for row in rows:
+            source_ref = str(row["source"] or "")
+            source_type, _, source_id = source_ref.partition(":")
+            canonical_type = source_type if source_type in EMBEDDING_TABLES else "knowledge"
+            canonical_ref = f"{canonical_type}:{source_id}" if source_id else source_ref
+            fallback_results.append(
+                {
+                    "entry_id": canonical_ref,
+                    "source_type": canonical_type,
+                    "source_id": source_id or str(row["id"]),
+                    "score": float(row["confidence"] or 0.0),
+                    "content": str(row["content"] or "")[:240],
+                    "links": memory_links.get_memory_links(canonical_ref),
+                }
+            )
+        results = fallback_results
+
+    conn.close()
+    emit_event(LOGFILE, "brain_memory_vector_search_complete", status="ok", result_count=len(results))
+    return results

package/brain/runtime/model_roles.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from brain.runtime import config
+
+
+def get_model_for_role(role: str) -> str:
+    if role == "memory":
+        return config.OCMEMOG_MEMORY_MODEL
+    if role == "embedding":
+        return config.OCMEMOG_OPENAI_EMBED_MODEL
+    return config.OCMEMOG_MEMORY_MODEL

package/brain/runtime/model_router.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from brain.runtime import config
+
+
+@dataclass(frozen=True)
+class ModelSelection:
+    provider_id: str = ""
+    model: str = ""
+
+
+def get_provider_for_role(role: str) -> ModelSelection:
+    if role != "embedding":
+        return ModelSelection()
+    provider = (config.BRAIN_EMBED_MODEL_PROVIDER or "").strip().lower()
+    if provider in {"openai", "openai_compatible", "openai-compatible"}:
+        return ModelSelection(provider_id="openai", model=config.OCMEMOG_OPENAI_EMBED_MODEL)
+    if provider in {"ollama", "local-ollama"}:
+        return ModelSelection(provider_id="ollama", model=config.OCMEMOG_OLLAMA_EMBED_MODEL)
+    return ModelSelection()

package/brain/runtime/providers.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+import os
+import urllib.request
+
+from brain.runtime import config, state_store
+from brain.runtime.instrumentation import emit_event
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+
+class ProviderExecute:
+    def execute_embedding_call(self, selection, text: str) -> dict[str, object]:
+        provider_id = getattr(selection, "provider_id", "") or ""
+        model = getattr(selection, "model", "") or config.OCMEMOG_OPENAI_EMBED_MODEL
+        if provider_id == "openai":
+            api_key = os.environ.get("OCMEMOG_OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY")
+            if not api_key:
+                return {}
+            url = f"{config.OCMEMOG_OPENAI_API_BASE.rstrip('/')}/embeddings"
+            payload = json.dumps({"model": model, "input": text}).encode("utf-8")
+            req = urllib.request.Request(url, data=payload, method="POST")
+            req.add_header("Authorization", f"Bearer {api_key}")
+            req.add_header("Content-Type", "application/json")
+            try:
+                with urllib.request.urlopen(req, timeout=20) as resp:
+                    data = json.loads(resp.read().decode("utf-8"))
+            except Exception as exc:
+                emit_event(LOGFILE, "brain_embedding_provider_error", status="error", provider="openai", error=str(exc))
+                return {}
+            try:
+                embedding = data["data"][0]["embedding"]
+            except Exception as exc:
+                emit_event(LOGFILE, "brain_embedding_provider_error", status="error", provider="openai", error=str(exc))
+                return {}
+            return {"embedding": embedding}
+
+        if provider_id == "ollama":
+            url = f"{config.OCMEMOG_OLLAMA_HOST.rstrip('/')}/api/embeddings"
+            payload = json.dumps({"model": model, "prompt": text}).encode("utf-8")
+            req = urllib.request.Request(url, data=payload, method="POST")
+            req.add_header("Content-Type", "application/json")
+            try:
+                with urllib.request.urlopen(req, timeout=20) as resp:
+                    data = json.loads(resp.read().decode("utf-8"))
+            except Exception as exc:
+                emit_event(LOGFILE, "brain_embedding_provider_error", status="error", provider="ollama", error=str(exc))
+                return {}
+            embedding = data.get("embedding")
+            if not isinstance(embedding, list):
+                emit_event(LOGFILE, "brain_embedding_provider_error", status="error", provider="ollama", error="invalid_embedding")
+                return {}
+            return {"embedding": embedding}
+
+        return {}
+
+
+provider_execute = ProviderExecute()

package/brain/runtime/security/__init__.py

@@ -0,0 +1,3 @@
+from . import redaction
+
+__all__ = ["redaction"]

package/brain/runtime/security/redaction.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import re
+
+EMAIL_RE = re.compile(r"\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b")
+PHONE_RE = re.compile(r"\b(?:\+?1[-.\s]?)?(?:\(?\d{3}\)?[-.\s]?){2}\d{4}\b")
+
+
+def redact_text(text: str) -> tuple[str, bool]:
+    if not isinstance(text, str):
+        return "", False
+    redacted = EMAIL_RE.sub("[redacted-email]", text)
+    redacted = PHONE_RE.sub("[redacted-phone]", redacted)
+    return redacted, redacted != text

package/brain/runtime/state_store.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from . import storage_paths
+
+
+def root_dir() -> Path:
+    return storage_paths.root_dir()
+
+
+def data_dir() -> Path:
+    return storage_paths.data_dir()
+
+
+def memory_dir() -> Path:
+    return storage_paths.memory_dir()
+
+
+def reports_dir() -> Path:
+    return storage_paths.reports_dir()
+
+
+def memory_db_path() -> Path:
+    return storage_paths.memory_db_path()

package/brain/runtime/storage_paths.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def root_dir() -> Path:
+    configured = os.environ.get("OCMEMOG_STATE_DIR") or os.environ.get("BRAIN_STATE_DIR")
+    if configured:
+        base = Path(configured).expanduser()
+    else:
+        base = Path(__file__).resolve().parents[2] / ".ocmemog-state"
+    base.mkdir(parents=True, exist_ok=True)
+    return base
+
+
+def data_dir() -> Path:
+    path = root_dir() / "data"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def memory_dir() -> Path:
+    path = root_dir() / "memory"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def reports_dir() -> Path:
+    path = root_dir() / "reports"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def memory_db_path() -> Path:
+    override = os.environ.get("OCMEMOG_DB_PATH")
+    if override:
+        path = Path(override).expanduser()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        return path
+    return memory_dir() / "brain_memory.sqlite3"