@simbimbo/memory-ocmemog 0.1.4

package/brain/runtime/memory/distill.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Dict, Any, List
+
+from brain.runtime.instrumentation import emit_event
+from brain.runtime import state_store
+from brain.runtime.memory import candidate, provenance, store
+from brain.runtime.security import redaction
+from brain.runtime import inference
+from brain.runtime import model_roles
+
+
+def _normalize(text: str) -> str:
+    return re.sub(r"\s+", " ", text.lower()).strip()
+
+
+def _heuristic_summary(text: str) -> str:
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    if not lines:
+        return ""
+    return lines[0][:240]
+
+
+def _verification_points(text: str) -> List[str]:
+    points = []
+    if "verify" in text.lower():
+        points.append("Verify referenced assumptions")
+    if "risk" in text.lower():
+        points.append("Validate risk and mitigation")
+    if not points:
+        points.append("Confirm key facts before promotion")
+    return points[:3]
+
+
+def _candidate_score(summary: str, source: str) -> float:
+    if not source:
+        return 0.0
+    ratio = len(summary) / max(1, len(source))
+    score = 1.0 - min(1.0, ratio * 0.5)
+    return round(max(0.1, min(1.0, score)), 3)
+
+
+def _row_value(row: Any, key: str, fallback_index: int | None = None) -> Any:
+    if isinstance(row, dict):
+        return row.get(key)
+    try:
+        return row[key]
+    except Exception:
+        if fallback_index is None:
+            return None
+        try:
+            return row[fallback_index]
+        except Exception:
+            return None
+
+
+
+def distill_experiences(limit: int = 10) -> List[Dict[str, Any]]:
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_start", status="ok")
+    conn = store.connect()
+    rows = conn.execute(
+        "SELECT id, task_id, outcome, source_module, metadata_json FROM experiences ORDER BY id DESC LIMIT ?",
+        (limit,),
+    ).fetchall()
+    conn.close()
+
+    distilled: List[Dict[str, Any]] = []
+    seen = set()
+
+    for row in rows:
+        source_id = _row_value(row, "id", 0)
+        task_id = _row_value(row, "task_id", 1)
+        content = _row_value(row, "outcome", 2) or ""
+        source_module = _row_value(row, "source_module", 3)
+        raw_metadata = _row_value(row, "metadata_json", 4) or "{}"
+        try:
+            experience_metadata = json.loads(raw_metadata) if isinstance(raw_metadata, str) else dict(raw_metadata or {})
+        except Exception:
+            experience_metadata = {}
+        content, _ = redaction.redact_text(content)
+
+        summary = ""
+        try:
+            model = model_roles.get_model_for_role("memory")
+            result = inference.infer(
+                f"Distill this experience into a concise summary:\n\n{content}".strip(),
+                provider_name=model,
+            )
+            if result.get("status") == "ok":
+                summary = str(result.get("output", "")).strip()
+        except Exception:
+            summary = ""
+
+        if not summary or len(summary) > len(content):
+            summary = _heuristic_summary(content)
+
+        summary, _ = redaction.redact_text(summary)
+        norm = _normalize(summary)
+        if not norm or norm in seen:
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+            continue
+
+        seen.add(norm)
+        verification = _verification_points(content)
+        score = _candidate_score(summary, content)
+        ratio = len(summary) / max(1, len(content))
+
+        if score <= 0.1:
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+            continue
+
+        candidate_metadata = provenance.normalize_metadata(
+            {
+                **experience_metadata,
+                "compression_ratio": round(ratio, 3),
+                "task_id": task_id,
+                "source_event_id": source_id,
+                "experience_reference": f"experiences:{source_id}",
+                "derived_via": "distill",
+                "kind": "distilled_candidate",
+                "source_labels": [*(experience_metadata.get("source_labels") or []), *( [source_module] if source_module else [])],
+            },
+            source=source_module,
+        )
+        candidate_result = candidate.create_candidate(
+            source_event_id=source_id,
+            distilled_summary=summary,
+            verification_points=verification,
+            confidence_score=score,
+            metadata=candidate_metadata,
+        )
+
+        distilled.append({
+            "source_event_id": source_id,
+            "distilled_summary": summary,
+            "verification_points": verification,
+            "confidence_score": score,
+            "compression_ratio": round(ratio, 3),
+            "candidate_id": candidate_result.get("candidate_id"),
+            "duplicate": candidate_result.get("duplicate"),
+            "provenance": provenance.preview_from_metadata(candidate_metadata),
+        })
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_success", status="ok")
+
+    return distilled
+
+
+def distill_artifact(artifact: Dict[str, Any]) -> List[Dict[str, Any]]:
+    text = artifact.get("content_text", "")
+    if not isinstance(text, str) or not text.strip():
+        return []
+
+    text, _ = redaction.redact_text(text)
+    summary = _heuristic_summary(text)
+    summary, _ = redaction.redact_text(summary)
+    norm = _normalize(summary)
+    if not norm:
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+        return []
+
+    verification = _verification_points(text)
+    score = _candidate_score(summary, text)
+    ratio = len(summary) / max(1, len(text))
+
+    if score <= 0.1:
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+        return []
+
+    candidate_metadata = provenance.normalize_metadata(
+        {
+            "compression_ratio": round(ratio, 3),
+            "artifact_id": artifact.get("artifact_id"),
+            "derived_via": "artifact_distill",
+            "kind": "distilled_candidate",
+            "source_labels": ["artifact"],
+        }
+    )
+    candidate_result = candidate.create_candidate(
+        source_event_id=0,
+        distilled_summary=summary,
+        verification_points=verification,
+        confidence_score=score,
+        metadata=candidate_metadata,
+    )
+
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_success", status="ok")
+    return [{
+        "source_event_id": 0,
+        "distilled_summary": summary,
+        "verification_points": verification,
+        "confidence_score": score,
+        "compression_ratio": round(ratio, 3),
+        "candidate_id": candidate_result.get("candidate_id"),
+        "duplicate": candidate_result.get("duplicate"),
+        "provenance": provenance.preview_from_metadata(candidate_metadata),
+    }]

package/brain/runtime/memory/embedding_engine.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import hashlib
+from typing import List, Any
+
+from brain.runtime import config, state_store, model_router
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.providers import provider_execute
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+_MODEL_CACHE: dict[str, Any] = {}
+
+
+def _simple_embedding(text: str, dims: int = 8) -> List[float]:
+    digest = hashlib.sha256(text.encode("utf-8")).digest()
+    values = [digest[i] / 255.0 for i in range(dims)]
+    return values
+
+
+def _load_sentence_transformer(model_name: str) -> Any | None:
+    if model_name in _MODEL_CACHE:
+        return _MODEL_CACHE[model_name]
+    try:
+        from sentence_transformers import SentenceTransformer
+    except Exception:
+        return None
+    model = SentenceTransformer(model_name)
+    _MODEL_CACHE[model_name] = model
+    return model
+
+
+def _provider_embedding(text: str, model_name: str) -> tuple[List[float] | None, dict[str, str]]:
+    selection = model_router.get_provider_for_role("embedding")
+    if not selection.provider_id:
+        return None, {}
+    response = provider_execute.execute_embedding_call(selection, text)
+    embedding = response.get("embedding") if isinstance(response, dict) else None
+    meta = {
+        "provider_id": str(getattr(selection, "provider_id", "") or ""),
+        "model": str(model_name or getattr(selection, "model", "") or ""),
+    }
+    if isinstance(embedding, list):
+        return [float(x) for x in embedding], meta
+    return None, meta
+
+
+def generate_embedding(text: str) -> List[float] | None:
+    emit_event(LOGFILE, "brain_embedding_start", status="ok")
+    if not isinstance(text, str) or not text.strip():
+        emit_event(LOGFILE, "brain_embedding_failed", status="error", reason="empty_text")
+        return None
+    local_model = getattr(config, "BRAIN_EMBED_MODEL_LOCAL", "simple")
+    provider_model = getattr(config, "BRAIN_EMBED_MODEL_PROVIDER", "")
+
+    if provider_model:
+        try:
+            embedding, provider_meta = _provider_embedding(text, provider_model)
+        except Exception:
+            embedding, provider_meta = None, {}
+        if embedding:
+            emit_event(
+                LOGFILE,
+                "brain_embedding_complete",
+                status="ok",
+                provider="provider",
+                provider_id=provider_meta.get("provider_id", ""),
+                model=provider_meta.get("model", ""),
+            )
+            emit_event(
+                LOGFILE,
+                "brain_embedding_generated",
+                status="ok",
+                provider="provider",
+                dimensions=len(embedding),
+                provider_id=provider_meta.get("provider_id", ""),
+                model=provider_meta.get("model", ""),
+            )
+            return embedding
+
+    if local_model:
+        if local_model in {"simple", "hash"}:
+            embedding = _simple_embedding(text)
+            emit_event(LOGFILE, "brain_embedding_complete", status="ok", provider="local_simple")
+            emit_event(LOGFILE, "brain_embedding_generated", status="ok", provider="local_simple", dimensions=len(embedding))
+            return embedding
+        model = _load_sentence_transformer(local_model)
+        if model is not None:
+            embedding = model.encode([text])[0]
+            emit_event(LOGFILE, "brain_embedding_complete", status="ok", provider="local_model")
+            vector = [float(x) for x in embedding]
+            emit_event(LOGFILE, "brain_embedding_generated", status="ok", provider="local_model", dimensions=len(vector))
+            return vector
+    emit_event(LOGFILE, "brain_embedding_failed", status="error", reason="no_embedding")
+    return None

package/brain/runtime/memory/freshness.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.memory import store
+
+DEFAULT_STALE_DAYS = 30
+DEFAULT_CONFIDENCE_THRESHOLD = 0.6
+DEFAULT_LIMIT = 25
+
+
+def scan_freshness(
+    stale_days: int = DEFAULT_STALE_DAYS,
+    confidence_threshold: float = DEFAULT_CONFIDENCE_THRESHOLD,
+    limit: int = DEFAULT_LIMIT,
+) -> Dict[str, Any]:
+    emit_event(
+        state_store.reports_dir() / "brain_memory.log.jsonl",
+        "brain_memory_freshness_scan_start",
+        status="ok",
+        stale_days=stale_days,
+        confidence_threshold=confidence_threshold,
+    )
+    conn = store.connect()
+    stale_rows = conn.execute(
+        """
+        SELECT 'knowledge' AS memory_type, id, timestamp, confidence, content
+        FROM knowledge
+        WHERE timestamp <= datetime('now', ?)
+        ORDER BY timestamp ASC
+        LIMIT ?
+        """,
+        (f"-{max(1, stale_days)} days", limit),
+    ).fetchall()
+    low_conf_rows = conn.execute(
+        """
+        SELECT 'knowledge' AS memory_type, id, timestamp, confidence, content
+        FROM knowledge
+        WHERE confidence < ?
+        ORDER BY confidence ASC, timestamp ASC
+        LIMIT ?
+        """,
+        (confidence_threshold, limit),
+    ).fetchall()
+    conn.close()
+    advisories: List[Dict[str, Any]] = []
+    refresh_candidates: List[Dict[str, Any]] = []
+    now_ts = time.time()
+    for category, rows in (("stale", stale_rows), ("low_confidence", low_conf_rows)):
+        for row in rows:
+            age_seconds = 0.0
+            if row["timestamp"]:
+                try:
+                    age_seconds = max(0.0, now_ts - time.mktime(time.strptime(row["timestamp"], "%Y-%m-%d %H:%M:%S")))
+                except Exception:
+                    age_seconds = 0.0
+            confidence = float(row["confidence"] or 0.0)
+            freshness_score = max(0.0, 1.0 - min(age_seconds / (stale_days * 86400), 1.0)) * (0.5 + confidence / 2.0)
+            refresh_recommended = freshness_score < 0.4 or category == "stale"
+            entry = {
+                "category": category,
+                "memory_type": row["memory_type"],
+                "memory_id": row["id"],
+                "timestamp": row["timestamp"],
+                "confidence": confidence,
+                "summary": str(row["content"])[:120],
+                "freshness_score": round(freshness_score, 3),
+                "refresh_recommended": refresh_recommended,
+            }
+            advisories.append(entry)
+            refresh_candidates.append(entry)
+    emit_event(
+        state_store.reports_dir() / "brain_memory.log.jsonl",
+        "brain_memory_freshness_scan_complete",
+        status="ok",
+        advisory_count=len(advisories),
+        refresh_candidates=len(refresh_candidates),
+    )
+    return {
+        "ok": True,
+        "advisory_only": True,
+        "advisories": advisories,
+        "refresh_candidates": refresh_candidates,
+    }
+
+
+def freshness_weight(score: float) -> float:
+    return max(0.0, min(1.0, float(score)))

package/brain/runtime/memory/health.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Dict, Any
+
+from brain.runtime.memory import store, integrity
+
+
+EMBED_TABLES = ("knowledge", "runbooks", "lessons", "directives", "reflections", "tasks")
+
+
+def get_memory_health() -> Dict[str, Any]:
+    conn = store.connect()
+    counts: Dict[str, int] = {}
+    for table in ["experiences", "candidates", "promotions", "memory_index", "knowledge", "runbooks", "lessons", "directives", "reflections", "tasks", "vector_embeddings"]:
+        try:
+            counts[table] = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+        except Exception:
+            counts[table] = 0
+
+    vector_count = 0
+    try:
+        vector_count = conn.execute(
+            "SELECT COUNT(*) FROM vector_embeddings WHERE source_type IN ('knowledge','runbooks','lessons','directives','reflections','tasks')"
+        ).fetchone()[0]
+    except Exception:
+        vector_count = 0
+
+    total_embed_sources = sum(counts.get(table, 0) for table in EMBED_TABLES)
+    conn.close()
+    integrity_result = integrity.run_integrity_check()
+
+    coverage = 0.0
+    if total_embed_sources:
+        coverage = round(vector_count / total_embed_sources, 3)
+
+    return {
+        "counts": counts,
+        "vector_index_count": vector_count,
+        "vector_index_coverage": coverage,
+        "vector_index_integrity_status": integrity_result.get("ok"),
+        "integrity": integrity_result,
+    }

package/brain/runtime/memory/integrity.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from typing import Dict, Any, List
+
+from brain.runtime.instrumentation import emit_event
+from brain.runtime import state_store
+from brain.runtime.memory import store
+
+
+EMBED_TABLES = ("knowledge", "runbooks", "lessons", "directives", "reflections", "tasks")
+
+
+def run_integrity_check() -> Dict[str, Any]:
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_start", status="ok")
+    conn = store.connect()
+    issues: List[str] = []
+    repairable: List[str] = []
+    sqlite_ok = True
+
+    # required tables
+    required = {
+        "experiences",
+        "knowledge",
+        "reflections",
+        "tasks",
+        "directives",
+        "promotions",
+        "candidates",
+        "memory_index",
+        "vector_embeddings",
+        "runbooks",
+        "lessons",
+    }
+    tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
+    missing = required - tables
+    if missing:
+        issues.append(f"missing_tables:{','.join(sorted(missing))}")
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
+
+    # orphan candidates (source_event_id missing in experiences)
+    try:
+        orphan = conn.execute(
+            "SELECT COUNT(*) FROM candidates WHERE source_event_id NOT IN (SELECT id FROM experiences)",
+        ).fetchone()[0]
+        if orphan:
+            issues.append(f"orphan_candidates:{orphan}")
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
+    except Exception:
+        pass
+
+    # duplicate promotions
+    try:
+        dup = conn.execute(
+            "SELECT COUNT(*) FROM promotions GROUP BY source, content HAVING COUNT(*) > 1",
+        ).fetchone()
+        if dup:
+            issues.append("duplicate_promotions")
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
+    except Exception:
+        pass
+
+    # reinforcement references missing
+    try:
+        missing_ref = conn.execute(
+            "SELECT COUNT(*) FROM experiences WHERE memory_reference IS NULL OR memory_reference = ''",
+        ).fetchone()[0]
+        if missing_ref:
+            issues.append(f"missing_memory_reference:{missing_ref}")
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
+    except Exception:
+        pass
+
+    # vector index mismatch (knowledge/runbooks/lessons vs vector_embeddings)
+    missing_vectors = 0
+    orphan_vectors = 0
+    try:
+        for table in EMBED_TABLES:
+            missing_vectors += conn.execute(
+                f"SELECT COUNT(*) FROM {table} WHERE id NOT IN (SELECT CAST(source_id AS INTEGER) FROM vector_embeddings WHERE source_type=?)",
+                (table,),
+            ).fetchone()[0]
+    except Exception:
+        pass
+
+    try:
+        for table in EMBED_TABLES:
+            orphan_vectors += conn.execute(
+                "SELECT COUNT(*) FROM vector_embeddings WHERE source_type=? AND CAST(source_id AS INTEGER) NOT IN (SELECT id FROM %s)"
+                % table,
+                (table,),
+            ).fetchone()[0]
+    except Exception:
+        pass
+
+    if missing_vectors:
+        issues.append(f"vector_missing:{missing_vectors}")
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_vector_integrity_issue", status="warn")
+
+    if orphan_vectors:
+        issues.append(f"vector_orphan:{orphan_vectors}")
+        repairable.append("vector_orphan")
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_vector_integrity_issue", status="warn")
+
+    try:
+        quick_check = conn.execute("PRAGMA quick_check(1)").fetchone()
+        quick_value = str((quick_check or ["ok"])[0] or "ok")
+        if quick_value.lower() != "ok":
+            sqlite_ok = False
+            issues.append(f"sqlite_quick_check:{quick_value}")
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_issue", status="warn")
+    except Exception:
+        sqlite_ok = False
+
+    warning_type = ""
+    warning_summary = ""
+    for issue in issues:
+        if issue.startswith("vector_missing"):
+            warning_type = "vector_missing"
+            warning_summary = "Vector embeddings missing entries"
+            break
+        if issue.startswith("vector_orphan"):
+            warning_type = "vector_orphan"
+            warning_summary = "Vector embeddings have orphan entries"
+            break
+
+    conn.close()
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_complete", status="ok")
+    return {
+        "issues": issues,
+        "ok": len(issues) == 0 and sqlite_ok,
+        "warning_type": warning_type,
+        "warning_summary": warning_summary,
+        "repairable_issues": repairable,
+        "sqlite_ok": sqlite_ok,
+    }
+
+
+def repair_integrity() -> Dict[str, Any]:
+    repaired: List[str] = []
+
+    def _write() -> Dict[str, Any]:
+        conn = store.connect()
+        removed_orphans = 0
+        try:
+            tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
+            if "vector_embeddings" in tables:
+                for table in EMBED_TABLES:
+                    if table not in tables:
+                        continue
+                    removed_orphans += conn.execute(
+                        f"""
+                        DELETE FROM vector_embeddings
+                        WHERE source_type = ?
+                          AND NOT EXISTS (
+                            SELECT 1 FROM {table} source
+                            WHERE source.id = CAST(vector_embeddings.source_id AS INTEGER)
+                          )
+                        """,
+                        (table,),
+                    ).rowcount
+            conn.commit()
+            return {"removed_orphan_vectors": int(removed_orphans)}
+        finally:
+            conn.close()
+
+    result = store.submit_write(_write, timeout=30.0)
+    if int(result.get("removed_orphan_vectors") or 0) > 0:
+        repaired.append(f"vector_orphan:{int(result['removed_orphan_vectors'])}")
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_integrity_repair", status="ok", repaired="vector_orphan", count=int(result["removed_orphan_vectors"]))
+    return {"ok": True, "repaired": repaired, **result}

package/brain/runtime/memory/interaction_memory.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from typing import Dict, List
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.memory import person_memory
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+
+def _connect() -> sqlite3.Connection:
+    path = state_store.data_dir() / "interaction_memory.db"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS interaction_memory (
+            interaction_id INTEGER PRIMARY KEY AUTOINCREMENT,
+            person_id TEXT,
+            timestamp TEXT,
+            channel TEXT,
+            thread_id TEXT,
+            sentiment TEXT,
+            outcome TEXT
+        )
+        """
+    )
+    return conn
+
+
+def record_interaction(person_id: str, channel: str, thread_id: str, sentiment: str, outcome: str) -> None:
+    conn = _connect()
+    conn.execute(
+        "INSERT INTO interaction_memory (person_id, timestamp, channel, thread_id, sentiment, outcome) VALUES (?, ?, ?, ?, ?, ?)",
+        (person_id, time.strftime("%Y-%m-%d %H:%M:%S"), channel, thread_id, sentiment, outcome[:80]),
+    )
+    conn.commit()
+    conn.close()
+    person_memory.update_person(
+        person_id,
+        {"interaction_count": (person_memory.get_person(person_id) or {}).get("interaction_count", 0) + 1, "last_seen": time.strftime("%Y-%m-%d %H:%M:%S")},
+    )
+    emit_event(LOGFILE, "brain_person_interaction_recorded", status="ok", person_id=person_id)
+
+
+def get_recent_interactions(person_id: str, limit: int = 10) -> List[Dict[str, object]]:
+    conn = _connect()
+    rows = conn.execute(
+        "SELECT timestamp, channel, thread_id, sentiment, outcome FROM interaction_memory WHERE person_id=? ORDER BY timestamp DESC LIMIT ?",
+        (person_id, limit),
+    ).fetchall()
+    conn.close()
+    return [dict(row) for row in rows]