@simbimbo/memory-ocmemog 0.1.10 → 0.1.12

package/ocmemog/runtime/memory/distill.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any, Dict, List
+
+from ocmemog.runtime import inference, model_roles, state_store
+from ocmemog.runtime.instrumentation import emit_event
+from ocmemog.runtime.memory import candidate, provenance, store
+from ocmemog.runtime.security import redaction
+
+
+def _normalize(text: str) -> str:
+    return re.sub(r"\s+", " ", text.lower()).strip()
+
+
+def _heuristic_summary(text: str) -> str:
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    if not lines:
+        return ""
+    return lines[0][:240]
+
+
+def _should_skip_local_distill(text: str) -> bool:
+    cleaned = _normalize(text)
+    if not cleaned or len(cleaned) < 24:
+        return True
+    if cleaned in {"ok", "okay", "done", "fixed", "working", "success", "positive feedback"}:
+        return True
+    return False
+
+
+def _local_distill_summary(text: str) -> str:
+    if _should_skip_local_distill(text):
+        return ""
+    prompt = (
+        "Distill this experience into one concise operational summary. "
+        "Prefer concrete cause/effect, decision, or reusable takeaway. "
+        "Keep it under 220 characters. Return NONE if there is no meaningful takeaway.\n\n"
+        f"Experience:\n{text}\n\n"
+        "Summary:"
+    )
+    model = os.environ.get("OCMEMOG_PONDER_MODEL", "local-openai:qwen2.5-7b-instruct")
+    try:
+        result = inference.infer(prompt, provider_name=model)
+    except Exception:
+        return ""
+    if result.get("status") != "ok":
+        return ""
+    output = str(result.get("output", "")).strip()
+    output = re.sub(r"^(Summary|Sentence|Lesson):\s*", "", output, flags=re.IGNORECASE).strip()
+    if not output or output.upper().startswith("NONE"):
+        return ""
+    return output[:240]
+
+
+def _frontier_distill_summary(text: str) -> str:
+    try:
+        model = model_roles.get_model_for_role("memory")
+        result = inference.infer(
+            f"Distill this experience into a concise summary:\n\n{text}".strip(),
+            provider_name=model,
+        )
+        if result.get("status") == "ok":
+            return str(result.get("output", "")).strip()[:240]
+    except Exception:
+        return ""
+    return ""
+
+
+def _needs_frontier_refine(summary: str, source: str) -> bool:
+    if not summary:
+        return True
+    lowered = summary.lower().strip()
+    if lowered.startswith(("be ", "always ", "remember ", "good job", "be careful")):
+        return True
+    if len(summary) < 24:
+        return True
+    if len(summary) > len(source):
+        return True
+    if _normalize(summary) == _normalize(_heuristic_summary(source)):
+        return True
+    return False
+
+
+def _reject_distilled_summary(summary: str, source: str) -> bool:
+    lowered = _normalize(summary)
+    if not lowered:
+        return True
+    if lowered in {"ok", "okay", "done", "fixed", "working", "positive feedback", "success", "passed"}:
+        return True
+    if len(lowered) < 16:
+        return True
+    if lowered.startswith(("good job", "be proactive", "be thorough", "always check", "always remember")):
+        return True
+    if source and lowered == _normalize(source):
+        compact_source = re.sub(r"\s+", " ", str(source or "")).strip()
+        if "\n" in compact_source or len(compact_source) > 120:
+            return True
+    return False
+
+
+def _verification_points(text: str) -> List[str]:
+    points = []
+    if "verify" in text.lower():
+        points.append("Verify referenced assumptions")
+    if "risk" in text.lower():
+        points.append("Validate risk and mitigation")
+    if not points:
+        points.append("Confirm key facts before promotion")
+    return points[:3]
+
+
+def _candidate_score(summary: str, source: str) -> float:
+    if not source:
+        return 0.0
+    ratio = len(summary) / max(1, len(source))
+    score = 1.0 - min(1.0, ratio * 0.5)
+    return round(max(0.1, min(1.0, score)), 3)
+
+
+def _row_value(row: Any, key: str, fallback_index: int | None = None) -> Any:
+    if isinstance(row, dict):
+        return row.get(key)
+    try:
+        return row[key]
+    except Exception:
+        if fallback_index is None:
+            return None
+        try:
+            return row[fallback_index]
+        except Exception:
+            return None
+
+
+def distill_experiences(limit: int = 10) -> List[Dict[str, Any]]:
+    emit_event(state_store.report_log_path(), "brain_memory_distill_start", status="ok")
+    conn = store.connect()
+    rows = conn.execute(
+        "SELECT id, task_id, outcome, source_module, metadata_json FROM experiences ORDER BY id DESC LIMIT ?",
+        (limit,),
+    ).fetchall()
+    conn.close()
+
+    distilled: List[Dict[str, Any]] = []
+    seen = set()
+
+    for row in rows:
+        source_id = _row_value(row, "id", 0)
+        task_id = _row_value(row, "task_id", 1)
+        content = _row_value(row, "outcome", 2) or ""
+        source_module = _row_value(row, "source_module", 3)
+        raw_metadata = _row_value(row, "metadata_json", 4) or "{}"
+        try:
+            experience_metadata = json.loads(raw_metadata) if isinstance(raw_metadata, str) else dict(raw_metadata or {})
+        except Exception:
+            experience_metadata = {}
+        content, _ = redaction.redact_text(content)
+
+        heuristic_summary = _heuristic_summary(content)
+        summary = _local_distill_summary(content)
+        if _needs_frontier_refine(summary, content):
+            refined = _frontier_distill_summary(content)
+            if refined:
+                summary = refined
+
+        if not summary or len(summary) > len(content):
+            summary = heuristic_summary
+
+        summary, _ = redaction.redact_text(summary)
+        norm = _normalize(summary)
+        if _reject_distilled_summary(summary, content):
+            emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+            continue
+        if not norm or norm in seen:
+            emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+            continue
+
+        seen.add(norm)
+        verification = _verification_points(content)
+        score = _candidate_score(summary, content)
+        ratio = len(summary) / max(1, len(content))
+
+        if score <= 0.1:
+            emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+            continue
+
+        candidate_metadata = provenance.normalize_metadata(
+            {
+                **experience_metadata,
+                "compression_ratio": round(ratio, 3),
+                "task_id": task_id,
+                "source_event_id": source_id,
+                "experience_reference": f"experiences:{source_id}",
+                "derived_via": "distill",
+                "kind": "distilled_candidate",
+                "source_labels": [*(experience_metadata.get("source_labels") or []), *( [source_module] if source_module else [])],
+            },
+            source=source_module,
+        )
+        candidate_result = candidate.create_candidate(
+            source_event_id=source_id,
+            distilled_summary=summary,
+            verification_points=verification,
+            confidence_score=score,
+            metadata=candidate_metadata,
+        )
+
+        distilled.append({
+            "source_event_id": source_id,
+            "distilled_summary": summary,
+            "verification_points": verification,
+            "confidence_score": score,
+            "compression_ratio": round(ratio, 3),
+            "candidate_id": candidate_result.get("candidate_id"),
+            "duplicate": candidate_result.get("duplicate"),
+            "provenance": provenance.preview_from_metadata(candidate_metadata),
+        })
+        emit_event(state_store.report_log_path(), "brain_memory_distill_success", status="ok")
+
+    return distilled
+
+
+def distill_artifact(artifact: Dict[str, Any]) -> List[Dict[str, Any]]:
+    text = artifact.get("content_text", "")
+    if not isinstance(text, str) or not text.strip():
+        return []
+
+    text, _ = redaction.redact_text(text)
+    summary = _local_distill_summary(text)
+    if _needs_frontier_refine(summary, text):
+        refined = _frontier_distill_summary(text)
+        if refined:
+            summary = refined
+    if not summary or len(summary) > len(text):
+        summary = _heuristic_summary(text)
+    summary, _ = redaction.redact_text(summary)
+    if _reject_distilled_summary(summary, text):
+        emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+        return []
+    norm = _normalize(summary)
+    if not norm:
+        emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+        return []
+
+    verification = _verification_points(text)
+    score = _candidate_score(summary, text)
+    ratio = len(summary) / max(1, len(text))
+
+    if score <= 0.1:
+        emit_event(state_store.report_log_path(), "brain_memory_distill_rejected", status="ok")
+        return []
+
+    artifact_metadata = provenance.normalize_metadata(
+        {
+            "compression_ratio": round(ratio, 3),
+            "artifact_id": artifact.get("artifact_id"),
+            "derived_via": "artifact_distill",
+            "kind": "distilled_candidate",
+            "source_labels": ["artifact"],
+        }
+    )
+    candidate_result = candidate.create_candidate(
+        source_event_id=0,
+        distilled_summary=summary,
+        verification_points=verification,
+        confidence_score=score,
+        metadata=artifact_metadata,
+    )
+
+    emit_event(state_store.report_log_path(), "brain_memory_distill_success", status="ok")
+    return [{
+        "source_event_id": 0,
+        "distilled_summary": summary,
+        "verification_points": verification,
+        "confidence_score": score,
+        "compression_ratio": round(ratio, 3),
+        "candidate_id": candidate_result.get("candidate_id"),
+        "duplicate": candidate_result.get("duplicate"),
+        "provenance": provenance.preview_from_metadata(artifact_metadata),
+    }]

package/ocmemog/runtime/memory/embedding_engine.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import hashlib
+from typing import List, Any
+
+from ocmemog.runtime import config, model_router, state_store
+from ocmemog.runtime.instrumentation import emit_event
+from ocmemog.runtime.providers import provider_execute
+
+LOGFILE = state_store.report_log_path()
+_MODEL_CACHE: dict[str, Any] = {}
+
+
+def _local_embedding(text: str, local_model: str) -> List[float] | None:
+    if local_model in {"simple", "hash"}:
+        return _simple_embedding(text)
+    model = _load_sentence_transformer(local_model)
+    if model is None:
+        return None
+    return [float(x) for x in model.encode([text])[0]]
+
+
+def _simple_embedding(text: str, dims: int = 8) -> List[float]:
+    digest = hashlib.sha256(text.encode("utf-8")).digest()
+    values = [digest[i] / 255.0 for i in range(dims)]
+    return values
+
+
+def _load_sentence_transformer(model_name: str) -> Any | None:
+    if model_name in _MODEL_CACHE:
+        return _MODEL_CACHE[model_name]
+    try:
+        from sentence_transformers import SentenceTransformer
+    except Exception:
+        return None
+    model = SentenceTransformer(model_name)
+    _MODEL_CACHE[model_name] = model
+    return model
+
+
+def _provider_embedding(text: str, model_name: str) -> tuple[List[float] | None, dict[str, str]]:
+    selection = model_router.get_provider_for_role("embedding")
+    if not selection.provider_id:
+        return None, {}
+    response = provider_execute.execute_embedding_call(selection, text)
+    embedding = response.get("embedding") if isinstance(response, dict) else None
+    meta = {
+        "provider_id": str(getattr(selection, "provider_id", "") or ""),
+        "model": str(model_name or getattr(selection, "model", "") or ""),
+    }
+    if isinstance(embedding, list):
+        return [float(x) for x in embedding], meta
+    return None, meta
+
+
+def generate_embedding(
+    text: str,
+    *,
+    skip_provider: bool = False,
+) -> List[float] | None:
+    emit_event(LOGFILE, "brain_embedding_start", status="ok")
+    if not isinstance(text, str) or not text.strip():
+        emit_event(LOGFILE, "brain_embedding_failed", status="error", reason="empty_text")
+        return None
+    local_model = str(
+        getattr(config, "OCMEMOG_EMBED_MODEL_LOCAL", "")
+        or getattr(config, "BRAIN_EMBED_MODEL_LOCAL", getattr(config, "OCMEMOG_EMBED_LOCAL_MODEL", "simple"))
+        or ""
+    )
+    provider_model = (
+        getattr(config, "OCMEMOG_EMBED_PROVIDER", "")
+        or getattr(config, "OCMEMOG_EMBED_MODEL_PROVIDER", "")
+        or getattr(config, "BRAIN_EMBED_MODEL_PROVIDER", "")
+    )
+    embedding: List[float] | None = None
+    provider_meta: dict[str, str] = {}
+
+    if provider_model and not skip_provider:
+        try:
+            embedding, provider_meta = _provider_embedding(text, provider_model)
+        except TimeoutError as exc:
+            emit_event(
+                LOGFILE,
+                "brain_embedding_failed",
+                status="error",
+                reason="provider_timeout",
+                provider=provider_model,
+                model=provider_model,
+                error=str(exc),
+            )
+            if not local_model:
+                raise
+            embedding, provider_meta = None, {}
+        except Exception as exc:
+            emit_event(
+                LOGFILE,
+                "brain_embedding_failed",
+                status="error",
+                reason="provider_error",
+                provider=provider_model,
+                model=provider_model,
+                error=str(exc),
+            )
+            embedding, provider_meta = None, {}
+        if not embedding:
+            emit_event(
+                LOGFILE,
+                "brain_embedding_failed",
+                status="error",
+                reason="provider_no_embedding",
+                provider=provider_model,
+                model=provider_meta.get("model", ""),
+                fallback="local" if local_model else "disabled",
+            )
+        elif embedding:
+            emit_event(
+                LOGFILE,
+                "brain_embedding_complete",
+                status="ok",
+                provider="provider",
+                provider_id=provider_meta.get("provider_id", ""),
+                model=provider_meta.get("model", ""),
+            )
+            emit_event(
+                LOGFILE,
+                "brain_embedding_generated",
+                status="ok",
+                provider="provider",
+                dimensions=len(embedding),
+                provider_id=provider_meta.get("provider_id", ""),
+                model=provider_meta.get("model", ""),
+            )
+            return embedding
+    elif provider_model:
+        emit_event(
+            LOGFILE,
+            "brain_embedding_start",
+            status="ok",
+            provider=provider_model,
+            provider_skipped="true",
+        )
+
+    if local_model:
+        embedding = _local_embedding(text, local_model)
+        if embedding:
+            provider = "local_simple" if local_model in {"simple", "hash"} else "local_model"
+            emit_event(LOGFILE, "brain_embedding_complete", status="ok", provider=provider)
+            emit_event(LOGFILE, "brain_embedding_generated", status="ok", provider=provider, dimensions=len(embedding))
+            return embedding
+    emit_event(LOGFILE, "brain_embedding_failed", status="error", reason="no_embedding")
+    return None

package/ocmemog/runtime/memory/freshness.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List
+
+from ocmemog.runtime import state_store
+from ocmemog.runtime.instrumentation import emit_event
+from ocmemog.runtime.memory import store
+
+DEFAULT_STALE_DAYS = 30
+DEFAULT_CONFIDENCE_THRESHOLD = 0.6
+DEFAULT_LIMIT = 25
+
+
+_BAD_SUMMARY_PREFIXES = (
+    "promoted",
+    "candidate_promoted",
+    "no local memory summary available",
+    "summary",
+)
+
+
+def _summary_from_content(content: Any, limit: int = 120) -> str:
+    text = str(content or "").strip()
+    if not text:
+        return "(empty memory content)"
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    for line in lines:
+        lowered = line.lower()
+        if lowered in _BAD_SUMMARY_PREFIXES:
+            continue
+        if any(lowered.startswith(prefix + ":") for prefix in _BAD_SUMMARY_PREFIXES):
+            continue
+        return line[:limit]
+    return "(needs summary cleanup)"
+
+
+def scan_freshness(
+    stale_days: int = DEFAULT_STALE_DAYS,
+    confidence_threshold: float = DEFAULT_CONFIDENCE_THRESHOLD,
+    limit: int = DEFAULT_LIMIT,
+) -> Dict[str, Any]:
+    emit_event(
+        state_store.report_log_path(),
+        "brain_memory_freshness_scan_start",
+        status="ok",
+        stale_days=stale_days,
+        confidence_threshold=confidence_threshold,
+    )
+    conn = store.connect()
+    stale_rows = conn.execute(
+        """
+        SELECT 'knowledge' AS memory_type, id, timestamp, confidence, content
+        FROM knowledge
+        WHERE timestamp <= datetime('now', ?)
+        ORDER BY timestamp ASC
+        LIMIT ?
+        """,
+        (f"-{max(1, stale_days)} days", limit),
+    ).fetchall()
+    low_conf_rows = conn.execute(
+        """
+        SELECT 'knowledge' AS memory_type, id, timestamp, confidence, content
+        FROM knowledge
+        WHERE confidence < ?
+        ORDER BY confidence ASC, timestamp ASC
+        LIMIT ?
+        """,
+        (confidence_threshold, limit),
+    ).fetchall()
+    conn.close()
+    advisories: List[Dict[str, Any]] = []
+    refresh_candidates: List[Dict[str, Any]] = []
+    now_ts = time.time()
+    for category, rows in (("stale", stale_rows), ("low_confidence", low_conf_rows)):
+        for row in rows:
+            age_seconds = 0.0
+            if row["timestamp"]:
+                try:
+                    age_seconds = max(0.0, now_ts - time.mktime(time.strptime(row["timestamp"], "%Y-%m-%d %H:%M:%S")))
+                except Exception:
+                    age_seconds = 0.0
+            confidence = float(row["confidence"] or 0.0)
+            freshness_score = max(0.0, 1.0 - min(age_seconds / (stale_days * 86400), 1.0)) * (0.5 + confidence / 2.0)
+            refresh_recommended = freshness_score < 0.4 or category == "stale"
+            entry = {
+                "category": category,
+                "memory_type": row["memory_type"],
+                "memory_id": row["id"],
+                "timestamp": row["timestamp"],
+                "confidence": confidence,
+                "summary": _summary_from_content(row["content"]),
+                "freshness_score": round(freshness_score, 3),
+                "refresh_recommended": refresh_recommended,
+            }
+            advisories.append(entry)
+            refresh_candidates.append(entry)
+    emit_event(
+        state_store.report_log_path(),
+        "brain_memory_freshness_scan_complete",
+        status="ok",
+        advisory_count=len(advisories),
+        refresh_candidates=len(refresh_candidates),
+    )
+    return {
+        "ok": True,
+        "advisory_only": True,
+        "advisories": advisories,
+        "refresh_candidates": refresh_candidates,
+    }
+
+
+def freshness_weight(score: float) -> float:
+    return max(0.0, min(1.0, float(score)))

package/ocmemog/runtime/memory/health.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from ocmemog.runtime.memory import integrity, store
+
+EMBED_TABLES = tuple(store.MEMORY_TABLES)
+
+
+def get_memory_health() -> Dict[str, Any]:
+    conn = store.connect()
+    counts: Dict[str, int] = {}
+    for table in ["experiences", "candidates", "promotions", "memory_index", *store.MEMORY_TABLES, "vector_embeddings"]:
+        try:
+            counts[table] = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+        except Exception:
+            counts[table] = 0
+
+    vector_index_count = 0
+    vector_covered_sources = 0
+    try:
+        for table in EMBED_TABLES:
+            vector_index_count += conn.execute(
+                "SELECT COUNT(*) FROM vector_embeddings WHERE source_type=?",
+                (table,),
+            ).fetchone()[0]
+            vector_covered_sources += conn.execute(
+                f"""
+                SELECT COUNT(*) FROM {table} AS source
+                WHERE EXISTS (
+                    SELECT 1
+                    FROM vector_embeddings AS embeddings
+                    WHERE embeddings.source_type = ?
+                      AND CAST(embeddings.source_id AS TEXT) = CAST(source.id AS TEXT)
+                )
+                """,
+                (table,),
+            ).fetchone()[0]
+    except Exception:
+        vector_covered_sources = 0
+        vector_index_count = 0
+
+    total_embed_sources = sum(counts.get(table, 0) for table in EMBED_TABLES)
+    conn.close()
+    integrity_result = integrity.run_integrity_check()
+
+    coverage = 0.0
+    if total_embed_sources:
+        coverage = round(vector_covered_sources / total_embed_sources, 3)
+
+    return {
+        "counts": counts,
+        "vector_index_count": vector_index_count,
+        "vector_index_coverage": coverage,
+        "vector_index_integrity_status": integrity_result.get("ok"),
+        "integrity": integrity_result,
+    }