@simbimbo/memory-ocmemog 0.1.5 → 0.1.7

package/brain/runtime/memory/context_builder.py

@@ -1,87 +1,112 @@
 from __future__ import annotations
 
-from typing import Dict, Iterable, List
+import json
+import os
+import re
+from typing import Any, Dict, List
 
 from brain.runtime.instrumentation import emit_event
-from brain.runtime import state_store
+from brain.runtime import state_store, inference
 from brain.runtime.memory import retrieval
 
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
 
-def build_context(
-    prompt: str,
-    max_context_blocks: int = 5,
-    *,
-    memory_queries: Iterable[str] | None = None,
-    memory_priorities: Iterable[str] | None = None,
-    role_id: str | None = None,
-) -> Dict[str, List[str]]:
-    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_context_build_start", status="ok")
-    queries = [query for query in (memory_queries or ()) if isinstance(query, str) and query.strip()]
-    categories = [category for category in (memory_priorities or ()) if isinstance(category, str) and category.strip()]
-    role_priorities: List[str] = []
-    if role_id:
-        try:
-            from brain.runtime.roles import role_registry
-            role = role_registry.get_role(role_id)
-            role_priorities = list(role.memory_priority) if role else []
-        except Exception:
-            role_priorities = []
-    combined_priorities = [*categories, *role_priorities]
-    if queries:
-        mem = retrieval.retrieve_for_queries(queries, categories=combined_priorities or None)
-    else:
-        mem = retrieval.retrieve(prompt, categories=combined_priorities or None)
 
-    ranked_blocks: List[Dict[str, str | float]] = []
-    for item in mem.get("knowledge", []):
-        ranked_blocks.append(
-            {
-                "content": item.get("content"),
-                "source": "knowledge",
-                "score": float(item.get("score") or item.get("confidence") or 0.0),
-            }
-        )
-    for item in mem.get("tasks", []):
-        ranked_blocks.append(
-            {
-                "content": item.get("content"),
-                "source": "tasks",
-                "score": float(item.get("score") or item.get("confidence") or 0.0),
-            }
-        )
-    if role_priorities:
-        for item in ranked_blocks:
-            if item.get("source") in role_priorities:
-                item["score"] = float(item.get("score", 0.0)) + 0.2
-        emit_event(
-            state_store.reports_dir() / "brain_memory.log.jsonl",
-            "brain_role_context_weighted",
-            status="ok",
-            role_id=role_id,
-            priorities=len(role_priorities),
-        )
-    ranked_blocks.sort(key=lambda item: item.get("score", 0.0), reverse=True)
-    if len(ranked_blocks) > max_context_blocks:
-        ranked_blocks = ranked_blocks[:max_context_blocks]
-        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_context_trim", status="ok")
+def _heuristic_queries(prompt: str, limit: int = 3) -> List[str]:
+    cleaned = re.sub(r"\s+", " ", prompt or "").strip()
+    parts = re.split(r",| and | then | also ", cleaned)
+    queries = []
+    for part in parts:
+        q = part.strip(" .")
+        if len(q) >= 8 and q.lower() not in {cleaned.lower()}:
+            queries.append(q)
+    if cleaned and cleaned not in queries:
+        queries.insert(0, cleaned)
+    deduped: List[str] = []
+    seen = set()
+    for q in queries:
+        key = q.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(q)
+        if len(deduped) >= limit:
+            break
+    return deduped
 
-    context_blocks = [item["content"] for item in ranked_blocks if item.get("content")]
-    context_scores = [item.get("score", 0.0) for item in ranked_blocks]
-    synthesis = mem.get("synthesis", []) if isinstance(mem, dict) else []
-    for item in synthesis[:2]:
-        summary = item.get("summary") if isinstance(item, dict) else None
-        if summary:
-            context_blocks.append(str(summary))
 
-    context = {
-        "context_blocks": context_blocks,
-        "context_scores": context_scores,
-        "ranked_blocks": ranked_blocks,
-        "knowledge": mem.get("knowledge", []),
-        "tasks": mem.get("tasks", []),
-        "directives": [item["content"] if isinstance(item, dict) else item for item in mem.get("directives", [])],
-        "reflections": [item["content"] if isinstance(item, dict) else item for item in mem.get("reflections", [])],
-        "used_queries": queries,
+def _should_skip_query_grooming(prompt: str) -> bool:
+    cleaned = re.sub(r"\s+", " ", prompt or "").strip()
+    if not cleaned:
+        return True
+    if len(cleaned) <= 32 and ',' not in cleaned and ' and ' not in cleaned.lower():
+        return True
+    words = cleaned.split()
+    if 1 <= len(words) <= 5 and all(len(w) >= 3 for w in words):
+        return True
+    return False
+
+
+def _groom_queries(prompt: str, limit: int = 3) -> List[str]:
+    cleaned = re.sub(r"\s+", " ", prompt or "").strip()
+    if not cleaned:
+        return []
+    if _should_skip_query_grooming(cleaned):
+        return _heuristic_queries(cleaned, limit=limit)
+    model = os.environ.get("OCMEMOG_PONDER_MODEL", "local-openai:qwen2.5-7b-instruct")
+    ask = (
+        "Rewrite this raw memory request into up to 3 short search queries. "
+        "Return strict JSON as {\"queries\":[\"...\"]}. "
+        "Prefer compact entity/topic phrases, not full sentences.\n\n"
+        f"Request: {cleaned}\n"
+    )
+    try:
+        result = inference.infer(ask, provider_name=model)
+    except Exception:
+        return _heuristic_queries(cleaned, limit=limit)
+    if result.get("status") != "ok":
+        return _heuristic_queries(cleaned, limit=limit)
+    output = str(result.get("output") or "").strip()
+    try:
+        payload = json.loads(output)
+        raw_queries = payload.get("queries") or []
+        queries = [str(q).strip() for q in raw_queries if str(q).strip()]
+    except Exception:
+        queries = []
+    cleaned_queries: List[str] = []
+    seen = set()
+    for q in queries:
+        key = q.lower()
+        if len(q) < 4 or key in seen:
+            continue
+        seen.add(key)
+        cleaned_queries.append(q)
+        if len(cleaned_queries) >= limit:
+            break
+    return cleaned_queries or _heuristic_queries(cleaned, limit=limit)
+
+
+def build_context(prompt: str, memory_queries: List[str] | None = None, limit: int = 5) -> Dict[str, Any]:
+    emit_event(LOGFILE, "brain_memory_context_build_start", status="ok")
+    queries = memory_queries or _groom_queries(prompt, limit=3)
+    memories: List[Dict[str, Any]] = []
+    seen: set[str] = set()
+    for query in queries:
+        for item in retrieval.retrieve_memories(query, limit=limit):
+            ref = str(item.get("reference") or item.get("id") or "")
+            if ref and ref in seen:
+                continue
+            if ref:
+                seen.add(ref)
+            memories.append(item)
+            if len(memories) >= limit:
+                break
+        if len(memories) >= limit:
+            break
+
+    emit_event(LOGFILE, "brain_memory_context_build_complete", status="ok", item_count=len(memories), query_count=len(queries))
+    return {
+        "prompt": prompt,
+        "queries": queries,
+        "memories": memories,
     }
-    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_context_build_complete", status="ok")
-    return context

package/brain/runtime/memory/distill.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import os
 import re
 from typing import Dict, Any, List
 
@@ -23,6 +24,83 @@ def _heuristic_summary(text: str) -> str:
     return lines[0][:240]
 
 
+def _should_skip_local_distill(text: str) -> bool:
+    cleaned = _normalize(text)
+    if not cleaned or len(cleaned) < 24:
+        return True
+    if cleaned in {"ok", "okay", "done", "fixed", "working", "success", "positive feedback"}:
+        return True
+    return False
+
+
+def _local_distill_summary(text: str) -> str:
+    if _should_skip_local_distill(text):
+        return ""
+    prompt = (
+        "Distill this experience into one concise operational summary. "
+        "Prefer concrete cause/effect, decision, or reusable takeaway. "
+        "Keep it under 220 characters. Return NONE if there is no meaningful takeaway.\n\n"
+        f"Experience:\n{text}\n\n"
+        "Summary:"
+    )
+    model = os.environ.get("OCMEMOG_PONDER_MODEL", "local-openai:qwen2.5-7b-instruct")
+    try:
+        result = inference.infer(prompt, provider_name=model)
+    except Exception:
+        return ""
+    if result.get("status") != "ok":
+        return ""
+    output = str(result.get("output", "")).strip()
+    output = re.sub(r"^(Summary|Sentence|Lesson):\s*", "", output, flags=re.IGNORECASE).strip()
+    if not output or output.upper().startswith("NONE"):
+        return ""
+    return output[:240]
+
+
+def _frontier_distill_summary(text: str) -> str:
+    try:
+        model = model_roles.get_model_for_role("memory")
+        result = inference.infer(
+            f"Distill this experience into a concise summary:\n\n{text}".strip(),
+            provider_name=model,
+        )
+        if result.get("status") == "ok":
+            return str(result.get("output", "")).strip()[:240]
+    except Exception:
+        return ""
+    return ""
+
+
+def _needs_frontier_refine(summary: str, source: str) -> bool:
+    if not summary:
+        return True
+    lowered = summary.lower().strip()
+    if lowered.startswith(("be ", "always ", "remember ", "good job", "be careful")):
+        return True
+    if len(summary) < 24:
+        return True
+    if len(summary) > len(source):
+        return True
+    if _normalize(summary) == _normalize(_heuristic_summary(source)):
+        return True
+    return False
+
+
+def _reject_distilled_summary(summary: str, source: str) -> bool:
+    lowered = _normalize(summary)
+    if not lowered:
+        return True
+    if lowered in {"ok", "okay", "done", "fixed", "working", "positive feedback", "success", "passed"}:
+        return True
+    if len(lowered) < 16:
+        return True
+    if lowered.startswith(("good job", "be proactive", "be thorough", "always check", "always remember")):
+        return True
+    if source and lowered == _normalize(source):
+        return True
+    return False
+
+
 def _verification_points(text: str) -> List[str]:
     points = []
     if "verify" in text.lower():
@@ -81,23 +159,21 @@ def distill_experiences(limit: int = 10) -> List[Dict[str, Any]]:
             experience_metadata = {}
         content, _ = redaction.redact_text(content)
 
-        summary = ""
-        try:
-            model = model_roles.get_model_for_role("memory")
-            result = inference.infer(
-                f"Distill this experience into a concise summary:\n\n{content}".strip(),
-                provider_name=model,
-            )
-            if result.get("status") == "ok":
-                summary = str(result.get("output", "")).strip()
-        except Exception:
-            summary = ""
+        heuristic_summary = _heuristic_summary(content)
+        summary = _local_distill_summary(content)
+        if _needs_frontier_refine(summary, content):
+            refined = _frontier_distill_summary(content)
+            if refined:
+                summary = refined
 
         if not summary or len(summary) > len(content):
-            summary = _heuristic_summary(content)
+            summary = heuristic_summary
 
         summary, _ = redaction.redact_text(summary)
         norm = _normalize(summary)
+        if _reject_distilled_summary(summary, content):
+            emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+            continue
         if not norm or norm in seen:
             emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
             continue
@@ -153,8 +229,17 @@ def distill_artifact(artifact: Dict[str, Any]) -> List[Dict[str, Any]]:
         return []
 
     text, _ = redaction.redact_text(text)
-    summary = _heuristic_summary(text)
+    summary = _local_distill_summary(text)
+    if _needs_frontier_refine(summary, text):
+        refined = _frontier_distill_summary(text)
+        if refined:
+            summary = refined
+    if not summary or len(summary) > len(text):
+        summary = _heuristic_summary(text)
     summary, _ = redaction.redact_text(summary)
+    if _reject_distilled_summary(summary, text):
+        emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
+        return []
     norm = _normalize(summary)
     if not norm:
         emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_rejected", status="ok")
@@ -196,3 +281,61 @@ def distill_artifact(artifact: Dict[str, Any]) -> List[Dict[str, Any]]:
         "duplicate": candidate_result.get("duplicate"),
         "provenance": provenance.preview_from_metadata(candidate_metadata),
     }]
+
+    candidate_metadata = provenance.normalize_metadata(
+        {
+            "compression_ratio": round(ratio, 3),
+            "artifact_id": artifact.get("artifact_id"),
+            "derived_via": "artifact_distill",
+            "kind": "distilled_candidate",
+            "source_labels": ["artifact"],
+        }
+    )
+    candidate_result = candidate.create_candidate(
+        source_event_id=0,
+        distilled_summary=summary,
+        verification_points=verification,
+        confidence_score=score,
+        metadata=candidate_metadata,
+    )
+
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_success", status="ok")
+    return [{
+        "source_event_id": 0,
+        "distilled_summary": summary,
+        "verification_points": verification,
+        "confidence_score": score,
+        "compression_ratio": round(ratio, 3),
+        "candidate_id": candidate_result.get("candidate_id"),
+        "duplicate": candidate_result.get("duplicate"),
+        "provenance": provenance.preview_from_metadata(candidate_metadata),
+    }]
+
+    candidate_metadata = provenance.normalize_metadata(
+        {
+            "compression_ratio": round(ratio, 3),
+            "artifact_id": artifact.get("artifact_id"),
+            "derived_via": "artifact_distill",
+            "kind": "distilled_candidate",
+            "source_labels": ["artifact"],
+        }
+    )
+    candidate_result = candidate.create_candidate(
+        source_event_id=0,
+        distilled_summary=summary,
+        verification_points=verification,
+        confidence_score=score,
+        metadata=candidate_metadata,
+    )
+
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_distill_success", status="ok")
+    return [{
+        "source_event_id": 0,
+        "distilled_summary": summary,
+        "verification_points": verification,
+        "confidence_score": score,
+        "compression_ratio": round(ratio, 3),
+        "candidate_id": candidate_result.get("candidate_id"),
+        "duplicate": candidate_result.get("duplicate"),
+        "provenance": provenance.preview_from_metadata(candidate_metadata),
+    }]

package/brain/runtime/memory/pondering_engine.py

@@ -297,6 +297,8 @@ def _refine_unresolved_summary(summary: str, reference: str = "") -> str:
     raw = _heuristic_summary(summary, limit=500)
     if not _needs_unresolved_refine(raw):
         return _heuristic_summary(raw)
+    if raw and not raw.startswith(("#", "*", "1)", "2)", "TODO:")) and len(raw.split()) >= 5:
+        return _heuristic_summary(raw, limit=180)
     prompt = (
         "Rewrite this unresolved item as one concise actionable unresolved summary. "
         "Keep it under 180 characters. Focus on the decision, blocker, or next action. "

package/brain/runtime/memory/promote.py

@@ -140,6 +140,12 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
         emit_event(LOGFILE, "brain_memory_reinforcement_created", status="ok")
         if memory_id:
             vector_index.insert_memory(memory_id, candidate.get("distilled_summary", ""), confidence)
+            try:
+                from brain.runtime.memory import api as memory_api
+
+                memory_api._auto_attach_governance_candidates(promoted_reference)
+            except Exception as exc:
+                emit_event(LOGFILE, "brain_memory_promotion_governance_failed", status="error", error=str(exc), reference=promoted_reference)
 
     return {"decision": decision, "confidence": confidence, "promotion_id": promotion_id, "destination": destination}
 

package/brain/runtime/memory/provenance.py

@@ -106,6 +106,16 @@ def normalize_metadata(metadata: Optional[Dict[str, Any]], *, source: Optional[s
         "derived_from_promotion_id",
         "derived_via",
         "kind",
+        "memory_status",
+        "superseded_by",
+        "supersedes",
+        "duplicate_of",
+        "duplicate_candidates",
+        "contradicts",
+        "contradiction_candidates",
+        "contradiction_status",
+        "canonical_reference",
+        "supersession_recommendation",
     ):
         if raw.get(key) is not None and provenance.get(key) is None:
             provenance[key] = raw.get(key)
@@ -177,6 +187,20 @@ def apply_links(reference: str, metadata: Optional[Dict[str, Any]]) -> None:
         _link_once(reference, "candidate", f"candidate:{provenance['derived_from_candidate_id']}")
     if provenance.get("derived_from_promotion_id"):
         _link_once(reference, "promotion", f"promotions:{provenance['derived_from_promotion_id']}")
+    if provenance.get("superseded_by"):
+        _link_once(reference, "superseded_by", str(provenance.get("superseded_by")))
+    if provenance.get("supersedes"):
+        _link_once(reference, "supersedes", str(provenance.get("supersedes")))
+    if provenance.get("duplicate_of"):
+        _link_once(reference, "duplicate_of", str(provenance.get("duplicate_of")))
+    for candidate in provenance.get("duplicate_candidates") or []:
+        _link_once(reference, "duplicate_candidate", str(candidate))
+    for target in provenance.get("contradicts") or []:
+        _link_once(reference, "contradicts", str(target))
+    for target in provenance.get("contradiction_candidates") or []:
+        _link_once(reference, "contradiction_candidate", str(target))
+    if provenance.get("canonical_reference"):
+        _link_once(reference, "canonical", str(provenance.get("canonical_reference")))
 
 
 def update_memory_metadata(reference: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]:
@@ -201,6 +225,34 @@ def update_memory_metadata(reference: str, updates: Dict[str, Any]) -> Optional[
     return merged
 
 
+def force_update_memory_metadata(reference: str, updates: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    table, sep, raw_id = reference.partition(":")
+    if not sep or table not in _MEMORY_TABLES or not raw_id.isdigit():
+        return None
+    conn = store.connect()
+    try:
+        row = conn.execute(f"SELECT metadata_json FROM {table} WHERE id = ?", (int(raw_id),)).fetchone()
+        if not row:
+            return None
+        current = _load_json(row["metadata_json"], {})
+        provenance_meta = current.get("provenance") if isinstance(current.get("provenance"), dict) else {}
+        for key, value in updates.items():
+            if value is None or value == "":
+                provenance_meta.pop(key, None)
+            else:
+                provenance_meta[key] = value
+        current["provenance"] = provenance_meta
+        conn.execute(
+            f"UPDATE {table} SET metadata_json = ? WHERE id = ?",
+            (json.dumps(current, ensure_ascii=False), int(raw_id)),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+    apply_links(reference, current)
+    return current
+
+
 def fetch_reference(reference: str) -> Optional[Dict[str, Any]]:
     prefix, sep, raw_id = reference.partition(":")
     if not sep or not prefix: