@simbimbo/memory-ocmemog 0.1.16 → 0.1.17

package/ocmemog/runtime/memory/api.py

@@ -18,20 +18,20 @@ _REVIEW_KIND_METADATA: Dict[str, Dict[str, str]] = {
     "duplicate_candidate": {
         "relationship": "duplicate_of",
         "label": "Duplicate candidate",
-        "approve_label": "Approve duplicate merge",
-        "reject_label": "Reject duplicate merge",
+        "apply_label": "Apply duplicate merge",
+        "dismiss_label": "Dismiss duplicate merge",
     },
     "contradiction_candidate": {
         "relationship": "contradicts",
         "label": "Contradiction candidate",
-        "approve_label": "Mark as contradiction",
-        "reject_label": "Dismiss contradiction",
+        "apply_label": "Apply contradiction",
+        "dismiss_label": "Dismiss contradiction",
     },
     "supersession_recommendation": {
         "relationship": "supersedes",
         "label": "Supersession recommendation",
-        "approve_label": "Approve supersession",
-        "reject_label": "Dismiss supersession",
+        "apply_label": "Apply supersession",
+        "dismiss_label": "Dismiss supersession",
     },
 }
 
@@ -1021,20 +1021,40 @@ def _plain_english_supersession_summary(
     return summary
 
 
+def _review_explanation(kind: str, *, signal: float, reason: str, source: Dict[str, Any], target: Dict[str, Any]) -> Dict[str, Any]:
+    source_status = str(source.get("memory_status") or "active")
+    target_status = str(target.get("memory_status") or "active")
+    reason_text = str(reason or "").strip() or "no explicit rationale captured"
+    if kind == "duplicate_candidate":
+        short = f"Possible duplicate merge with signal {signal:.2f}."
+    elif kind == "contradiction_candidate":
+        short = f"Possible contradiction with signal {signal:.2f}."
+    elif kind == "supersession_recommendation":
+        short = f"Possible supersession with signal {signal:.2f}."
+    else:
+        short = f"Governance review item with signal {signal:.2f}."
+    return {
+        "short": short,
+        "reason": reason_text,
+        "source_status": source_status,
+        "target_status": target_status,
+    }
+
+
 def _review_actions(kind: str, relationship: str) -> List[Dict[str, Any]]:
     meta = _REVIEW_KIND_METADATA.get(kind, {})
     return [
         {
-            "decision": "approve",
+            "decision": "apply",
             "approved": True,
             "relationship": relationship,
-            "label": meta.get("approve_label") or "Approve",
+            "label": meta.get("apply_label") or "Apply",
         },
         {
-            "decision": "reject",
+            "decision": "dismiss",
             "approved": False,
             "relationship": relationship,
-            "label": meta.get("reject_label") or "Reject",
+            "label": meta.get("dismiss_label") or "Dismiss",
         },
     ]
 
@@ -1047,6 +1067,18 @@ def _relationship_for_review(kind: str | None = None, relationship: str | None =
     return _REVIEW_KIND_METADATA.get(kind_key, {}).get("relationship", "")
 
 
+def _review_priority_label(priority: int) -> str:
+    if priority >= 90:
+        return "critical"
+    if priority >= 70:
+        return "high"
+    if priority >= 40:
+        return "medium"
+    if priority > 0:
+        return "low"
+    return "none"
+
+
 def list_governance_review_items(
     *,
     categories: Optional[List[str]] = None,
@@ -1070,19 +1102,24 @@ def list_governance_review_items(
             plain_english = str(item.get("plain_english") or "").strip()
             if plain_english:
                 summary = plain_english
+        signal = float(item.get("signal") or 0.0)
+        reason = str(item.get("reason") or "")
+        priority = int(item.get("priority") or 0)
         review_items.append({
             "review_id": f"{kind}:{reference}->{target_reference}",
             "kind": kind,
             "kind_label": _REVIEW_KIND_METADATA.get(kind, {}).get("label") or kind.replace("_", " "),
             "relationship": relationship,
-            "priority": int(item.get("priority") or 0),
+            "priority": priority,
+            "priority_label": _review_priority_label(priority),
             "timestamp": item.get("timestamp"),
             "bucket": item.get("bucket"),
-            "signal": float(item.get("signal") or 0.0),
-            "reason": item.get("reason"),
+            "signal": signal,
+            "reason": reason,
             "reference": reference,
             "target_reference": target_reference,
             "summary": summary,
+            "explanation": _review_explanation(kind, signal=signal, reason=reason, source=source, target=target),
             "actions": _review_actions(kind, relationship),
             "source": source,
             "target": target,
@@ -1234,6 +1271,27 @@ def rollback_governance_decision(
     return None
 
 
+def _governance_queue_explanation(kind: str, *, target_reference: str | None, priority: int, reason: str | None = None, signal: float | None = None) -> Dict[str, Any]:
+    reason_text = str(reason or "").strip() or "no explicit rationale captured"
+    if kind == "duplicate_candidate":
+        short = f"Possible duplicate relationship queued at priority {priority}."
+    elif kind == "contradiction_candidate":
+        short = f"Possible contradiction queued at priority {priority}."
+    elif kind == "supersession_recommendation":
+        short = f"Possible supersession queued at priority {priority}."
+    else:
+        short = f"Governance queue item queued at priority {priority}."
+    payload = {
+        "short": short,
+        "reason": reason_text,
+        "target_reference": target_reference,
+        "priority": priority,
+    }
+    if signal is not None:
+        payload["signal"] = float(signal)
+    return payload
+
+
 def governance_queue(*, categories: Optional[List[str]] = None, limit: int = 100, scan_limit: int = 3000) -> List[Dict[str, Any]]:
     allowed = set(store.MEMORY_TABLES)
     tables = [table for table in (categories or list(allowed)) if table in allowed]
@@ -1260,36 +1318,62 @@ def governance_queue(*, categories: Optional[List[str]] = None, limit: int = 100
                 supersession_recommendation = prov.get("supersession_recommendation") or {}
 
                 for target in duplicate_candidates:
+                    priority = 40
                     items.append({
                         "reference": reference,
                         "target_reference": target,
                         "kind": "duplicate_candidate",
-                        "priority": 40,
+                        "priority": priority,
+                        "priority_label": _review_priority_label(priority),
                         "timestamp": timestamp,
                         "bucket": table,
                         "content": content,
+                        "explanation": _governance_queue_explanation(
+                            "duplicate_candidate",
+                            target_reference=target,
+                            priority=priority,
+                        ),
                     })
                 for target in contradiction_candidates:
+                    priority = 70
                     items.append({
                         "reference": reference,
                         "target_reference": target,
                         "kind": "contradiction_candidate",
-                        "priority": 70,
+                        "priority": priority,
+                        "priority_label": _review_priority_label(priority),
                         "timestamp": timestamp,
                         "bucket": table,
                         "content": content,
+                        "explanation": _governance_queue_explanation(
+                            "contradiction_candidate",
+                            target_reference=target,
+                            priority=priority,
+                        ),
                     })
                 if isinstance(supersession_recommendation, dict) and supersession_recommendation.get("recommended"):
+                    priority = 90
+                    signal = float(supersession_recommendation.get("signal") or 0.0)
+                    reason = supersession_recommendation.get("reason")
+                    target_reference = supersession_recommendation.get("target_reference")
                     items.append({
                         "reference": reference,
-                        "target_reference": supersession_recommendation.get("target_reference"),
+                        "target_reference": target_reference,
                         "kind": "supersession_recommendation",
-                        "priority": 90,
+                        "priority": priority,
+                        "priority_label": _review_priority_label(priority),
                         "timestamp": timestamp,
                         "bucket": table,
-                        "signal": float(supersession_recommendation.get("signal") or 0.0),
-                        "reason": supersession_recommendation.get("reason"),
+                        "signal": signal,
+                        "reason": reason,
                         "content": content,
+                        "explanation": _governance_queue_explanation(
+                            "supersession_recommendation",
+                            target_reference=target_reference,
+                            priority=priority,
+                            reason=reason,
+                            signal=signal,
+                        ),
                     })
         items.sort(key=lambda item: (int(item.get("priority") or 0), str(item.get("timestamp") or "")), reverse=True)
         return items[:limit]

package/ocmemog/runtime/memory/embedding_engine.py

@@ -9,6 +9,7 @@ from ocmemog.runtime.providers import provider_execute
 
 LOGFILE = state_store.report_log_path()
 _MODEL_CACHE: dict[str, Any] = {}
+_LAST_EMBEDDING_DIAGNOSTICS: dict[str, Any] = {}
 
 
 def _local_embedding(text: str, local_model: str) -> List[float] | None:
@@ -53,11 +54,16 @@ def _provider_embedding(text: str, model_name: str) -> tuple[List[float] | None,
     return None, meta
 
 
+def get_last_embedding_diagnostics() -> dict[str, Any]:
+    return dict(_LAST_EMBEDDING_DIAGNOSTICS)
+
+
 def generate_embedding(
     text: str,
     *,
     skip_provider: bool = False,
 ) -> List[float] | None:
+    global _LAST_EMBEDDING_DIAGNOSTICS
     emit_event(LOGFILE, "brain_embedding_start", status="ok")
     if not isinstance(text, str) or not text.strip():
         emit_event(LOGFILE, "brain_embedding_failed", status="error", reason="empty_text")
@@ -72,10 +78,22 @@ def generate_embedding(
         or getattr(config, "OCMEMOG_EMBED_MODEL_PROVIDER", "")
         or getattr(config, "BRAIN_EMBED_MODEL_PROVIDER", "")
     )
+    _LAST_EMBEDDING_DIAGNOSTICS = {
+        "provider_configured": bool(provider_model),
+        "provider_attempted": False,
+        "provider_skipped": bool(provider_model and skip_provider),
+        "provider_succeeded": False,
+        "local_model": local_model or "simple",
+        "local_used": False,
+        "local_mode": "local_simple" if (local_model or "simple") in {"simple", "hash"} else "local_model",
+        "path_used": None,
+        "embedding_generated": False,
+    }
     embedding: List[float] | None = None
     provider_meta: dict[str, str] = {}
 
     if provider_model and not skip_provider:
+        _LAST_EMBEDDING_DIAGNOSTICS["provider_attempted"] = True
         try:
             embedding, provider_meta = _provider_embedding(text, provider_model)
         except TimeoutError as exc:
@@ -113,6 +131,9 @@ def generate_embedding(
                 fallback="local" if local_model else "disabled",
             )
         elif embedding:
+            _LAST_EMBEDDING_DIAGNOSTICS["provider_succeeded"] = True
+            _LAST_EMBEDDING_DIAGNOSTICS["path_used"] = "provider"
+            _LAST_EMBEDDING_DIAGNOSTICS["embedding_generated"] = True
             emit_event(
                 LOGFILE,
                 "brain_embedding_complete",
@@ -144,6 +165,9 @@ def generate_embedding(
         embedding = _local_embedding(text, local_model)
         if embedding:
             provider = "local_simple" if local_model in {"simple", "hash"} else "local_model"
+            _LAST_EMBEDDING_DIAGNOSTICS["local_used"] = True
+            _LAST_EMBEDDING_DIAGNOSTICS["path_used"] = provider
+            _LAST_EMBEDDING_DIAGNOSTICS["embedding_generated"] = True
             emit_event(LOGFILE, "brain_embedding_complete", status="ok", provider=provider)
             emit_event(LOGFILE, "brain_embedding_generated", status="ok", provider=provider, dimensions=len(embedding))
             return embedding

package/ocmemog/runtime/memory/promote.py

@@ -60,13 +60,130 @@ def _destination_table(summary: str) -> str:
     return "knowledge"
 
 
+def _normalized_text(text: str) -> str:
+    return " ".join((text or "").strip().lower().split())
+
+
+def _is_redundant_generic_candidate(summary_text: str) -> bool:
+    normalized = _normalized_text(summary_text)
+    if not normalized:
+        return False
+    conn = store.connect()
+    try:
+        rows = conn.execute(
+            "SELECT content FROM knowledge ORDER BY id DESC LIMIT 200"
+        ).fetchall()
+    finally:
+        conn.close()
+    for row in rows:
+        existing = _normalized_text(row[0] if row else "")
+        if existing and existing == normalized:
+            return True
+    return False
+
+
+def _should_reject_as_cruft(*, confidence: float, threshold: float, destination: str, summary_text: str) -> bool:
+    if destination != "knowledge" or confidence >= threshold:
+        return False
+    return bool(_normalized_text(summary_text))
+
+
+def _is_ambiguous_specific_candidate(*, confidence: float, threshold: float, destination: str) -> bool:
+    if destination == "knowledge":
+        return False
+    margin = confidence - threshold
+    return margin < 0 and margin >= -0.2
+
+
+def _quality_summary(*, decision: str, confidence: float, threshold: float, destination: str, redundant_generic: bool = False, ambiguous_specific: bool = False) -> Dict[str, Any]:
+    margin = round(confidence - threshold, 3)
+    if decision == "promote":
+        quality = "high" if margin >= 0.2 else "medium"
+        keep_recommendation = "keep"
+        noise_risk = "low"
+    else:
+        if destination == "knowledge":
+            quality = "low"
+            keep_recommendation = "drop"
+            noise_risk = "high"
+        elif ambiguous_specific:
+            quality = "medium"
+            keep_recommendation = "review"
+            noise_risk = "medium"
+        else:
+            quality = "medium"
+            keep_recommendation = "review"
+            noise_risk = "medium"
+    return {
+        "quality": quality,
+        "keep_recommendation": keep_recommendation,
+        "noise_risk": noise_risk,
+        "margin": margin,
+        "destination_specificity": "generic" if destination == "knowledge" else "specific",
+        "redundant_generic": bool(redundant_generic),
+        "ambiguous_specific": bool(ambiguous_specific),
+    }
+
+
+def _verification_summary(*, decision: str, confidence: float, threshold: float, destination: str, redundant_generic: bool = False, ambiguous_specific: bool = False) -> Dict[str, Any]:
+    margin = round(confidence - threshold, 3)
+    if decision == "promote":
+        status = "verified"
+        reason = "meets_threshold"
+    else:
+        status = "needs_review"
+        if destination == "knowledge" and redundant_generic:
+            reason = "rejected_as_redundant_generic_cruft"
+        elif destination == "knowledge":
+            reason = "rejected_as_generic_cruft"
+        elif ambiguous_specific:
+            reason = "rejected_as_ambiguous_specific_memory"
+        else:
+            reason = "below_threshold"
+    return {
+        "status": status,
+        "reason": reason,
+        "confidence": round(confidence, 3),
+        "threshold": round(threshold, 3),
+        "margin": margin,
+    }
+
+
+def _promotion_explanation(*, decision: str, destination: str, confidence: float, threshold: float, summary: str, redundant_generic: bool = False, ambiguous_specific: bool = False) -> Dict[str, Any]:
+    if decision == "promote":
+        short = f"Promoted to {destination} because confidence {confidence:.2f} met threshold {threshold:.2f}."
+        reason = "confidence_threshold"
+    else:
+        if destination == "knowledge" and redundant_generic:
+            short = f"Rejected as redundant memory cruft because confidence {confidence:.2f} was below threshold {threshold:.2f} and the summary closely matched existing generic knowledge."
+            reason = "rejected_as_redundant_generic_cruft"
+        elif destination == "knowledge":
+            short = f"Rejected as likely memory cruft because confidence {confidence:.2f} was below threshold {threshold:.2f} and the summary did not strongly fit a more specific bucket."
+            reason = "rejected_as_generic_cruft"
+        elif ambiguous_specific:
+            short = f"Rejected as an ambiguous specific memory because confidence {confidence:.2f} was below threshold {threshold:.2f} and the summary only weakly fit destination {destination}."
+            reason = "rejected_as_ambiguous_specific_memory"
+        else:
+            short = f"Rejected because confidence {confidence:.2f} was below threshold {threshold:.2f} for destination {destination}."
+            reason = "below_threshold"
+    return {
+        "short": short,
+        "reason": reason,
+        "destination": destination,
+        "confidence": round(confidence, 3),
+        "threshold": round(threshold, 3),
+        "summary_preview": summary[:160],
+    }
+
+
 def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
     from ocmemog.runtime.memory import api, reinforcement, vector_index
 
     emit_event(LOGFILE, "brain_memory_promote_start", status="ok")
     confidence = float(candidate.get("confidence_score", 0.0))
-    decision = "promote" if _should_promote(confidence) else "reject"
+    threshold = float(config.OCMEMOG_PROMOTION_THRESHOLD)
     candidate_id = str(candidate.get("candidate_id") or "")
+    summary_text = str(candidate.get("distilled_summary", "") or "")
 
     candidate_metadata = provenance.normalize_metadata(candidate.get("metadata", {}), source="promote")
     candidate_metadata["candidate_id"] = candidate_id
@@ -75,11 +192,37 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
 
     conn = store.connect()
     promotion_id = None
-    destination = _destination_table(str(candidate.get("distilled_summary", "")))
+    destination = _destination_table(summary_text)
+    redundant_generic = False
+    should_promote = _should_promote(confidence, threshold)
+    ambiguous_specific = _is_ambiguous_specific_candidate(
+        confidence=confidence,
+        threshold=threshold,
+        destination=destination,
+    )
+    if not should_promote and destination == "knowledge":
+        redundant_generic = _is_redundant_generic_candidate(summary_text)
+    reject_as_cruft = _should_reject_as_cruft(
+        confidence=confidence,
+        threshold=threshold,
+        destination=destination,
+        summary_text=summary_text,
+    )
+    decision = "promote" if should_promote and not reject_as_cruft else "reject"
+    decision_reason = "confidence_threshold"
+    if decision == "reject":
+        if destination == "knowledge" and redundant_generic:
+            decision_reason = "rejected_as_redundant_generic_cruft"
+        elif destination == "knowledge":
+            decision_reason = "rejected_as_generic_cruft"
+        elif ambiguous_specific:
+            decision_reason = "rejected_as_ambiguous_specific_memory"
+        else:
+            decision_reason = "below_threshold"
     if decision == "promote":
         row = conn.execute(
             "SELECT id FROM promotions WHERE source=? AND content=?",
-            (str(candidate.get("source_event_id")), candidate.get("distilled_summary", "")),
+            (str(candidate.get("source_event_id")), summary_text),
         ).fetchone()
         if not row:
             cur = conn.execute(
@@ -94,9 +237,9 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
                     str(candidate.get("source_event_id")),
                     confidence,
                     "promoted",
-                    "confidence_threshold",
+                    decision_reason,
                     json.dumps(candidate_metadata, ensure_ascii=False),
-                    candidate.get("distilled_summary", ""),
+                    summary_text,
                     store.SCHEMA_VERSION,
                 ),
             )
@@ -106,7 +249,7 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
                     str(candidate.get("source_event_id")),
                     confidence,
                     json.dumps(candidate_metadata, ensure_ascii=False),
-                    candidate.get("distilled_summary", ""),
+                    summary_text,
                     store.SCHEMA_VERSION,
                 ),
             )
@@ -142,9 +285,9 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
                 str(candidate.get("source_event_id")),
                 confidence,
                 "rejected",
-                "below_threshold",
+                decision_reason,
                 json.dumps(candidate_metadata, ensure_ascii=False),
-                candidate.get("distilled_summary", ""),
+                summary_text,
                 store.SCHEMA_VERSION,
             ),
         )
@@ -173,7 +316,7 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
         )
         emit_event(LOGFILE, "brain_memory_reinforcement_created", status="ok")
         if memory_id:
-            vector_index.insert_memory(memory_id, candidate.get("distilled_summary", ""), confidence, source_type=destination)
+            vector_index.insert_memory(memory_id, summary_text, confidence, source_type=destination)
             try:
                 api._auto_attach_governance_candidates(promoted_reference)
             except Exception as exc:
@@ -185,7 +328,37 @@ def promote_candidate(candidate: Dict[str, Any]) -> Dict[str, Any]:
                     reference=promoted_reference,
                 )
 
-    return {"decision": decision, "confidence": confidence, "promotion_id": promotion_id, "destination": destination}
+    return {
+        "decision": decision,
+        "confidence": confidence,
+        "promotion_id": promotion_id,
+        "destination": destination,
+        "quality_summary": _quality_summary(
+            decision=decision,
+            confidence=confidence,
+            threshold=threshold,
+            destination=destination,
+            redundant_generic=redundant_generic,
+            ambiguous_specific=ambiguous_specific,
+        ),
+        "verification_summary": _verification_summary(
+            decision=decision,
+            confidence=confidence,
+            threshold=threshold,
+            destination=destination,
+            redundant_generic=redundant_generic,
+            ambiguous_specific=ambiguous_specific,
+        ),
+        "explanation": _promotion_explanation(
+            decision=decision,
+            destination=destination,
+            confidence=confidence,
+            threshold=threshold,
+            summary=str(candidate.get("distilled_summary", "") or ""),
+            redundant_generic=redundant_generic,
+            ambiguous_specific=ambiguous_specific,
+        ),
+    }
 
 
 def promote_candidate_by_id(candidate_id: str) -> Dict[str, Any]: