get-claudia 1.9.0 → 1.9.2

package/memory-daemon/claudia_memory/config.py

@@ -40,6 +40,14 @@ class MemoryConfig:
     recency_weight: float = 0.10  # Weight for recency
     fts_weight: float = 0.15  # Weight for FTS5 full-text search match
 
+    # Memory merging
+    similarity_merge_threshold: float = 0.92  # Cosine similarity threshold for merging
+    enable_memory_merging: bool = True  # Toggle memory merging during consolidation
+
+    # Verification
+    verify_interval_minutes: int = 60  # How often to run background verification
+    verify_batch_size: int = 20  # Max memories to verify per run
+
     # Health check
     health_port: int = 3848
 

package/memory-daemon/claudia_memory/daemon/scheduler.py

@@ -19,6 +19,7 @@ from ..services.consolidate import (
     run_decay,
     run_full_consolidation,
 )
+from ..services.verify import run_verification
 
 logger = logging.getLogger(__name__)
 
@@ -73,6 +74,15 @@ class MemoryScheduler:
             replace_existing=True,
         )
 
+        # Periodic: Memory verification
+        self.scheduler.add_job(
+            self._run_memory_verification,
+            IntervalTrigger(minutes=self.config.verify_interval_minutes),
+            id="memory_verification",
+            name="Background memory verification",
+            replace_existing=True,
+        )
+
         self.scheduler.start()
         self._started = True
         logger.info("Memory scheduler started")
@@ -136,6 +146,15 @@ class MemoryScheduler:
         except Exception as e:
             logger.exception("Error in prediction generation")
 
+    def _run_memory_verification(self) -> None:
+        """Run background memory verification"""
+        try:
+            logger.debug("Running memory verification")
+            result = run_verification()
+            logger.debug(f"Memory verification complete: {result}")
+        except Exception as e:
+            logger.exception("Error in memory verification")
+
 
 # Global scheduler instance
 _scheduler: Optional[MemoryScheduler] = None

package/memory-daemon/claudia_memory/database.py

@@ -303,6 +303,43 @@ class Database:
                 logger.warning(f"Migration 4 (FTS5) failed: {e}. FTS5 may not be available.")
                 # FTS5 is optional; the system degrades gracefully without it
 
+        if current_version < 5:
+            # Migration 5: Add verification columns to memories, pattern_name to predictions
+            migration_stmts = [
+                "ALTER TABLE memories ADD COLUMN verified_at TEXT",
+                "ALTER TABLE memories ADD COLUMN verification_status TEXT DEFAULT 'pending'",
+                "ALTER TABLE predictions ADD COLUMN prediction_pattern_name TEXT",
+            ]
+            for stmt in migration_stmts:
+                try:
+                    conn.execute(stmt)
+                except sqlite3.OperationalError as e:
+                    if "duplicate column" not in str(e).lower():
+                        logger.warning(f"Migration 5 statement failed: {e}")
+
+            # Index for verification queries
+            try:
+                conn.execute(
+                    "CREATE INDEX IF NOT EXISTS idx_memories_verification ON memories(verification_status)"
+                )
+            except sqlite3.OperationalError as e:
+                logger.warning(f"Migration 5 index failed: {e}")
+
+            # Grandfather existing memories as verified
+            try:
+                conn.execute(
+                    """UPDATE memories SET verification_status = 'verified', verified_at = datetime('now')
+                       WHERE verification_status = 'pending' OR verification_status IS NULL"""
+                )
+            except sqlite3.OperationalError as e:
+                logger.warning(f"Migration 5 grandfather failed: {e}")
+
+            conn.execute(
+                "INSERT OR IGNORE INTO schema_migrations (version, description) VALUES (5, 'Add verification columns to memories, prediction_pattern_name to predictions')"
+            )
+            conn.commit()
+            logger.info("Applied migration 5: memory verification and prediction feedback")
+
     def execute(
         self, sql: str, params: Tuple = (), fetch: bool = False
     ) -> Optional[List[sqlite3.Row]]:

package/memory-daemon/claudia_memory/mcp/server.py

@@ -570,6 +570,24 @@ async def list_tools() -> ListToolsResult:
                 "properties": {},
             },
         ),
+        Tool(
+            name="memory.prediction_feedback",
+            description="Provide feedback on a prediction -- mark whether the user acted on it. This trains future prediction priority.",
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "prediction_id": {
+                        "type": "integer",
+                        "description": "The prediction ID to provide feedback for",
+                    },
+                    "acted_on": {
+                        "type": "boolean",
+                        "description": "Whether the user acted on this prediction",
+                    },
+                },
+                "required": ["prediction_id", "acted_on"],
+            },
+        ),
         Tool(
             name="cognitive.ingest",
             description=(
@@ -1021,6 +1039,21 @@ async def call_tool(name: str, arguments: Dict[str, Any]) -> CallToolResult:
                 ]
             )
 
+        elif name == "memory.prediction_feedback":
+            svc = get_consolidate_service()
+            svc.mark_prediction_acted_on(
+                prediction_id=arguments["prediction_id"],
+                acted_on=arguments["acted_on"],
+            )
+            return CallToolResult(
+                content=[
+                    TextContent(
+                        type="text",
+                        text=json.dumps({"success": True, "prediction_id": arguments["prediction_id"]}),
+                    )
+                ]
+            )
+
         elif name == "memory.trace":
             result = trace_memory(memory_id=arguments["memory_id"])
             return CallToolResult(

package/memory-daemon/claudia_memory/schema.sql

@@ -58,6 +58,8 @@ CREATE TABLE IF NOT EXISTS memories (
     updated_at TEXT DEFAULT (datetime('now')),
     last_accessed_at TEXT,  -- For rehearsal-based importance boost
     access_count INTEGER DEFAULT 0,
+    verified_at TEXT,  -- When this memory was verified
+    verification_status TEXT DEFAULT 'pending',  -- pending, verified, flagged, contradicts
     metadata TEXT  -- JSON blob for flexible attributes
 );
 
@@ -65,6 +67,7 @@ CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
 CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance DESC);
 CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_memories_hash ON memories(content_hash);
+CREATE INDEX IF NOT EXISTS idx_memories_verification ON memories(verification_status);
 
 -- Junction table linking memories to entities
 CREATE TABLE IF NOT EXISTS memory_entities (
@@ -171,6 +174,7 @@ CREATE TABLE IF NOT EXISTS predictions (
     is_acted_on INTEGER DEFAULT 0,  -- Whether user acted on this
     created_at TEXT DEFAULT (datetime('now')),
     shown_at TEXT,
+    prediction_pattern_name TEXT,  -- Links to pattern for feedback loop
     metadata TEXT
 );
 
@@ -256,3 +260,6 @@ VALUES (3, 'Add source_context to memories, is_archived to turn_buffer for episo
 -- NOTE: FTS5 full-text search (migration v4) is created by database.py migration code
 -- rather than here, because CREATE TRIGGER statements contain internal semicolons
 -- that the schema.sql line-based parser cannot handle.
+
+INSERT OR IGNORE INTO schema_migrations (version, description)
+VALUES (5, 'Add verification columns to memories, prediction_pattern_name to predictions');

package/memory-daemon/claudia_memory/services/consolidate.py

@@ -18,6 +18,18 @@ from ..database import get_db
 logger = logging.getLogger(__name__)
 
 
+def _cosine_similarity(a: list, b: list) -> float:
+    """Pure Python cosine similarity between two vectors."""
+    if len(a) != len(b) or len(a) == 0:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = sum(x * x for x in a) ** 0.5
+    norm_b = sum(x * x for x in b) ** 0.5
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
 @dataclass
 class DetectedPattern:
     """A pattern detected in the user's behavior or data"""
@@ -38,6 +50,7 @@ class Prediction:
     priority: float
     expires_at: Optional[datetime]
     metadata: Optional[Dict] = None
+    pattern_name: Optional[str] = None
 
 
 class ConsolidateService:
@@ -409,13 +422,15 @@ class ConsolidateService:
         )
 
         for pattern in cooling_patterns:
+            feedback = self._get_pattern_feedback("suggestion", pattern["name"])
             predictions.append(
                 Prediction(
                     content=pattern["description"],
                     prediction_type="suggestion",
-                    priority=pattern["confidence"],
+                    priority=pattern["confidence"] * feedback,
                     expires_at=datetime.utcnow() + timedelta(days=7),
                     metadata={"pattern_id": pattern["id"]},
+                    pattern_name=pattern["name"],
                 )
             )
 
@@ -438,13 +453,16 @@ class ConsolidateService:
             days_old = (datetime.utcnow() - created).days
 
             if days_old > 3:
+                pattern_name = f"commitment_reminder_{commitment['id']}"
+                feedback = self._get_pattern_feedback("reminder", pattern_name)
                 predictions.append(
                     Prediction(
                         content=f"Commitment from {days_old} days ago: {commitment['content'][:100]}",
                         prediction_type="reminder",
-                        priority=min(1.0, 0.5 + days_old / 14),
+                        priority=min(1.0, 0.5 + days_old / 14) * feedback,
                         expires_at=datetime.utcnow() + timedelta(days=2),
                         metadata={"memory_id": commitment["id"]},
+                        pattern_name=pattern_name,
                     )
                 )
 
@@ -464,13 +482,15 @@ class ConsolidateService:
 
         for pattern in patterns:
             if pattern["pattern_type"] == "behavioral":
+                feedback = self._get_pattern_feedback("insight", pattern["name"])
                 predictions.append(
                     Prediction(
                         content=f"Pattern noticed: {pattern['description']}",
                         prediction_type="insight",
-                        priority=pattern["confidence"] * 0.8,
+                        priority=pattern["confidence"] * 0.8 * feedback,
                         expires_at=datetime.utcnow() + timedelta(days=14),
                         metadata={"pattern_id": pattern["id"]},
+                        pattern_name=pattern["name"],
                     )
                 )
 
@@ -478,19 +498,19 @@ class ConsolidateService:
 
     def _store_prediction(self, prediction: Prediction) -> int:
         """Store a prediction in the database"""
-        return self.db.insert(
-            "predictions",
-            {
-                "content": prediction.content,
-                "prediction_type": prediction.prediction_type,
-                "priority": prediction.priority,
-                "expires_at": prediction.expires_at.isoformat() if prediction.expires_at else None,
-                "is_shown": 0,
-                "is_acted_on": 0,
-                "created_at": datetime.utcnow().isoformat(),
-                "metadata": json.dumps(prediction.metadata) if prediction.metadata else None,
-            },
-        )
+        data = {
+            "content": prediction.content,
+            "prediction_type": prediction.prediction_type,
+            "priority": prediction.priority,
+            "expires_at": prediction.expires_at.isoformat() if prediction.expires_at else None,
+            "is_shown": 0,
+            "is_acted_on": 0,
+            "created_at": datetime.utcnow().isoformat(),
+            "metadata": json.dumps(prediction.metadata) if prediction.metadata else None,
+        }
+        if prediction.pattern_name:
+            data["prediction_pattern_name"] = prediction.pattern_name
+        return self.db.insert("predictions", data)
 
     def get_predictions(
         self,
@@ -539,6 +559,208 @@ class ConsolidateService:
             (prediction_id,),
         )
 
+    def mark_prediction_acted_on(self, prediction_id: int, acted_on: bool) -> None:
+        """Mark a prediction as acted on (or not) by the user"""
+        self.db.update(
+            "predictions",
+            {"is_acted_on": 1 if acted_on else 0},
+            "id = ?",
+            (prediction_id,),
+        )
+
+    def _get_pattern_feedback(self, prediction_type: str, pattern_name: str) -> float:
+        """
+        Return a priority multiplier based on past user engagement with similar predictions.
+
+        Rules:
+        - <5 shown predictions for this type: return 1.0 (insufficient data)
+        - act_ratio < 0.1: return 0.5 (user ignores these, halve priority)
+        - act_ratio > 0.5: return 1.25 (user values these, boost priority)
+        - Otherwise: return 1.0 (neutral)
+        """
+        try:
+            rows = self.db.execute(
+                """
+                SELECT COUNT(*) as total,
+                       SUM(CASE WHEN is_acted_on = 1 THEN 1 ELSE 0 END) as acted
+                FROM predictions
+                WHERE prediction_type = ? AND is_shown = 1
+                """,
+                (prediction_type,),
+                fetch=True,
+            )
+            if not rows:
+                return 1.0
+
+            total = rows[0]["total"] or 0
+            acted = rows[0]["acted"] or 0
+
+            if total < 5:
+                return 1.0
+
+            act_ratio = acted / total
+            if act_ratio < 0.1:
+                return 0.5
+            elif act_ratio > 0.5:
+                return 1.25
+            return 1.0
+        except Exception as e:
+            logger.debug(f"Feedback lookup failed: {e}")
+            return 1.0
+
+    def merge_similar_memories(self) -> int:
+        """
+        Merge semantically similar memories during consolidation.
+        Uses existing stored embeddings -- no new Ollama calls.
+
+        Returns:
+            Count of merged memory pairs
+        """
+        if not self.config.enable_memory_merging:
+            return 0
+
+        threshold = self.config.similarity_merge_threshold
+        merged_count = 0
+
+        try:
+            # Find entities with 5+ linked memories (high-memory entities first)
+            entity_rows = self.db.execute(
+                """
+                SELECT me.entity_id, COUNT(DISTINCT me.memory_id) as mem_count
+                FROM memory_entities me
+                GROUP BY me.entity_id
+                HAVING mem_count >= 5
+                ORDER BY mem_count DESC
+                LIMIT 50
+                """,
+                fetch=True,
+            ) or []
+
+            for entity_row in entity_rows:
+                entity_id = entity_row["entity_id"]
+
+                # Load memory IDs and embeddings for this entity
+                mem_rows = self.db.execute(
+                    """
+                    SELECT me.memory_id, m.importance, m.access_count,
+                           emb.embedding
+                    FROM memory_entities me
+                    JOIN memories m ON me.memory_id = m.id
+                    LEFT JOIN memory_embeddings emb ON m.id = emb.memory_id
+                    WHERE me.entity_id = ?
+                      AND m.importance > 0.01
+                    ORDER BY m.importance DESC
+                    """,
+                    (entity_id,),
+                    fetch=True,
+                ) or []
+
+                # Parse embeddings
+                memories_with_emb = []
+                for row in mem_rows:
+                    if row["embedding"]:
+                        try:
+                            emb = json.loads(row["embedding"]) if isinstance(row["embedding"], str) else row["embedding"]
+                            memories_with_emb.append({
+                                "id": row["memory_id"],
+                                "importance": row["importance"],
+                                "access_count": row["access_count"] or 0,
+                                "embedding": emb,
+                            })
+                        except (json.JSONDecodeError, TypeError):
+                            continue
+
+                if len(memories_with_emb) < 2:
+                    continue
+
+                # Pairwise cosine similarity
+                already_merged = set()
+                for i in range(len(memories_with_emb)):
+                    if memories_with_emb[i]["id"] in already_merged:
+                        continue
+                    for j in range(i + 1, len(memories_with_emb)):
+                        if memories_with_emb[j]["id"] in already_merged:
+                            continue
+
+                        sim = _cosine_similarity(
+                            memories_with_emb[i]["embedding"],
+                            memories_with_emb[j]["embedding"],
+                        )
+                        if sim >= threshold:
+                            # Keep the one with higher importance * (1 + access_count)
+                            score_i = memories_with_emb[i]["importance"] * (1 + memories_with_emb[i]["access_count"])
+                            score_j = memories_with_emb[j]["importance"] * (1 + memories_with_emb[j]["access_count"])
+
+                            if score_i >= score_j:
+                                primary_id = memories_with_emb[i]["id"]
+                                duplicate_id = memories_with_emb[j]["id"]
+                            else:
+                                primary_id = memories_with_emb[j]["id"]
+                                duplicate_id = memories_with_emb[i]["id"]
+
+                            self._merge_memory_pair(primary_id, duplicate_id)
+                            already_merged.add(duplicate_id)
+                            merged_count += 1
+
+        except Exception as e:
+            logger.warning(f"Memory merging failed: {e}")
+
+        if merged_count > 0:
+            logger.info(f"Merged {merged_count} near-duplicate memory pairs")
+        return merged_count
+
+    def _merge_memory_pair(self, primary_id: int, duplicate_id: int) -> None:
+        """
+        Merge a duplicate memory into the primary.
+
+        - Transfers entity links from duplicate to primary
+        - Adds merged_from to primary's metadata
+        - Sets duplicate importance to 0.001
+        """
+        # Transfer entity links
+        dup_links = self.db.execute(
+            "SELECT entity_id, relationship FROM memory_entities WHERE memory_id = ?",
+            (duplicate_id,),
+            fetch=True,
+        ) or []
+
+        for link in dup_links:
+            try:
+                self.db.insert(
+                    "memory_entities",
+                    {
+                        "memory_id": primary_id,
+                        "entity_id": link["entity_id"],
+                        "relationship": link["relationship"],
+                    },
+                )
+            except Exception:
+                pass  # Duplicate link, ignore
+
+        # Update primary's metadata with merge info
+        primary = self.db.get_one("memories", where="id = ?", where_params=(primary_id,))
+        if primary:
+            meta = json.loads(primary["metadata"] or "{}")
+            merged_from = meta.get("merged_from", [])
+            merged_from.append(duplicate_id)
+            meta["merged_from"] = merged_from
+            self.db.update(
+                "memories",
+                {"metadata": json.dumps(meta), "updated_at": datetime.utcnow().isoformat()},
+                "id = ?",
+                (primary_id,),
+            )
+
+        # Suppress duplicate (don't delete, just minimize importance)
+        self.db.update(
+            "memories",
+            {"importance": 0.001, "updated_at": datetime.utcnow().isoformat()},
+            "id = ?",
+            (duplicate_id,),
+        )
+
+        logger.debug(f"Merged memory {duplicate_id} into {primary_id}")
+
     def run_full_consolidation(self) -> Dict[str, Any]:
         """
         Run complete consolidation: decay, patterns, predictions.
@@ -554,6 +776,9 @@ class ConsolidateService:
         # Boost accessed memories
         results["boosted"] = self.boost_accessed_memories()
 
+        # Merge near-duplicate memories
+        results["merged"] = self.merge_similar_memories()
+
         # Detect patterns
         patterns = self.detect_patterns()
         results["patterns_detected"] = len(patterns)

package/memory-daemon/claudia_memory/services/guards.py

@@ -0,0 +1,131 @@
+"""
+Deterministic Guards for Claudia Memory System
+
+Pure-Python validation on memory writes. Zero LLM cost, always on.
+Guards are advisory -- they warn and auto-correct, never block writes.
+"""
+
+import logging
+import re
+from dataclasses import dataclass, field
+from difflib import SequenceMatcher
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# Regex patterns for commitment deadline detection
+DEADLINE_PATTERNS = [
+    re.compile(r"\b(by|before|due|until|deadline)\s+\w+", re.IGNORECASE),
+    re.compile(r"\b\d{1,2}[/-]\d{1,2}([/-]\d{2,4})?\b"),  # Date formats: 1/15, 01-15-2025
+    re.compile(r"\b(january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2}\b", re.IGNORECASE),
+    re.compile(r"\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b", re.IGNORECASE),
+    re.compile(r"\b(tomorrow|tonight|next week|next month|end of (week|month|day|year))\b", re.IGNORECASE),
+    re.compile(r"\bEOD\b|\bEOW\b|\bEOM\b"),
+]
+
+
+@dataclass
+class ValidationResult:
+    """Result of a validation check"""
+    is_valid: bool = True
+    warnings: List[str] = field(default_factory=list)
+    adjustments: Dict[str, Any] = field(default_factory=dict)
+
+
+def validate_memory(
+    content: str,
+    memory_type: str = "fact",
+    importance: float = 1.0,
+    metadata: Optional[Dict] = None,
+) -> ValidationResult:
+    """
+    Validate a memory before storage.
+
+    Checks:
+    - Content length (warn >500, truncate >1000)
+    - Commitment deadline detection via regex
+    - Importance clamped to [0, 1]
+    """
+    result = ValidationResult()
+
+    # Content length checks
+    if len(content) > 1000:
+        result.warnings.append(f"Content truncated from {len(content)} to 1000 characters")
+        result.adjustments["content"] = content[:1000]
+    elif len(content) > 500:
+        result.warnings.append(f"Long content ({len(content)} chars) -- consider breaking into multiple memories")
+
+    # Importance clamping
+    if importance < 0:
+        result.warnings.append(f"Importance {importance} clamped to 0.0")
+        result.adjustments["importance"] = 0.0
+    elif importance > 1:
+        result.warnings.append(f"Importance {importance} clamped to 1.0")
+        result.adjustments["importance"] = 1.0
+
+    # Commitment deadline detection
+    if memory_type == "commitment":
+        has_deadline = any(p.search(content) for p in DEADLINE_PATTERNS)
+        if not has_deadline:
+            result.warnings.append("Commitment has no detected deadline -- consider adding a target date")
+
+    return result
+
+
+def validate_entity(
+    name: str,
+    entity_type: str = "",
+    existing_canonical_names: Optional[List[str]] = None,
+) -> ValidationResult:
+    """
+    Validate an entity before storage.
+
+    Checks:
+    - Type is required (non-empty)
+    - Name is not empty
+    - Near-duplicate name detection via SequenceMatcher (ratio > 0.85)
+    """
+    result = ValidationResult()
+
+    # Name check
+    if not name or not name.strip():
+        result.is_valid = False
+        result.warnings.append("Entity name cannot be empty")
+        return result
+
+    # Type check
+    if not entity_type or not entity_type.strip():
+        result.warnings.append("Entity type is required but was empty -- defaulting to 'person'")
+        result.adjustments["entity_type"] = "person"
+
+    # Near-duplicate detection
+    if existing_canonical_names:
+        canonical = name.strip().lower()
+        for existing in existing_canonical_names:
+            ratio = SequenceMatcher(None, canonical, existing.lower()).ratio()
+            if ratio > 0.85 and canonical != existing.lower():
+                result.warnings.append(
+                    f"Near-duplicate entity name: '{name}' is similar to existing '{existing}' "
+                    f"(similarity: {ratio:.2f})"
+                )
+
+    return result
+
+
+def validate_relationship(strength: float = 1.0) -> ValidationResult:
+    """
+    Validate a relationship before storage.
+
+    Checks:
+    - Strength clamped to [0, 1]
+    """
+    result = ValidationResult()
+
+    if strength < 0:
+        result.warnings.append(f"Relationship strength {strength} clamped to 0.0")
+        result.adjustments["strength"] = 0.0
+    elif strength > 1:
+        result.warnings.append(f"Relationship strength {strength} clamped to 1.0")
+        result.adjustments["strength"] = 1.0
+
+    return result