@smilintux/skmemory 0.5.0 → 0.9.2

package/skmemory/store.py

@@ -8,11 +8,12 @@ or by search, and the polaroid comes back with everything intact.
 
 from __future__ import annotations
 
+import logging
 from datetime import datetime, timezone
-from typing import Optional
 
 from .backends.base import BaseBackend
 from .backends.file_backend import FileBackend
+from .backends.skgraph_backend import SKGraphBackend
 from .backends.sqlite_backend import CONTENT_PREVIEW_LENGTH, SQLiteBackend
 from .models import (
     EmotionalSnapshot,
@@ -22,23 +23,35 @@ from .models import (
     SeedMemory,
 )
 
+logger = logging.getLogger("skmemory.store")
+
+MAX_CONTENT_LENGTH = 10000
+CONTENT_OVERFLOW_STRATEGY = "split"  # "truncate" or "split"
+
 
 class MemoryStore:
     """Main entry point for all memory operations.
 
     Delegates to one or more backends. The primary backend handles
     all CRUD. A vector backend (optional) handles semantic search.
+    A graph backend (optional) indexes relationships for traversal.
 
     Args:
         primary: The primary storage backend (default: FileBackend).
-        vector: Optional vector search backend (e.g., QdrantBackend).
+        vector: Optional vector search backend (e.g., SKVectorBackend).
+        graph: Optional graph backend (e.g., SKGraphBackend) for relationship indexing.
+        max_content_length: Max chars before overflow strategy applies (default: 10000).
+        content_overflow_strategy: "truncate" or "split" (default: "split").
     """
 
     def __init__(
         self,
-        primary: Optional[BaseBackend] = None,
-        vector: Optional[BaseBackend] = None,
+        primary: BaseBackend | None = None,
+        vector: BaseBackend | None = None,
+        graph: SKGraphBackend | None = None,
         use_sqlite: bool = True,
+        max_content_length: int = MAX_CONTENT_LENGTH,
+        content_overflow_strategy: str = CONTENT_OVERFLOW_STRATEGY,
     ) -> None:
         if primary is not None:
             self.primary = primary
@@ -47,6 +60,9 @@ class MemoryStore:
         else:
             self.primary = FileBackend()
         self.vector = vector
+        self.graph = graph
+        self.max_content_length = max_content_length
+        self.content_overflow_strategy = content_overflow_strategy
 
     def snapshot(
         self,
@@ -55,12 +71,12 @@ class MemoryStore:
         *,
         layer: MemoryLayer = MemoryLayer.SHORT,
         role: MemoryRole = MemoryRole.GENERAL,
-        tags: Optional[list[str]] = None,
-        emotional: Optional[EmotionalSnapshot] = None,
+        tags: list[str] | None = None,
+        emotional: EmotionalSnapshot | None = None,
         source: str = "manual",
         source_ref: str = "",
-        related_ids: Optional[list[str]] = None,
-        metadata: Optional[dict] = None,
+        related_ids: list[str] | None = None,
+        metadata: dict | None = None,
     ) -> Memory:
         """Take a polaroid -- capture a moment as a memory.
 
@@ -82,6 +98,30 @@ class MemoryStore:
         Returns:
             Memory: The stored memory with its assigned ID.
         """
+        # Handle content overflow
+        if len(content) > self.max_content_length:
+            if self.content_overflow_strategy == "split":
+                return self._snapshot_split(
+                    title=title,
+                    content=content,
+                    layer=layer,
+                    role=role,
+                    tags=tags,
+                    emotional=emotional,
+                    source=source,
+                    source_ref=source_ref,
+                    related_ids=related_ids,
+                    metadata=metadata,
+                )
+            else:
+                logger.info(
+                    "Content truncated from %d to %d chars for '%s'",
+                    len(content),
+                    self.max_content_length,
+                    title,
+                )
+                content = content[: self.max_content_length]
+
         memory = Memory(
             title=title,
             content=content,
@@ -95,18 +135,122 @@ class MemoryStore:
             metadata=metadata or {},
         )
 
+        memory.seal()
+
         self.primary.save(memory)
 
         if self.vector:
             try:
                 self.vector.save(memory)
-            except Exception:
-                pass  # Reason: vector indexing is best-effort, don't fail the write
+            except Exception as exc:
+                logger.warning("Vector indexing failed for memory %s: %s", memory.id, exc)
+
+        if self.graph:
+            try:
+                self.graph.index_memory(memory)
+            except Exception as exc:
+                logger.warning("Graph indexing failed for memory %s: %s", memory.id, exc)
 
         return memory
 
-    def recall(self, memory_id: str) -> Optional[Memory]:
-        """Retrieve a specific memory by ID.
+    def _snapshot_split(
+        self,
+        title: str,
+        content: str,
+        *,
+        layer: MemoryLayer = MemoryLayer.SHORT,
+        role: MemoryRole = MemoryRole.GENERAL,
+        tags: list[str] | None = None,
+        emotional: EmotionalSnapshot | None = None,
+        source: str = "manual",
+        source_ref: str = "",
+        related_ids: list[str] | None = None,
+        metadata: dict | None = None,
+    ) -> Memory:
+        """Split oversized content into parent (summary) + child (chunk) memories.
+
+        The parent memory contains a summary (first 200 chars) and links to
+        child memories via related_ids. Each child holds one chunk.
+
+        Returns:
+            Memory: The parent memory.
+        """
+        chunk_size = self.max_content_length
+        chunks = [content[i : i + chunk_size] for i in range(0, len(content), chunk_size)]
+
+        logger.info(
+            "Splitting '%s' (%d chars) into %d chunks",
+            title,
+            len(content),
+            len(chunks),
+        )
+
+        # Create child memories first
+        child_ids: list[str] = []
+        for i, chunk in enumerate(chunks):
+            child = Memory(
+                title=f"{title} [part {i + 1}/{len(chunks)}]",
+                content=chunk,
+                layer=layer,
+                role=role,
+                tags=(tags or []) + ["content-chunk"],
+                emotional=emotional or EmotionalSnapshot(),
+                source=source,
+                source_ref=source_ref,
+                metadata={
+                    **(metadata or {}),
+                    "chunk_index": i,
+                    "chunk_total": len(chunks),
+                },
+            )
+            child.seal()
+            self.primary.save(child)
+            child_ids.append(child.id)
+
+        # Create parent with summary
+        summary = content[:200] + ("..." if len(content) > 200 else "")
+        all_related = (related_ids or []) + child_ids
+
+        parent = Memory(
+            title=title,
+            content=summary,
+            summary=summary,
+            layer=layer,
+            role=role,
+            tags=(tags or []) + ["content-split-parent"],
+            emotional=emotional or EmotionalSnapshot(),
+            source=source,
+            source_ref=source_ref,
+            related_ids=all_related,
+            metadata={
+                **(metadata or {}),
+                "split_children": child_ids,
+                "original_length": len(content),
+            },
+        )
+        parent.seal()
+        self.primary.save(parent)
+
+        if self.vector:
+            try:
+                self.vector.save(parent)
+            except Exception as exc:
+                logger.warning("Vector indexing failed for split parent %s: %s", parent.id, exc)
+
+        if self.graph:
+            try:
+                self.graph.index_memory(parent)
+            except Exception as exc:
+                logger.warning("Graph indexing failed for split parent %s: %s", parent.id, exc)
+
+        return parent
+
+    def recall(self, memory_id: str) -> Memory | None:
+        """Retrieve a specific memory by ID with integrity verification.
+
+        Automatically checks the integrity hash on recall. If the
+        memory has been tampered with, a warning is logged and the
+        memory's metadata is flagged with 'integrity_warning'.
 
         Args:
             memory_id: The memory's unique identifier.
@@ -114,7 +258,22 @@ class MemoryStore:
         Returns:
             Optional[Memory]: The memory if found.
         """
-        return self.primary.load(memory_id)
+        memory = self.primary.load(memory_id)
+        if memory is None:
+            return None
+
+        if memory.integrity_hash and not memory.verify_integrity():
+            logger.warning(
+                "TAMPER ALERT: Memory %s failed integrity check! "
+                "Content may have been modified since storage.",
+                memory_id,
+            )
+            memory.metadata["integrity_warning"] = (
+                f"Integrity check failed at {datetime.now(timezone.utc).isoformat()}. "
+                "This memory may have been tampered with."
+            )
+
+        return memory
 
     def search(self, query: str, limit: int = 10) -> list[Memory]:
         """Search memories by text.
@@ -133,8 +292,8 @@ class MemoryStore:
                 results = self.vector.search_text(query, limit=limit)
                 if results:
                     return results
-            except Exception:
-                pass  # Reason: fall through to primary text search
+            except Exception as exc:
+                logger.warning("Vector search failed, falling back to text search: %s", exc)
 
         return self.primary.search_text(query, limit=limit)
 
@@ -151,14 +310,19 @@ class MemoryStore:
         if self.vector:
             try:
                 self.vector.delete(memory_id)
-            except Exception:
-                pass
+            except Exception as exc:
+                logger.warning("Vector delete failed for memory %s: %s", memory_id, exc)
+        if self.graph:
+            try:
+                self.graph.remove_memory(memory_id)
+            except Exception as exc:
+                logger.warning("Graph delete failed for memory %s: %s", memory_id, exc)
         return deleted
 
     def list_memories(
         self,
-        layer: Optional[MemoryLayer] = None,
-        tags: Optional[list[str]] = None,
+        layer: MemoryLayer | None = None,
+        tags: list[str] | None = None,
         limit: int = 50,
     ) -> list[Memory]:
         """List memories with optional filtering.
@@ -178,7 +342,7 @@ class MemoryStore:
         memory_id: str,
         target: MemoryLayer,
         summary: str = "",
-    ) -> Optional[Memory]:
+    ) -> Memory | None:
         """Promote a memory to a higher persistence tier.
 
         Creates a new memory at the target layer linked to the original.
@@ -202,32 +366,65 @@ class MemoryStore:
         if self.vector:
             try:
                 self.vector.save(promoted)
-            except Exception:
-                pass
+            except Exception as exc:
+                logger.warning(
+                    "Vector indexing failed for promoted memory %s: %s", promoted.id, exc
+                )
+
+        if self.graph:
+            try:
+                self.graph.index_memory(promoted)
+            except Exception as exc:
+                logger.warning(
+                    "Graph indexing failed for promoted memory %s: %s", promoted.id, exc
+                )
 
         return promoted
 
-    def ingest_seed(self, seed: SeedMemory) -> Memory:
+    def ingest_seed(self, seed: SeedMemory, *, validate: bool = True) -> Memory:
         """Import a Cloud 9 seed as a long-term memory.
 
         Converts a seed into a Memory and stores it. This is how
         seeds planted by one AI instance become retrievable memories
         for the next.
 
+        When *validate* is True (default), basic integrity checks run
+        before storage: seed_id must be non-empty and
+        experience_summary must contain content.
+
         Args:
             seed: The SeedMemory to import.
+            validate: Run pre-import validation (default True).
 
         Returns:
             Memory: The created long-term memory.
+
+        Raises:
+            ValueError: If validation is enabled and the seed is invalid.
         """
+        if validate:
+            errors: list[str] = []
+            if not seed.seed_id or not seed.seed_id.strip():
+                errors.append("seed_id is empty")
+            if not seed.experience_summary or not seed.experience_summary.strip():
+                errors.append("experience_summary is empty")
+            if errors:
+                raise ValueError(f"Seed validation failed: {'; '.join(errors)}")
+
         memory = seed.to_memory()
         self.primary.save(memory)
 
         if self.vector:
             try:
                 self.vector.save(memory)
-            except Exception:
-                pass
+            except Exception as exc:
+                logger.warning("Vector indexing failed for seed memory %s: %s", memory.id, exc)
+
+        if self.graph:
+            try:
+                self.graph.index_memory(memory)
+            except Exception as exc:
+                logger.warning("Graph indexing failed for seed memory %s: %s", memory.id, exc)
 
         return memory
 
@@ -249,7 +446,7 @@ class MemoryStore:
         self,
         session_id: str,
         summary: str,
-        emotional: Optional[EmotionalSnapshot] = None,
+        emotional: EmotionalSnapshot | None = None,
     ) -> Memory:
         """Compress a session's short-term memories into a single mid-term memory.
 
@@ -291,89 +488,156 @@ class MemoryStore:
 
     def load_context(
         self,
-        max_tokens: int = 3000,
+        max_tokens: int = 4000,
         strongest_count: int = 5,
         recent_count: int = 5,
         include_seeds: bool = True,
     ) -> dict:
-        """Load a token-efficient memory context for agent injection.
+        """Load tiered memory context for agent injection (lazy loading).
 
-        Uses the SQLite index to pull summaries without reading full files.
-        Designed to fit within a reasonable context window.
+        Uses date-based tiers per memory-architecture.md:
+        - Today's memories: full content (title + body)
+        - Yesterday's memories: summary only (title + first 2 sentences)
+        - Older than 2 days: reference count only
 
         Args:
-            max_tokens: Approximate token budget (1 token ~= 4 chars).
+            max_tokens: Approximate token budget (default: 4000).
+                Uses word_count * 1.3 approximation for estimation.
             strongest_count: How many top-intensity memories to include.
             recent_count: How many recent memories to include.
             include_seeds: Whether to include seed memories.
 
         Returns:
-            dict: Token-efficient context with summaries and metadata.
+            dict: Token-efficient tiered context with metadata.
         """
-        char_budget = max_tokens * 4
-        context: dict = {"memories": [], "seeds": [], "stats": {}}
-        used = 0
+        context: dict = {
+            "today": [],
+            "yesterday": [],
+            "older_summary": {},
+            "seeds": [],
+            "stats": {},
+        }
+        used_tokens = 0
 
         if isinstance(self.primary, SQLiteBackend):
-            strongest = self.primary.list_summaries(
-                limit=strongest_count,
-                order_by="emotional_intensity",
-                min_intensity=3.0,
-            )
-            recent = self.primary.list_summaries(
-                limit=recent_count,
-                order_by="created_at",
-            )
-
-            seen_ids: set[str] = set()
-            for mem in strongest + recent:
-                if mem["id"] in seen_ids:
-                    continue
-                seen_ids.add(mem["id"])
-
-                entry_text = mem["title"] + (mem["summary"] or mem["content_preview"])
-                entry_size = len(entry_text)
-                if used + entry_size > char_budget:
+            conn = self.primary._get_conn()
+
+            # --- Tier 1: Today's memories (full content) ---
+            today_rows = conn.execute(
+                "SELECT * FROM memories WHERE DATE(created_at) = DATE('now') "
+                "ORDER BY importance DESC, created_at DESC LIMIT 20"
+            ).fetchall()
+
+            for row in today_rows:
+                summary_dict = self.primary._row_to_memory_summary(row)
+                # Include full content for today
+                content = summary_dict.get("summary") or summary_dict.get("content_preview") or ""
+                entry = {
+                    "id": summary_dict["id"],
+                    "title": summary_dict["title"],
+                    "content": content,
+                    "tags": summary_dict["tags"],
+                    "layer": summary_dict["layer"],
+                    "emotional_intensity": summary_dict["emotional_intensity"],
+                }
+                entry_tokens = _estimate_tokens(entry["title"] + " " + content)
+                if used_tokens + entry_tokens > max_tokens:
                     break
-                used += entry_size
-                context["memories"].append(mem)
-
+                used_tokens += entry_tokens
+                context["today"].append(entry)
+
+            # --- Tier 2: Yesterday's memories (summary only: title + first 2 sentences) ---
+            yesterday_rows = conn.execute(
+                "SELECT * FROM memories WHERE DATE(created_at) = DATE('now', '-1 day') "
+                "ORDER BY importance DESC, created_at DESC LIMIT 20"
+            ).fetchall()
+
+            for row in yesterday_rows:
+                summary_dict = self.primary._row_to_memory_summary(row)
+                raw_text = summary_dict.get("summary") or summary_dict.get("content_preview") or ""
+                short_summary = _first_n_sentences(raw_text, 2)
+                entry = {
+                    "id": summary_dict["id"],
+                    "title": summary_dict["title"],
+                    "summary": short_summary,
+                }
+                entry_tokens = _estimate_tokens(entry["title"] + " " + short_summary)
+                if used_tokens + entry_tokens > max_tokens:
+                    break
+                used_tokens += entry_tokens
+                context["yesterday"].append(entry)
+
+            # --- Tier 3: Older memories (reference count only) ---
+            mid_count = conn.execute(
+                "SELECT COUNT(*) FROM memories WHERE DATE(created_at) < DATE('now', '-1 day') "
+                "AND layer = 'mid-term'"
+            ).fetchone()[0]
+            long_count = conn.execute(
+                "SELECT COUNT(*) FROM memories WHERE DATE(created_at) < DATE('now', '-1 day') "
+                "AND layer = 'long-term'"
+            ).fetchone()[0]
+            short_old_count = conn.execute(
+                "SELECT COUNT(*) FROM memories WHERE DATE(created_at) < DATE('now', '-1 day') "
+                "AND layer = 'short-term'"
+            ).fetchone()[0]
+
+            context["older_summary"] = {
+                "mid_term_count": mid_count,
+                "long_term_count": long_count,
+                "short_term_count": short_old_count,
+                "total": mid_count + long_count + short_old_count,
+                "hint": (
+                    f"{mid_count} mid-term memories, {long_count} long-term memories "
+                    "available via memory_search"
+                ),
+            }
+            used_tokens += _estimate_tokens(context["older_summary"]["hint"])
+
+            # --- Seeds (titles only to save tokens) ---
             if include_seeds:
-                seeds = self.primary.list_summaries(
+                seed_rows = self.primary.list_summaries(
                     tags=["seed"],
                     limit=10,
                     order_by="emotional_intensity",
                 )
-                for seed in seeds:
+                seen_ids = {m["id"] for m in context["today"]}
+                seen_ids.update(m["id"] for m in context["yesterday"])
+
+                for seed in seed_rows:
                     if seed["id"] in seen_ids:
                         continue
-                    entry_text = seed["title"] + seed["summary"]
-                    entry_size = len(entry_text)
-                    if used + entry_size > char_budget:
+                    entry = {
+                        "id": seed["id"],
+                        "title": seed["title"],
+                    }
+                    entry_tokens = _estimate_tokens(seed["title"])
+                    if used_tokens + entry_tokens > max_tokens:
                         break
-                    used += entry_size
-                    context["seeds"].append(seed)
+                    used_tokens += entry_tokens
+                    context["seeds"].append(entry)
 
             stats = self.primary.stats()
             context["stats"] = stats
         else:
-            # Reason: fallback for non-SQLite backends — uses full objects
+            # Fallback for non-SQLite backends: simple recent list
             all_mems = self.primary.list_memories(limit=strongest_count + recent_count)
             for mem in all_mems:
+                content_text = mem.summary or mem.content[:CONTENT_PREVIEW_LENGTH]
                 entry = {
                     "id": mem.id,
                     "title": mem.title,
-                    "summary": mem.summary or mem.content[:CONTENT_PREVIEW_LENGTH],
+                    "summary": _first_n_sentences(content_text, 2),
                     "emotional_intensity": mem.emotional.intensity,
                     "layer": mem.layer.value,
                 }
-                entry_size = len(entry["title"] + entry["summary"])
-                if used + entry_size > char_budget:
+                entry_tokens = _estimate_tokens(entry["title"] + " " + entry["summary"])
+                if used_tokens + entry_tokens > max_tokens:
                     break
-                used += entry_size
-                context["memories"].append(entry)
+                used_tokens += entry_tokens
+                context["today"].append(entry)
 
-        context["token_estimate"] = used // 4
+        context["token_estimate"] = used_tokens
+        context["token_budget"] = max_tokens
         return context
 
     def export_backup(self, output_path: str | None = None) -> str:
@@ -381,7 +645,7 @@ class MemoryStore:
 
         Args:
             output_path: Destination file. Defaults to
-                ``~/.skmemory/backups/skmemory-backup-YYYY-MM-DD.json``.
+                ``~/.skcapstone/backups/skmemory-backup-YYYY-MM-DD.json``.
 
         Returns:
             str: Path to the written backup file.
@@ -396,9 +660,7 @@ class MemoryStore:
             temp = SQLiteBackend(base_path=str(self.primary.base_path))
             temp.reindex()
             return temp.export_all(output_path)
-        raise RuntimeError(
-            f"Export not supported for backend: {type(self.primary).__name__}"
-        )
+        raise RuntimeError(f"Export not supported for backend: {type(self.primary).__name__}")
 
     def import_backup(self, backup_path: str) -> int:
         """Restore memories from a JSON backup file.
@@ -414,9 +676,37 @@ class MemoryStore:
         """
         if isinstance(self.primary, SQLiteBackend):
             return self.primary.import_backup(backup_path)
-        raise RuntimeError(
-            f"Import not supported for backend: {type(self.primary).__name__}"
-        )
+        raise RuntimeError(f"Import not supported for backend: {type(self.primary).__name__}")
+
+    def list_backups(self, backup_dir: str | None = None) -> list[dict]:
+        """List all skmemory backup files, sorted newest first.
+
+        Args:
+            backup_dir: Directory to scan. Defaults to
+                ``~/.skcapstone/backups/``.
+
+        Returns:
+            list[dict]: Backup entries with ``path``, ``name``,
+                ``size_bytes``, and ``date`` keys.
+        """
+        if isinstance(self.primary, SQLiteBackend):
+            return self.primary.list_backups(backup_dir)
+        return []
+
+    def prune_backups(self, keep: int = 7, backup_dir: str | None = None) -> list[str]:
+        """Delete oldest backups, keeping only the N most recent.
+
+        Args:
+            keep: Number of backups to retain (default: 7).
+            backup_dir: Directory to prune. Defaults to
+                ``~/.skcapstone/backups/``.
+
+        Returns:
+            list[str]: Paths of deleted backup files.
+        """
+        if isinstance(self.primary, SQLiteBackend):
+            return self.primary.prune_backups(keep=keep, backup_dir=backup_dir)
+        return []
 
     def reindex(self) -> int:
         """Rebuild the SQLite index from JSON files.
@@ -442,4 +732,47 @@ class MemoryStore:
                 status["vector"] = self.vector.health_check()
             except Exception as e:
                 status["vector"] = {"ok": False, "error": str(e)}
+        if self.graph:
+            try:
+                status["graph"] = self.graph.health_check()
+            except Exception as e:
+                status["graph"] = {"ok": False, "error": str(e)}
         return status
+
+
+def _estimate_tokens(text: str) -> int:
+    """Estimate token count using word_count * 1.3 approximation.
+
+    Args:
+        text: The text to estimate.
+
+    Returns:
+        int: Approximate token count.
+    """
+    if not text:
+        return 0
+    word_count = len(text.split())
+    return int(word_count * 1.3)
+
+
+def _first_n_sentences(text: str, n: int = 2) -> str:
+    """Extract the first N sentences from text.
+
+    Args:
+        text: Source text.
+        n: Number of sentences to extract.
+
+    Returns:
+        str: The first N sentences, or the full text if fewer exist.
+    """
+    if not text:
+        return ""
+    # Split on sentence-ending punctuation followed by whitespace
+    import re
+
+    sentences = re.split(r"(?<=[.!?])\s+", text.strip())
+    result = " ".join(sentences[:n])
+    # Cap at 200 chars as a safety net
+    if len(result) > 200:
+        result = result[:197] + "..."
+    return result