@smilintux/skmemory 0.5.0 → 0.9.2

package/skmemory/backends/sqlite_backend.py

@@ -25,44 +25,71 @@ Directory layout (same as FileBackend):
 from __future__ import annotations
 
 import json
-import os
 import sqlite3
 from pathlib import Path
-from typing import Optional
 
-from ..models import EmotionalSnapshot, Memory, MemoryLayer
+from ..config import SKMEMORY_HOME
+from ..models import Memory, MemoryLayer
 from .base import BaseBackend
 
-DEFAULT_BASE_PATH = os.path.expanduser("~/.skmemory/memories")
+DEFAULT_BASE_PATH = str(SKMEMORY_HOME / "memory")
 
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS memories (
-    id TEXT PRIMARY KEY,
-    title TEXT NOT NULL,
-    layer TEXT NOT NULL,
-    role TEXT NOT NULL DEFAULT 'general',
-    tags TEXT NOT NULL DEFAULT '',
-    source TEXT NOT NULL DEFAULT 'manual',
-    source_ref TEXT NOT NULL DEFAULT '',
-    summary TEXT NOT NULL DEFAULT '',
-    content_preview TEXT NOT NULL DEFAULT '',
-    emotional_intensity REAL NOT NULL DEFAULT 0.0,
-    emotional_valence REAL NOT NULL DEFAULT 0.0,
-    emotional_labels TEXT NOT NULL DEFAULT '',
-    cloud9_achieved INTEGER NOT NULL DEFAULT 0,
-    parent_id TEXT,
-    related_ids TEXT NOT NULL DEFAULT '',
-    created_at TEXT NOT NULL,
-    updated_at TEXT NOT NULL,
-    file_path TEXT NOT NULL,
-    content_hash TEXT NOT NULL DEFAULT ''
+id TEXT PRIMARY KEY,
+title TEXT NOT NULL,
+layer TEXT NOT NULL,
+role TEXT NOT NULL DEFAULT 'general',
+tags TEXT NOT NULL DEFAULT '',
+source TEXT NOT NULL DEFAULT 'manual',
+source_ref TEXT NOT NULL DEFAULT '',
+summary TEXT NOT NULL DEFAULT '',
+content_preview TEXT NOT NULL DEFAULT '',
+emotional_intensity REAL NOT NULL DEFAULT 0.0,
+emotional_valence REAL NOT NULL DEFAULT 0.0,
+emotional_labels TEXT NOT NULL DEFAULT '',
+cloud9_achieved INTEGER NOT NULL DEFAULT 0,
+importance REAL NOT NULL DEFAULT 0.5,  -- NEW: For prioritization (0.0-1.0)
+access_count INTEGER NOT NULL DEFAULT 0,  -- NEW: LRU tracking
+last_accessed TEXT,  -- NEW: For expiration/promotion decisions
+parent_id TEXT,
+related_ids TEXT NOT NULL DEFAULT '',
+created_at TEXT NOT NULL,
+updated_at TEXT NOT NULL,
+file_path TEXT NOT NULL,
+content_hash TEXT NOT NULL DEFAULT ''
 );
 
+-- Core indexes
 CREATE INDEX IF NOT EXISTS idx_layer ON memories(layer);
 CREATE INDEX IF NOT EXISTS idx_created ON memories(created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_intensity ON memories(emotional_intensity DESC);
 CREATE INDEX IF NOT EXISTS idx_source ON memories(source);
 CREATE INDEX IF NOT EXISTS idx_parent ON memories(parent_id);
+
+-- NEW: Date-based indexes for lazy loading
+CREATE INDEX IF NOT EXISTS idx_date_layer ON memories(DATE(created_at), layer);
+CREATE INDEX IF NOT EXISTS idx_importance ON memories(importance DESC);
+CREATE INDEX IF NOT EXISTS idx_accessed ON memories(last_accessed DESC);
+CREATE INDEX IF NOT EXISTS idx_access_count ON memories(access_count DESC);
+
+-- NEW: View for active context (today + recent summaries)
+CREATE VIEW IF NOT EXISTS active_memories AS
+SELECT
+    id, title, summary, content_preview, tags, layer, created_at,
+    importance, access_count,
+    CASE
+        WHEN DATE(created_at) = CURRENT_DATE THEN 'today'
+        WHEN DATE(created_at) = DATE('now', '-1 day') THEN 'yesterday'
+        WHEN DATE(created_at) >= DATE('now', '-7 days') THEN 'week'
+        ELSE 'historical'
+    END as context_tier
+FROM memories
+WHERE created_at >= DATE('now', '-30 days')
+ORDER BY
+    context_tier,
+    importance DESC,
+    access_count DESC;
 """
 
 # Reason: 150 chars is enough for an agent to decide if it needs the full memory.
@@ -80,7 +107,7 @@ class SQLiteBackend(BaseBackend):
         self.base_path = Path(base_path)
         self._ensure_dirs()
         self._db_path = self.base_path / "index.db"
-        self._conn: Optional[sqlite3.Connection] = None
+        self._conn: sqlite3.Connection | None = None
         self._ensure_db()
 
     def _ensure_dirs(self) -> None:
@@ -105,8 +132,36 @@ class SQLiteBackend(BaseBackend):
         return self._conn
 
     def _ensure_db(self) -> None:
-        """Initialize the database schema."""
+        """Initialize the database schema, migrating old tables if needed."""
         conn = self._get_conn()
+
+        # Migrate: add columns that may be missing from older schemas.
+        # We check pragma_table_info first so this is safe on fresh DBs too.
+        row = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='memories'"
+        ).fetchone()
+        if row is not None:
+            existing = {r[1] for r in conn.execute("PRAGMA table_info(memories)").fetchall()}
+            migrations = [
+                (
+                    "importance",
+                    "ALTER TABLE memories ADD COLUMN importance REAL NOT NULL DEFAULT 0.5",
+                ),
+                (
+                    "access_count",
+                    "ALTER TABLE memories ADD COLUMN access_count INTEGER NOT NULL DEFAULT 0",
+                ),
+                ("last_accessed", "ALTER TABLE memories ADD COLUMN last_accessed TEXT"),
+            ]
+            for col, ddl in migrations:
+                if col not in existing:
+                    conn.execute(ddl)
+            conn.commit()
+
+            # Drop stale view so CREATE VIEW IF NOT EXISTS picks up new columns.
+            conn.execute("DROP VIEW IF EXISTS active_memories")
+            conn.commit()
+
         conn.executescript(_SCHEMA)
         conn.commit()
 
@@ -121,7 +176,7 @@ class SQLiteBackend(BaseBackend):
         """
         return self.base_path / memory.layer.value / f"{memory.id}.json"
 
-    def _find_file(self, memory_id: str) -> Optional[Path]:
+    def _find_file(self, memory_id: str) -> Path | None:
         """Locate a memory file using the index first, then fallback.
 
         Args:
@@ -131,9 +186,7 @@ class SQLiteBackend(BaseBackend):
             Optional[Path]: Path to the file if found.
         """
         conn = self._get_conn()
-        row = conn.execute(
-            "SELECT file_path FROM memories WHERE id = ?", (memory_id,)
-        ).fetchone()
+        row = conn.execute("SELECT file_path FROM memories WHERE id = ?", (memory_id,)).fetchone()
         if row:
             path = Path(row["file_path"])
             if path.exists():
@@ -211,16 +264,14 @@ class SQLiteBackend(BaseBackend):
             "content_preview": row["content_preview"],
             "emotional_intensity": row["emotional_intensity"],
             "emotional_valence": row["emotional_valence"],
-            "emotional_labels": [
-                l for l in row["emotional_labels"].split(",") if l
-            ],
+            "emotional_labels": [lbl for lbl in row["emotional_labels"].split(",") if lbl],
             "cloud9_achieved": bool(row["cloud9_achieved"]),
             "created_at": row["created_at"],
             "parent_id": row["parent_id"],
             "related_ids": [r for r in row["related_ids"].split(",") if r],
         }
 
-    def _row_to_memory(self, row: sqlite3.Row) -> Optional[Memory]:
+    def _row_to_memory(self, row: sqlite3.Row) -> Memory | None:
         """Load the full Memory object from disk using the index path.
 
         Args:
@@ -256,7 +307,7 @@ class SQLiteBackend(BaseBackend):
         self._index_memory(memory, path)
         return memory.id
 
-    def load(self, memory_id: str) -> Optional[Memory]:
+    def load(self, memory_id: str) -> Memory | None:
         """Load a memory by ID, using the index for fast lookup.
 
         Args:
@@ -297,8 +348,8 @@ class SQLiteBackend(BaseBackend):
 
     def list_memories(
         self,
-        layer: Optional[MemoryLayer] = None,
-        tags: Optional[list[str]] = None,
+        layer: MemoryLayer | None = None,
+        tags: list[str] | None = None,
         limit: int = 50,
     ) -> list[Memory]:
         """List memories using the index for filtering, loading full objects.
@@ -328,8 +379,7 @@ class SQLiteBackend(BaseBackend):
         params.append(limit)
 
         rows = conn.execute(
-            f"SELECT * FROM memories WHERE {where} "
-            f"ORDER BY created_at DESC LIMIT ?",
+            f"SELECT * FROM memories WHERE {where} ORDER BY created_at DESC LIMIT ?",
             params,
         ).fetchall()
 
@@ -342,8 +392,8 @@ class SQLiteBackend(BaseBackend):
 
     def list_summaries(
         self,
-        layer: Optional[MemoryLayer] = None,
-        tags: Optional[list[str]] = None,
+        layer: MemoryLayer | None = None,
+        tags: list[str] | None = None,
         limit: int = 50,
         min_intensity: float = 0.0,
         order_by: str = "created_at",
@@ -383,7 +433,19 @@ class SQLiteBackend(BaseBackend):
 
         where = " AND ".join(conditions) if conditions else "1=1"
 
-        if order_by == "emotional_intensity":
+        if order_by == "recency_weighted_intensity":
+            # Combine intensity with recency: recent high-intensity memories
+            # score higher than old high-intensity ones.
+            # julianday('now') - julianday(created_at) gives days ago.
+            # Decay: halve the recency bonus every 7 days.
+            order = (
+                "(emotional_intensity + "
+                "CASE WHEN julianday('now') - julianday(created_at) < 1 THEN 5.0 "
+                "WHEN julianday('now') - julianday(created_at) < 3 THEN 3.0 "
+                "WHEN julianday('now') - julianday(created_at) < 7 THEN 1.5 "
+                "ELSE 0.0 END) DESC"
+            )
+        elif order_by == "emotional_intensity":
             order = "emotional_intensity DESC"
         else:
             order = "created_at DESC"
@@ -400,7 +462,9 @@ class SQLiteBackend(BaseBackend):
     def search_text(self, query: str, limit: int = 10) -> list[Memory]:
         """Search memories using the SQLite index (title, summary, preview).
 
-        Falls back to full file scan only if the index doesn't find matches.
+        For multi-word queries, tries AND first (all words must match).
+        If AND returns nothing, falls back to OR (any word matches),
+        ranked by how many query words each memory contains.
 
         Args:
             query: Search string.
@@ -410,19 +474,63 @@ class SQLiteBackend(BaseBackend):
             list[Memory]: Matching memories.
         """
         conn = self._get_conn()
-        query_param = f"%{query}%"
+        words = query.split()
+
+        if not words:
+            return []
+
+        cols = ["title", "summary", "content_preview", "tags"]
+
+        def _word_clause(word: str) -> str:
+            return "(" + " OR ".join(f"{c} LIKE ?" for c in cols) + ")"
+
+        def _word_params(word: str) -> list[str]:
+            pattern = f"%{word}%"
+            return [pattern] * len(cols)
+
+        # Try AND first: all words must match
+        and_clauses = [_word_clause(w) for w in words]
+        and_params: list[str] = []
+        for w in words:
+            and_params.extend(_word_params(w))
+        and_params.append(str(limit))
 
         rows = conn.execute(
-            """
+            f"""
             SELECT * FROM memories
-            WHERE title LIKE ? OR summary LIKE ? OR content_preview LIKE ?
-                  OR tags LIKE ?
+            WHERE {" AND ".join(and_clauses)}
             ORDER BY created_at DESC
             LIMIT ?
             """,
-            (query_param, query_param, query_param, query_param, limit),
+            and_params,
         ).fetchall()
 
+        if not rows and len(words) > 1:
+            # Fall back to OR: any word matches, ranked by match count
+            or_clauses = [_word_clause(w) for w in words]
+            # Use SUM of CASE expressions to count matching words
+            score_expr = " + ".join(
+                f"CASE WHEN {_word_clause(w)} THEN 1 ELSE 0 END" for w in words
+            )
+            or_params: list[str] = []
+            # params for WHERE (OR)
+            for w in words:
+                or_params.extend(_word_params(w))
+            # params for ORDER BY score (same patterns again)
+            for w in words:
+                or_params.extend(_word_params(w))
+            or_params.append(str(limit))
+
+            rows = conn.execute(
+                f"""
+                SELECT * FROM memories
+                WHERE {" OR ".join(or_clauses)}
+                ORDER BY ({score_expr}) DESC, created_at DESC
+                LIMIT ?
+                """,
+                or_params,
+            ).fetchall()
+
         results = []
         for row in rows:
             mem = self._row_to_memory(row)
@@ -446,9 +554,7 @@ class SQLiteBackend(BaseBackend):
         results: list[dict] = []
 
         # Reason: seed the frontier from the starting node's relationships
-        row = conn.execute(
-            "SELECT * FROM memories WHERE id = ?", (memory_id,)
-        ).fetchone()
+        row = conn.execute("SELECT * FROM memories WHERE id = ?", (memory_id,)).fetchone()
         if row is None:
             return results
 
@@ -464,20 +570,14 @@ class SQLiteBackend(BaseBackend):
                     continue
                 visited.add(mid)
 
-                neighbor = conn.execute(
-                    "SELECT * FROM memories WHERE id = ?", (mid,)
-                ).fetchone()
+                neighbor = conn.execute("SELECT * FROM memories WHERE id = ?", (mid,)).fetchone()
                 if neighbor is None:
                     continue
 
                 results.append(self._row_to_memory_summary(neighbor))
 
-                child_related = [
-                    r for r in neighbor["related_ids"].split(",") if r
-                ]
-                next_frontier.extend(
-                    r for r in child_related if r not in visited
-                )
+                child_related = [r for r in neighbor["related_ids"].split(",") if r]
+                next_frontier.extend(r for r in child_related if r not in visited)
                 if neighbor["parent_id"] and neighbor["parent_id"] not in visited:
                     next_frontier.append(neighbor["parent_id"])
 
@@ -485,28 +585,80 @@ class SQLiteBackend(BaseBackend):
 
         return results
 
-    def export_all(self, output_path: Optional[str] = None) -> str:
+    def list_backups(self, backup_dir: str | None = None) -> list[dict]:
+        """List all skmemory backup files, sorted newest first.
+
+        Args:
+            backup_dir: Directory to scan. Defaults to
+                ``<base_path>/../backups/``.
+
+        Returns:
+            list[dict]: Backup entries, newest first. Each entry has:
+                ``path``, ``name``, ``size_bytes``, ``date``.
+        """
+        bdir = self.base_path.parent / "backups" if backup_dir is None else Path(backup_dir)
+
+        if not bdir.exists():
+            return []
+
+        entries = []
+        for f in sorted(bdir.glob("skmemory-backup-*.json"), reverse=True):
+            entries.append(
+                {
+                    "path": str(f),
+                    "name": f.name,
+                    "size_bytes": f.stat().st_size,
+                    "date": f.stem.replace("skmemory-backup-", ""),
+                }
+            )
+        return entries
+
+    def prune_backups(self, keep: int = 7, backup_dir: str | None = None) -> list[str]:
+        """Delete oldest backups, retaining only the N most recent.
+
+        Args:
+            keep: Number of most-recent backups to keep (default: 7).
+            backup_dir: Directory to prune. Defaults to
+                ``<base_path>/../backups/``.
+
+        Returns:
+            list[str]: Paths of the deleted backup files.
+        """
+        backups = self.list_backups(backup_dir)
+        to_delete = backups[keep:]  # list is already newest-first
+        deleted: list[str] = []
+        for entry in to_delete:
+            try:
+                Path(entry["path"]).unlink()
+                deleted.append(entry["path"])
+            except OSError:
+                pass
+        return deleted
+
+    def export_all(self, output_path: str | None = None) -> str:
         """Export all memories as a single JSON file for backup.
 
         Reads every JSON file on disk and bundles them into one
         git-friendly backup. One file per day by default (overwrites
-        same-day exports).
+        same-day exports). When using the default backup directory,
+        automatically prunes to keep the last 7 daily backups.
 
         Args:
             output_path: Where to write the backup. If None, uses
-                ``~/.skmemory/backups/skmemory-backup-YYYY-MM-DD.json``.
+                ``~/.skcapstone/backups/skmemory-backup-YYYY-MM-DD.json``
+                and triggers automatic rotation (keep last 7).
 
         Returns:
             str: Path to the written backup file.
         """
         from datetime import date as _date
 
+        _auto_rotate = output_path is None
+
         if output_path is None:
             backup_dir = self.base_path.parent / "backups"
             backup_dir.mkdir(parents=True, exist_ok=True)
-            output_path = str(
-                backup_dir / f"skmemory-backup-{_date.today().isoformat()}.json"
-            )
+            output_path = str(backup_dir / f"skmemory-backup-{_date.today().isoformat()}.json")
 
         memories: list[dict] = []
         for layer in MemoryLayer:
@@ -520,8 +672,10 @@ class SQLiteBackend(BaseBackend):
                 except (json.JSONDecodeError, Exception):
                     continue
 
+        from datetime import datetime as _dt
+        from datetime import timezone as _tz
+
         from .. import __version__
-        from datetime import datetime as _dt, timezone as _tz
 
         payload = {
             "skmemory_version": __version__,
@@ -531,9 +685,11 @@ class SQLiteBackend(BaseBackend):
             "memories": memories,
         }
 
-        Path(output_path).write_text(
-            json.dumps(payload, indent=2, default=str), encoding="utf-8"
-        )
+        Path(output_path).write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
+
+        if _auto_rotate:
+            self.prune_backups(keep=7)
+
         return output_path
 
     def import_backup(self, backup_path: str) -> int:
@@ -559,9 +715,7 @@ class SQLiteBackend(BaseBackend):
         data = json.loads(path.read_text(encoding="utf-8"))
 
         if "memories" not in data or not isinstance(data["memories"], list):
-            raise ValueError(
-                "Invalid backup file: missing 'memories' array"
-            )
+            raise ValueError("Invalid backup file: missing 'memories' array")
 
         count = 0
         for mem_data in data["memories"]: