@smilintux/skmemory 0.5.0 → 0.7.2

package/skmemory/backends/{qdrant_backend.py → skvector_backend.py}

@@ -1,12 +1,12 @@
 """
-Qdrant vector search backend (Level 2).
+SKVector — semantic vector search backend (Level 1).
 
-Enables semantic memory recall: instead of exact text matching,
-find memories by *meaning*. "That conversation where we felt connected"
-finds the right memory even if those exact words aren't in it.
+Powered by Qdrant. Enables semantic memory recall: instead of exact text
+matching, find memories by *meaning*. "That conversation where we felt
+connected" finds the right memory even if those exact words aren't in it.
 
 Requires:
-    pip install qdrant-client sentence-transformers
+    pip install skmemory[skvector]
 
 Qdrant free tier: 1GB storage, 256MB RAM -- enough for thousands of memories.
 SaaS endpoint: https://cloud.qdrant.io (free cluster available).
@@ -14,6 +14,7 @@ SaaS endpoint: https://cloud.qdrant.io (free cluster available).
 
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 from typing import Optional
@@ -28,15 +29,36 @@ EMBEDDING_MODEL = "all-MiniLM-L6-v2"
 VECTOR_DIM = 384
 
 
-class QdrantBackend(BaseBackend):
-    """Qdrant-powered semantic memory search.
+def _extract_status_code(exc: Exception, unexpected_cls: type | None) -> int | None:
+    """Pull an HTTP status code from a qdrant-client exception.
 
-    Stores memory embeddings in Qdrant for vector similarity search.
-    Falls back gracefully if Qdrant or the embedding model is unavailable.
+    Works across qdrant-client versions: checks ``status_code`` attr first,
+    then falls back to the string representation for patterns like ``401``.
+    """
+    code = getattr(exc, "status_code", None)
+    if code is not None:
+        return int(code)
+    if unexpected_cls is not None and isinstance(exc, unexpected_cls):
+        code = getattr(exc, "status_code", None)
+        if code is not None:
+            return int(code)
+    text = str(exc)
+    for candidate in (401, 403):
+        if str(candidate) in text:
+            return candidate
+    return None
+
+
+class SKVectorBackend(BaseBackend):
+    """SKVector — semantic memory search (powered by Qdrant).
+
+    Stores memory embeddings for vector similarity search.
+    Falls back gracefully if the vector engine or the embedding model
+    is unavailable.
 
     Args:
-        url: Qdrant server URL (default: localhost:6333).
-        api_key: API key for Qdrant Cloud.
+        url: SKVector server URL (default: localhost:6333).
+        api_key: API key for cloud-hosted SKVector.
         collection: Collection name (default: 'skmemory').
         embedding_model: Sentence-transformers model name.
     """
@@ -55,6 +77,7 @@ class QdrantBackend(BaseBackend):
         self._client = None
         self._embedder = None
         self._initialized = False
+        self._last_error: str | None = None
 
     def _ensure_initialized(self) -> bool:
         """Lazy-initialize Qdrant client and embedding model.
@@ -69,7 +92,7 @@ class QdrantBackend(BaseBackend):
             from qdrant_client import QdrantClient
             from qdrant_client.models import Distance, VectorParams
         except ImportError:
-            logger.warning("qdrant-client not installed: pip install qdrant-client")
+            logger.warning("qdrant-client not installed: pip install skmemory[skvector]")
             return False
 
         try:
@@ -77,10 +100,19 @@ class QdrantBackend(BaseBackend):
         except ImportError:
             logger.warning(
                 "sentence-transformers not installed: "
-                "pip install sentence-transformers"
+                "pip install skmemory[skvector]"
             )
             return False
 
+        try:
+            from qdrant_client.http.exceptions import (
+                ResponseHandlingException,
+                UnexpectedResponse,
+            )
+        except ImportError:
+            UnexpectedResponse = None
+            ResponseHandlingException = None
+
         try:
             self._client = QdrantClient(url=self.url, api_key=self.api_key)
             collections = [c.name for c in self._client.get_collections().collections]
@@ -100,7 +132,20 @@ class QdrantBackend(BaseBackend):
             return True
 
         except Exception as e:
-            logger.warning("Qdrant initialization failed: %s", e)
+            status = _extract_status_code(e, UnexpectedResponse)
+            if status in (401, 403):
+                hint = (
+                    "SKVector authentication failed (HTTP %d). "
+                    "Check your API key:\n"
+                    "  - CLI:  --skvector-key YOUR_KEY\n"
+                    "  - Env:  SKMEMORY_SKVECTOR_KEY=YOUR_KEY\n"
+                    "  - Code: SKVectorBackend(url=..., api_key='YOUR_KEY')"
+                )
+                logger.error(hint, status)
+                self._last_error = hint % status
+            else:
+                logger.warning("SKVector initialization failed: %s", e)
+                self._last_error = str(e)
             return False
 
     def _embed(self, text: str) -> list[float]:
@@ -155,8 +200,11 @@ class QdrantBackend(BaseBackend):
         if not embedding:
             return memory.id
 
+        # Use memory.id hash as Qdrant point ID (not content_hash which
+        # would collide if two memories have identical content).
+        point_id = int(hashlib.sha256(memory.id.encode()).hexdigest()[:15], 16)
         point = PointStruct(
-            id=memory.content_hash(),
+            id=point_id,
             vector=embedding,
             payload=self._memory_to_payload(memory),
         )
@@ -167,6 +215,10 @@ class QdrantBackend(BaseBackend):
         )
         return memory.id
 
+    def _id_to_point_id(self, memory_id: str) -> int:
+        """Convert a memory ID string to a deterministic Qdrant integer point ID."""
+        return int(hashlib.sha256(memory_id.encode()).hexdigest()[:15], 16)
+
     def load(self, memory_id: str) -> Optional[Memory]:
         """Retrieve a memory by ID from Qdrant.
 
@@ -175,77 +227,52 @@ class QdrantBackend(BaseBackend):
 
         Returns:
             Optional[Memory]: The memory if found.
-
-        Note:
-            Qdrant uses content hashes as point IDs, so this does
-            a scroll+filter. For direct ID lookup, use the file backend.
         """
         if not self._ensure_initialized():
             return None
 
-        from qdrant_client.models import FieldCondition, Filter, MatchValue
-
-        results = self._client.scroll(
-            collection_name=self.collection,
-            scroll_filter=Filter(
-                must=[
-                    FieldCondition(
-                        key="memory_json",
-                        match=MatchValue(value=memory_id),
-                    )
-                ]
-            ),
-            limit=1,
-        )
-
-        points = results[0] if results else []
-        if not points:
-            return None
-
         try:
+            points = self._client.retrieve(
+                collection_name=self.collection,
+                ids=[self._id_to_point_id(memory_id)],
+                with_payload=True,
+            )
+            if not points:
+                return None
             return Memory.model_validate_json(points[0].payload["memory_json"])
         except Exception:
             return None
 
     def delete(self, memory_id: str) -> bool:
-        """Remove a memory from Qdrant by scrolling for it.
+        """Remove a memory from Qdrant by its deterministic point ID.
+
+        Returns False if the memory was not found.
 
         Args:
             memory_id: The memory identifier.
 
         Returns:
-            bool: True if something was deleted.
+            bool: True if the memory existed and was deleted, False otherwise.
         """
         if not self._ensure_initialized():
             return False
 
-        from qdrant_client.models import FieldCondition, Filter, MatchValue
-
-        # Reason: Qdrant doesn't support delete-by-payload natively,
-        # so we scroll to find the point ID then delete by point ID.
-        results = self._client.scroll(
-            collection_name=self.collection,
-            scroll_filter=Filter(
-                must=[
-                    FieldCondition(
-                        key="memory_json",
-                        match=MatchValue(value=memory_id),
-                    )
-                ]
-            ),
-            limit=1,
-        )
-
-        points = results[0] if results else []
-        if not points:
+        try:
+            points = self._client.retrieve(
+                collection_name=self.collection,
+                ids=[self._id_to_point_id(memory_id)],
+                with_payload=False,
+            )
+            if not points:
+                return False
+            self._client.delete(
+                collection_name=self.collection,
+                points_selector=[self._id_to_point_id(memory_id)],
+            )
+            return True
+        except Exception:
             return False
 
-        self._client.delete(
-            collection_name=self.collection,
-            points_selector=[points[0].id],
-        )
-        return True
-
     def list_memories(
         self,
         layer: Optional[MemoryLayer] = None,
@@ -340,17 +367,20 @@ class QdrantBackend(BaseBackend):
             dict: Status with connection and collection info.
         """
         if not self._ensure_initialized():
+            error_msg = self._last_error or (
+                "Not initialized (missing dependencies or connection failed)"
+            )
             return {
                 "ok": False,
-                "backend": "QdrantBackend",
-                "error": "Not initialized (missing dependencies or connection failed)",
+                "backend": "SKVectorBackend",
+                "error": error_msg,
             }
 
         try:
             info = self._client.get_collection(self.collection)
             return {
                 "ok": True,
-                "backend": "QdrantBackend",
+                "backend": "SKVectorBackend",
                 "url": self.url,
                 "collection": self.collection,
                 "points_count": info.points_count,
@@ -359,6 +389,6 @@ class QdrantBackend(BaseBackend):
         except Exception as e:
             return {
                 "ok": False,
-                "backend": "QdrantBackend",
+                "backend": "SKVectorBackend",
                 "error": str(e),
             }

package/skmemory/backends/sqlite_backend.py

@@ -30,39 +30,68 @@ import sqlite3
 from pathlib import Path
 from typing import Optional
 
+from ..config import SKMEMORY_HOME
 from ..models import EmotionalSnapshot, Memory, MemoryLayer
 from .base import BaseBackend
 
-DEFAULT_BASE_PATH = os.path.expanduser("~/.skmemory/memories")
+DEFAULT_BASE_PATH = str(SKMEMORY_HOME)
 
 _SCHEMA = """
 CREATE TABLE IF NOT EXISTS memories (
-    id TEXT PRIMARY KEY,
-    title TEXT NOT NULL,
-    layer TEXT NOT NULL,
-    role TEXT NOT NULL DEFAULT 'general',
-    tags TEXT NOT NULL DEFAULT '',
-    source TEXT NOT NULL DEFAULT 'manual',
-    source_ref TEXT NOT NULL DEFAULT '',
-    summary TEXT NOT NULL DEFAULT '',
-    content_preview TEXT NOT NULL DEFAULT '',
-    emotional_intensity REAL NOT NULL DEFAULT 0.0,
-    emotional_valence REAL NOT NULL DEFAULT 0.0,
-    emotional_labels TEXT NOT NULL DEFAULT '',
-    cloud9_achieved INTEGER NOT NULL DEFAULT 0,
-    parent_id TEXT,
-    related_ids TEXT NOT NULL DEFAULT '',
-    created_at TEXT NOT NULL,
-    updated_at TEXT NOT NULL,
-    file_path TEXT NOT NULL,
-    content_hash TEXT NOT NULL DEFAULT ''
+id TEXT PRIMARY KEY,
+title TEXT NOT NULL,
+layer TEXT NOT NULL,
+role TEXT NOT NULL DEFAULT 'general',
+tags TEXT NOT NULL DEFAULT '',
+source TEXT NOT NULL DEFAULT 'manual',
+source_ref TEXT NOT NULL DEFAULT '',
+summary TEXT NOT NULL DEFAULT '',
+content_preview TEXT NOT NULL DEFAULT '',
+emotional_intensity REAL NOT NULL DEFAULT 0.0,
+emotional_valence REAL NOT NULL DEFAULT 0.0,
+emotional_labels TEXT NOT NULL DEFAULT '',
+cloud9_achieved INTEGER NOT NULL DEFAULT 0,
+importance REAL NOT NULL DEFAULT 0.5,  -- NEW: For prioritization (0.0-1.0)
+access_count INTEGER NOT NULL DEFAULT 0,  -- NEW: LRU tracking
+last_accessed TEXT,  -- NEW: For expiration/promotion decisions
+parent_id TEXT,
+related_ids TEXT NOT NULL DEFAULT '',
+created_at TEXT NOT NULL,
+updated_at TEXT NOT NULL,
+file_path TEXT NOT NULL,
+content_hash TEXT NOT NULL DEFAULT ''
 );
 
+-- Core indexes
 CREATE INDEX IF NOT EXISTS idx_layer ON memories(layer);
 CREATE INDEX IF NOT EXISTS idx_created ON memories(created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_intensity ON memories(emotional_intensity DESC);
 CREATE INDEX IF NOT EXISTS idx_source ON memories(source);
 CREATE INDEX IF NOT EXISTS idx_parent ON memories(parent_id);
+
+-- NEW: Date-based indexes for lazy loading
+CREATE INDEX IF NOT EXISTS idx_date_layer ON memories(DATE(created_at), layer);
+CREATE INDEX IF NOT EXISTS idx_importance ON memories(importance DESC);
+CREATE INDEX IF NOT EXISTS idx_accessed ON memories(last_accessed DESC);
+CREATE INDEX IF NOT EXISTS idx_access_count ON memories(access_count DESC);
+
+-- NEW: View for active context (today + recent summaries)
+CREATE VIEW IF NOT EXISTS active_memories AS
+SELECT 
+    id, title, summary, content_preview, tags, layer, created_at,
+    importance, access_count,
+    CASE 
+        WHEN DATE(created_at) = CURRENT_DATE THEN 'today'
+        WHEN DATE(created_at) = DATE('now', '-1 day') THEN 'yesterday'
+        WHEN DATE(created_at) >= DATE('now', '-7 days') THEN 'week'
+        ELSE 'historical'
+    END as context_tier
+FROM memories
+WHERE created_at >= DATE('now', '-30 days')
+ORDER BY 
+    context_tier,
+    importance DESC,
+    access_count DESC;
 """
 
 # Reason: 150 chars is enough for an agent to decide if it needs the full memory.
@@ -131,9 +160,7 @@ class SQLiteBackend(BaseBackend):
             Optional[Path]: Path to the file if found.
         """
         conn = self._get_conn()
-        row = conn.execute(
-            "SELECT file_path FROM memories WHERE id = ?", (memory_id,)
-        ).fetchone()
+        row = conn.execute("SELECT file_path FROM memories WHERE id = ?", (memory_id,)).fetchone()
         if row:
             path = Path(row["file_path"])
             if path.exists():
@@ -211,9 +238,7 @@ class SQLiteBackend(BaseBackend):
             "content_preview": row["content_preview"],
             "emotional_intensity": row["emotional_intensity"],
             "emotional_valence": row["emotional_valence"],
-            "emotional_labels": [
-                l for l in row["emotional_labels"].split(",") if l
-            ],
+            "emotional_labels": [l for l in row["emotional_labels"].split(",") if l],
             "cloud9_achieved": bool(row["cloud9_achieved"]),
             "created_at": row["created_at"],
             "parent_id": row["parent_id"],
@@ -328,8 +353,7 @@ class SQLiteBackend(BaseBackend):
         params.append(limit)
 
         rows = conn.execute(
-            f"SELECT * FROM memories WHERE {where} "
-            f"ORDER BY created_at DESC LIMIT ?",
+            f"SELECT * FROM memories WHERE {where} ORDER BY created_at DESC LIMIT ?",
             params,
         ).fetchall()
 
@@ -446,9 +470,7 @@ class SQLiteBackend(BaseBackend):
         results: list[dict] = []
 
         # Reason: seed the frontier from the starting node's relationships
-        row = conn.execute(
-            "SELECT * FROM memories WHERE id = ?", (memory_id,)
-        ).fetchone()
+        row = conn.execute("SELECT * FROM memories WHERE id = ?", (memory_id,)).fetchone()
         if row is None:
             return results
 
@@ -464,20 +486,14 @@ class SQLiteBackend(BaseBackend):
                     continue
                 visited.add(mid)
 
-                neighbor = conn.execute(
-                    "SELECT * FROM memories WHERE id = ?", (mid,)
-                ).fetchone()
+                neighbor = conn.execute("SELECT * FROM memories WHERE id = ?", (mid,)).fetchone()
                 if neighbor is None:
                     continue
 
                 results.append(self._row_to_memory_summary(neighbor))
 
-                child_related = [
-                    r for r in neighbor["related_ids"].split(",") if r
-                ]
-                next_frontier.extend(
-                    r for r in child_related if r not in visited
-                )
+                child_related = [r for r in neighbor["related_ids"].split(",") if r]
+                next_frontier.extend(r for r in child_related if r not in visited)
                 if neighbor["parent_id"] and neighbor["parent_id"] not in visited:
                     next_frontier.append(neighbor["parent_id"])
 
@@ -485,28 +501,83 @@ class SQLiteBackend(BaseBackend):
 
         return results
 
+    def list_backups(self, backup_dir: Optional[str] = None) -> list[dict]:
+        """List all skmemory backup files, sorted newest first.
+
+        Args:
+            backup_dir: Directory to scan. Defaults to
+                ``<base_path>/../backups/``.
+
+        Returns:
+            list[dict]: Backup entries, newest first. Each entry has:
+                ``path``, ``name``, ``size_bytes``, ``date``.
+        """
+        if backup_dir is None:
+            bdir = self.base_path.parent / "backups"
+        else:
+            bdir = Path(backup_dir)
+
+        if not bdir.exists():
+            return []
+
+        entries = []
+        for f in sorted(bdir.glob("skmemory-backup-*.json"), reverse=True):
+            entries.append(
+                {
+                    "path": str(f),
+                    "name": f.name,
+                    "size_bytes": f.stat().st_size,
+                    "date": f.stem.replace("skmemory-backup-", ""),
+                }
+            )
+        return entries
+
+    def prune_backups(self, keep: int = 7, backup_dir: Optional[str] = None) -> list[str]:
+        """Delete oldest backups, retaining only the N most recent.
+
+        Args:
+            keep: Number of most-recent backups to keep (default: 7).
+            backup_dir: Directory to prune. Defaults to
+                ``<base_path>/../backups/``.
+
+        Returns:
+            list[str]: Paths of the deleted backup files.
+        """
+        backups = self.list_backups(backup_dir)
+        to_delete = backups[keep:]  # list is already newest-first
+        deleted: list[str] = []
+        for entry in to_delete:
+            try:
+                Path(entry["path"]).unlink()
+                deleted.append(entry["path"])
+            except OSError:
+                pass
+        return deleted
+
     def export_all(self, output_path: Optional[str] = None) -> str:
         """Export all memories as a single JSON file for backup.
 
         Reads every JSON file on disk and bundles them into one
         git-friendly backup. One file per day by default (overwrites
-        same-day exports).
+        same-day exports). When using the default backup directory,
+        automatically prunes to keep the last 7 daily backups.
 
         Args:
             output_path: Where to write the backup. If None, uses
-                ``~/.skmemory/backups/skmemory-backup-YYYY-MM-DD.json``.
+                ``~/.skcapstone/backups/skmemory-backup-YYYY-MM-DD.json``
+                and triggers automatic rotation (keep last 7).
 
         Returns:
             str: Path to the written backup file.
         """
         from datetime import date as _date
 
+        _auto_rotate = output_path is None
+
         if output_path is None:
             backup_dir = self.base_path.parent / "backups"
             backup_dir.mkdir(parents=True, exist_ok=True)
-            output_path = str(
-                backup_dir / f"skmemory-backup-{_date.today().isoformat()}.json"
-            )
+            output_path = str(backup_dir / f"skmemory-backup-{_date.today().isoformat()}.json")
 
         memories: list[dict] = []
         for layer in MemoryLayer:
@@ -531,9 +602,11 @@ class SQLiteBackend(BaseBackend):
             "memories": memories,
         }
 
-        Path(output_path).write_text(
-            json.dumps(payload, indent=2, default=str), encoding="utf-8"
-        )
+        Path(output_path).write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
+
+        if _auto_rotate:
+            self.prune_backups(keep=7)
+
         return output_path
 
     def import_backup(self, backup_path: str) -> int:
@@ -559,9 +632,7 @@ class SQLiteBackend(BaseBackend):
         data = json.loads(path.read_text(encoding="utf-8"))
 
         if "memories" not in data or not isinstance(data["memories"], list):
-            raise ValueError(
-                "Invalid backup file: missing 'memories' array"
-            )
+            raise ValueError("Invalid backup file: missing 'memories' array")
 
         count = 0
         for mem_data in data["memories"]: