@smilintux/skmemory 0.5.0 → 0.7.2

package/skmemory/config.py

@@ -0,0 +1,173 @@
+"""
+SKMemory configuration persistence.
+
+Manages ``~/.skcapstone/agents/{agent_name}/config/skmemory.yaml``
+so backend URLs and setup state persist across CLI invocations.
+
+Resolution order:
+CLI args > env vars > config file > None
+
+Now supports multiple agents via ~/.skcapstone/agents/{agent_name}/
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+import yaml
+from pydantic import BaseModel, Field
+
+from .agents import AGENTS_BASE_DIR, get_agent_paths
+
+# Dynamic agent-aware paths
+# Uses ~/.skcapstone/agents/{active_agent}/ based on SKMEMORY_AGENT env var
+# Falls back to first non-template agent, or creates from template
+try:
+    default_paths = get_agent_paths()
+    SKMEMORY_HOME = default_paths["base"]
+    CONFIG_DIR = default_paths["config"]
+    CONFIG_PATH = default_paths["config_yaml"]
+except ValueError:
+    # Fallback if no agents exist — use platform-aware AGENTS_BASE_DIR
+    SKMEMORY_HOME = AGENTS_BASE_DIR / "lumina-template"
+    CONFIG_DIR = SKMEMORY_HOME / "config"
+    CONFIG_PATH = CONFIG_DIR / "skmemory.yaml"
+
+
+class EndpointConfig(BaseModel):
+    """A single backend endpoint with role and optional Tailscale IP."""
+
+    url: str
+    role: str = "primary"  # primary | replica
+    tailscale_ip: str = ""  # optional, for display
+
+
+class SKMemoryConfig(BaseModel):
+    """Persistent configuration for SKMemory backends."""
+
+    skvector_url: Optional[str] = None
+    skvector_key: Optional[str] = None
+    skgraph_url: Optional[str] = None
+    backends_enabled: list[str] = Field(default_factory=list)
+    docker_compose_file: Optional[str] = None
+    setup_completed_at: Optional[str] = None
+
+    # Multi-endpoint HA support
+    skvector_endpoints: list[EndpointConfig] = Field(default_factory=list)
+    skgraph_endpoints: list[EndpointConfig] = Field(default_factory=list)
+    routing_strategy: str = "failover"
+    heartbeat_discovery: bool = False
+
+
+def load_config(path: Path = CONFIG_PATH) -> Optional[SKMemoryConfig]:
+    """Load configuration from YAML.
+
+    Args:
+        path: Path to the config file.
+
+    Returns:
+        SKMemoryConfig if the file exists and is valid, None otherwise.
+    """
+    if not path.exists():
+        return None
+
+    try:
+        with open(path) as f:
+            data = yaml.safe_load(f)
+        if not isinstance(data, dict):
+            return None
+        return SKMemoryConfig(**data)
+    except Exception:
+        return None
+
+
+def save_config(config: SKMemoryConfig, path: Path = CONFIG_PATH) -> Path:
+    """Write configuration to YAML, creating the directory if needed.
+
+    Args:
+        config: The configuration to persist.
+        path: Destination path.
+
+    Returns:
+        The path written to.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w") as f:
+        yaml.safe_dump(
+            config.model_dump(exclude_none=True),
+            f,
+            default_flow_style=False,
+            sort_keys=False,
+        )
+    return path
+
+
+def merge_env_and_config(
+    cli_skvector_url: Optional[str] = None,
+    cli_skvector_key: Optional[str] = None,
+    cli_skgraph_url: Optional[str] = None,
+) -> tuple[Optional[str], Optional[str], Optional[str]]:
+    """Resolve backend URLs with precedence: CLI > env > config > None.
+
+    Args:
+        cli_skvector_url: URL passed via ``--skvector-url``.
+        cli_skvector_key: Key passed via ``--skvector-key``.
+        cli_skgraph_url: URL passed via ``--skgraph-url`` (future).
+
+    Returns:
+        Tuple of (skvector_url, skvector_key, skgraph_url).
+    """
+    cfg = load_config()
+
+    skvector_url = (
+        cli_skvector_url
+        or os.environ.get("SKMEMORY_SKVECTOR_URL")
+        or (cfg.skvector_url if cfg else None)
+    )
+    skvector_key = (
+        cli_skvector_key
+        or os.environ.get("SKMEMORY_SKVECTOR_KEY")
+        or (cfg.skvector_key if cfg else None)
+    )
+    skgraph_url = (
+        cli_skgraph_url
+        or os.environ.get("SKMEMORY_SKGRAPH_URL")
+        or (cfg.skgraph_url if cfg else None)
+    )
+
+    return skvector_url, skvector_key, skgraph_url
+
+
+def build_endpoint_list(
+    single_url: Optional[str],
+    endpoints: list[EndpointConfig],
+    default_role: str = "primary",
+) -> list[EndpointConfig]:
+    """Merge a single URL and an endpoints list into a unified list.
+
+    Backward compatibility bridge: if no endpoints are configured but a
+    single URL exists, it becomes the sole endpoint.  If both exist, the
+    endpoints list takes precedence and the single URL is prepended only
+    if it isn't already present.
+
+    Args:
+        single_url: Legacy single-URL field (skvector_url / skgraph_url).
+        endpoints: Explicit endpoint list from config.
+        default_role: Role to assign when promoting a single URL.
+
+    Returns:
+        Unified list of EndpointConfig (may be empty).
+    """
+    if endpoints:
+        urls = {ep.url for ep in endpoints}
+        if single_url and single_url not in urls:
+            return [EndpointConfig(url=single_url, role=default_role)] + list(endpoints)
+        return list(endpoints)
+
+    if single_url:
+        return [EndpointConfig(url=single_url, role=default_role)]
+
+    return []

package/skmemory/context_loader.py

@@ -0,0 +1,335 @@
+"""
+Lazy Memory Context Loader - Three-Tier Memory Architecture.
+
+Loads memories efficiently based on date tiers to optimize token usage:
+- TODAY: Full content (active work)
+- YESTERDAY: Summaries only (recent context)
+- HISTORICAL: Reference count (deep search available)
+
+Usage:
+    loader = LazyMemoryLoader("lumina")
+    context = loader.load_active_context()  # Token-optimized
+
+    # Deep search when needed
+    results = loader.deep_search("project gentis")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+
+from .agents import get_agent_paths
+from .backends.sqlite_backend import SQLiteBackend
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MemoryContext:
+    """Container for loaded memory context."""
+
+    today_memories: list[dict]  # Full memories
+    yesterday_summaries: list[dict]  # Summaries only
+    historical_count: int  # Reference count only
+
+    def to_context_string(self, max_tokens: int = 3000) -> str:
+        """Convert to token-optimized context string."""
+        sections = []
+
+        # Today's memories (full content)
+        if self.today_memories:
+            sections.append(f"## Today's Memories ({len(self.today_memories)})")
+            for mem in self.today_memories[:20]:  # Limit to 20
+                content = mem.get("content", "")[:200]  # Truncate if needed
+                sections.append(f"- {mem.get('title', 'Untitled')}: {content}")
+
+        # Yesterday's summaries
+        if self.yesterday_summaries:
+            sections.append(f"\n## Yesterday ({len(self.yesterday_summaries)} memories)")
+            for mem in self.yesterday_summaries[:10]:  # Limit to 10
+                summary = mem.get("summary", "No summary")[:150]
+                sections.append(f"- {mem.get('title', 'Untitled')}: {summary}")
+
+        # Historical reference
+        if self.historical_count > 0:
+            sections.append(f"\n## Historical Memory")
+            sections.append(f"- {self.historical_count} long-term memories available")
+            sections.append(f"- Use 'search memory [query]' to recall specific details")
+
+        return "\n".join(sections)
+
+
+class LazyMemoryLoader:
+    """Efficiently loads memories based on date tiers."""
+
+    def __init__(self, agent_name: Optional[str] = None):
+        self.agent_name = agent_name
+        self.paths = get_agent_paths(agent_name)
+        self.today = datetime.now().date()
+        self.db = SQLiteBackend(str(self.paths["index_db"]))
+
+    def load_active_context(self) -> MemoryContext:
+        """Load token-optimized context for current session.
+
+        Returns:
+            MemoryContext with today (full), yesterday (summaries), historical (count)
+        """
+        return MemoryContext(
+            today_memories=self._load_today(),
+            yesterday_summaries=self._load_yesterday_summaries(),
+            historical_count=self._count_historical(),
+        )
+
+    def _load_today(self) -> list[dict]:
+        """Load today's memories with full content."""
+        today_str = self.today.isoformat()
+        try:
+            cursor = self.db._conn.execute(
+                """
+                SELECT id, title, content, tags, emotional_signature
+                FROM memories 
+                WHERE DATE(created_at) = ? 
+                  AND layer = 'short'
+                ORDER BY created_at DESC
+                LIMIT 50
+                """,
+                (today_str,),
+            )
+            return [
+                {
+                    "id": row[0],
+                    "title": row[1],
+                    "content": row[2],
+                    "tags": json.loads(row[3]) if row[3] else [],
+                    "emotional": json.loads(row[4]) if row[4] else {},
+                }
+                for row in cursor.fetchall()
+            ]
+        except Exception as e:
+            logger.error(f"Failed to load today's memories: {e}")
+            return []
+
+    def _load_yesterday_summaries(self) -> list[dict]:
+        """Load yesterday's memories as summaries only."""
+        yesterday = (self.today - timedelta(days=1)).isoformat()
+        try:
+            cursor = self.db._conn.execute(
+                """
+                SELECT id, title, summary, tags
+                FROM memories 
+                WHERE DATE(created_at) = ? 
+                  AND layer IN ('short', 'medium')
+                ORDER BY importance DESC
+                LIMIT 20
+                """,
+                (yesterday,),
+            )
+            memories = []
+            for row in cursor.fetchall():
+                mem = {
+                    "id": row[0],
+                    "title": row[1],
+                    "summary": row[2] or self._generate_summary(row[1]),
+                    "tags": json.loads(row[3]) if row[3] else [],
+                }
+                memories.append(mem)
+            return memories
+        except Exception as e:
+            logger.error(f"Failed to load yesterday's summaries: {e}")
+            return []
+
+    def _count_historical(self) -> int:
+        """Count older memories (not loaded into context)."""
+        yesterday = (self.today - timedelta(days=1)).isoformat()
+        try:
+            cursor = self.db._conn.execute(
+                """
+                SELECT COUNT(*) FROM memories 
+                WHERE DATE(created_at) < ?
+                """,
+                (yesterday,),
+            )
+            return cursor.fetchone()[0]
+        except Exception as e:
+            logger.error(f"Failed to count historical memories: {e}")
+            return 0
+
+    def _generate_summary(self, content: str, sentences: int = 2) -> str:
+        """Generate a brief summary (fallback if no summary stored)."""
+        # Simple truncation-based summary
+        words = content.split()[:30]  # First 30 words
+        return " ".join(words) + "..." if len(words) >= 30 else content
+
+    def deep_search(self, query: str, max_results: int = 10) -> list[dict]:
+        """Search ALL memory tiers (on demand, token-heavy).
+
+        Args:
+            query: Search query
+            max_results: Maximum results to return
+
+        Returns:
+            List of full memory details
+        """
+        results = []
+
+        # Search SQLite (title, content, tags)
+        results.extend(self._search_sqlite(query))
+
+        # TODO: Add SKVector search if enabled
+        # results.extend(self._search_skvector(query))
+
+        # TODO: Add SKGraph search if enabled
+        # results.extend(self._search_skgraph(query))
+
+        # Sort by relevance (simple: contains query)
+        results = sorted(
+            results,
+            key=lambda x: (
+                x.get("content", "").lower().count(query.lower()),
+                x.get("title", "").lower().count(query.lower()),
+            ),
+            reverse=True,
+        )
+
+        return results[:max_results]
+
+    def _search_sqlite(self, query: str) -> list[dict]:
+        """Search SQLite for memories matching query."""
+        try:
+            pattern = f"%{query}%"
+            cursor = self.db._conn.execute(
+                """
+                SELECT id, title, content, summary, tags, layer, created_at
+                FROM memories 
+                WHERE title LIKE ? OR content LIKE ? OR tags LIKE ?
+                ORDER BY 
+                    CASE 
+                        WHEN title LIKE ? THEN 3
+                        WHEN content LIKE ? THEN 2
+                        ELSE 1
+                    END DESC,
+                    created_at DESC
+                LIMIT 50
+                """,
+                (pattern, pattern, pattern, pattern, pattern),
+            )
+            return [
+                {
+                    "id": row[0],
+                    "title": row[1],
+                    "content": row[2],
+                    "summary": row[3],
+                    "tags": json.loads(row[4]) if row[4] else [],
+                    "layer": row[5],
+                    "created_at": row[6],
+                }
+                for row in cursor.fetchall()
+            ]
+        except Exception as e:
+            logger.error(f"Failed to search SQLite: {e}")
+            return []
+
+    def get_memory_by_id(self, memory_id: str) -> Optional[dict]:
+        """Load full memory details by ID (for deep recall).
+
+        Args:
+            memory_id: UUID of the memory
+
+        Returns:
+            Full memory dict or None
+        """
+        try:
+            cursor = self.db._conn.execute(
+                """
+                SELECT id, title, content, summary, tags, 
+                       emotional_signature, layer, created_at
+                FROM memories 
+                WHERE id = ?
+                """,
+                (memory_id,),
+            )
+            row = cursor.fetchone()
+            if row:
+                return {
+                    "id": row[0],
+                    "title": row[1],
+                    "content": row[2],
+                    "summary": row[3],
+                    "tags": json.loads(row[4]) if row[4] else [],
+                    "emotional": json.loads(row[5]) if row[5] else {},
+                    "layer": row[6],
+                    "created_at": row[7],
+                }
+        except Exception as e:
+            logger.error(f"Failed to get memory {memory_id}: {e}")
+        return None
+
+    def promote_memory(self, memory_id: str, to_layer: str) -> bool:
+        """Promote memory to different tier and generate summary.
+
+        Args:
+            memory_id: Memory to promote
+            to_layer: Target layer ('short', 'medium', 'long')
+
+        Returns:
+            True if successful
+        """
+        try:
+            # Get memory content
+            memory = self.get_memory_by_id(memory_id)
+            if not memory:
+                return False
+
+            # Generate summary if promoting to medium/long
+            if to_layer in ("medium", "long") and not memory.get("summary"):
+                summary = self._generate_summary(memory["content"], 2)
+
+                # Update in database
+                self.db._conn.execute(
+                    """
+                    UPDATE memories 
+                    SET layer = ?, summary = ?
+                    WHERE id = ?
+                    """,
+                    (to_layer, summary, memory_id),
+                )
+                self.db._conn.commit()
+
+                # Also move flat file
+                self._move_flat_file(memory_id, to_layer)
+
+                logger.info(f"Promoted memory {memory_id} to {to_layer}")
+                return True
+
+        except Exception as e:
+            logger.error(f"Failed to promote memory {memory_id}: {e}")
+
+        return False
+
+    def _move_flat_file(self, memory_id: str, to_layer: str):
+        """Move memory flat file to appropriate tier directory."""
+        # Find current location
+        for layer in ["short", "medium", "long"]:
+            src = self.paths["memory_" + layer] / f"{memory_id}.json"
+            if src.exists():
+                dst = self.paths["memory_" + to_layer] / f"{memory_id}.json"
+                src.rename(dst)
+                logger.debug(f"Moved {src} -> {dst}")
+                break
+
+
+def get_context_for_session(agent_name: Optional[str] = None) -> str:
+    """Convenience function: get token-optimized context.
+
+    Usage:
+        context = get_context_for_session("lumina")
+        # Returns formatted string with today's + yesterday's summaries
+    """
+    loader = LazyMemoryLoader(agent_name)
+    context = loader.load_active_context()
+    return context.to_context_string()