claude-memory-agent 3.0.3 → 3.1.0

package/services/agent_registry.py

@@ -1,16 +1,24 @@
 """
-Agent Registry - Loads agent/MCP/hook catalogs from JSON data file.
+Agent Registry - Discovers real agents from disk + loads MCP/hook catalogs.
 
-Static display metadata lives in agent_catalog.json (~880 entries).
-Dynamic loading (hooks, MCPs) reads Claude Code settings files at runtime
-to determine which entries are truly configured.
+Agents are discovered from ~/.claude/agents/ (global) and
+<project>/.claude/agents/ (per-project) by scanning .md files with YAML
+frontmatter.  Enabled agents live in the root; disabled agents live in
+_disabled/<category>/ subdirectories.
+
+MCP and hook metadata still comes from agent_catalog.json; dynamic loading
+reads Claude Code settings files at runtime to check configured status.
 """
 
 import json
 import logging
+import re
+import shutil
 from pathlib import Path
 from typing import List, Dict, Any, Optional
 
+import yaml
+
 logger = logging.getLogger(__name__)
 
 # ---------------------------------------------------------------------------
@@ -24,8 +32,11 @@ _GLOBAL_SETTINGS_PATH = _HOME / ".claude" / "settings.json"
 _PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
 _LOCAL_SETTINGS_PATH = _PROJECT_ROOT / ".claude" / "settings.local.json"
 
+# Global agents directory
+_GLOBAL_AGENTS_DIR = _HOME / ".claude" / "agents"
+
 # ---------------------------------------------------------------------------
-# Load static catalog from JSON
+# Load static catalog from JSON (MCPs + hooks only now)
 # ---------------------------------------------------------------------------
 _CATALOG_PATH = Path(__file__).resolve().parent / "agent_catalog.json"
 
@@ -42,11 +53,249 @@ def _load_catalog() -> dict:
 
 _catalog = _load_catalog()
 
+# Categories are now auto-generated from discovered agents, but keep catalog
+# ones as fallback for MCPs/hooks
 AGENT_CATEGORIES = _catalog.get("categories", {})
-AVAILABLE_AGENTS = _catalog.get("agents", [])
 AVAILABLE_MCPS = _catalog.get("mcps", [])
 AVAILABLE_HOOKS = _catalog.get("hooks", [])
 
+# ---------------------------------------------------------------------------
+# Category metadata (colors + icons for known categories)
+# ---------------------------------------------------------------------------
+_CATEGORY_META = {
+    "design":              {"name": "Design",              "color": "#f778ba", "icon": "palette"},
+    "engineering":         {"name": "Engineering",         "color": "#58a6ff", "icon": "code"},
+    "marketing":           {"name": "Marketing",           "color": "#f0883e", "icon": "bullhorn"},
+    "product":             {"name": "Product",             "color": "#a371f7", "icon": "lightbulb"},
+    "project-management":  {"name": "Project Management",  "color": "#3fb950", "icon": "tasks"},
+    "spatial-computing":   {"name": "Spatial Computing",   "color": "#79c0ff", "icon": "cube"},
+    "specialized":         {"name": "Specialized",         "color": "#d2a8ff", "icon": "star"},
+    "support":             {"name": "Support",             "color": "#56d364", "icon": "headset"},
+    "testing":             {"name": "Testing",             "color": "#e3b341", "icon": "vial"},
+}
+
+# Color palette for agent frontmatter colors
+_COLOR_MAP = {
+    "blue":    "#58a6ff",
+    "green":   "#3fb950",
+    "red":     "#f85149",
+    "purple":  "#a371f7",
+    "orange":  "#f0883e",
+    "yellow":  "#e3b341",
+    "pink":    "#f778ba",
+    "cyan":    "#79c0ff",
+    "teal":    "#2ea043",
+}
+
+
+# ---------------------------------------------------------------------------
+# Agent discovery from disk
+# ---------------------------------------------------------------------------
+def _parse_frontmatter(file_path: Path) -> Optional[Dict[str, Any]]:
+    """Parse YAML frontmatter from an agent .md file."""
+    try:
+        text = file_path.read_text(encoding="utf-8")
+    except Exception as e:
+        logger.warning(f"Could not read {file_path}: {e}")
+        return None
+
+    # Match --- delimited frontmatter
+    match = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
+    if not match:
+        return None
+
+    try:
+        data = yaml.safe_load(match.group(1))
+        if not isinstance(data, dict):
+            return None
+        return data
+    except yaml.YAMLError as e:
+        logger.warning(f"Invalid YAML frontmatter in {file_path}: {e}")
+        return None
+
+
+def _infer_category(filename_stem: str) -> str:
+    """Infer category from filename prefix like 'engineering-senior-developer'."""
+    known = [
+        "design", "engineering", "marketing", "product",
+        "project-management", "spatial-computing", "specialized",
+        "support", "testing",
+    ]
+    for cat in known:
+        if filename_stem.startswith(cat):
+            return cat
+    # Try first word as category
+    parts = filename_stem.split("-")
+    if parts:
+        return parts[0]
+    return "uncategorized"
+
+
+def _resolve_color(color_str: Optional[str]) -> str:
+    """Resolve a color name or hex value to a hex color."""
+    if not color_str:
+        return "#8b949e"
+    if color_str.startswith("#"):
+        return color_str
+    return _COLOR_MAP.get(color_str.lower(), "#8b949e")
+
+
+def _parse_agent_file(
+    file_path: Path,
+    enabled: bool,
+    scope: str,
+    category: Optional[str] = None,
+    base_dir: Optional[Path] = None,
+) -> Optional[Dict[str, Any]]:
+    """Parse a single agent .md file into an agent dict."""
+    fm = _parse_frontmatter(file_path)
+    if fm is None:
+        # Minimal entry for files without valid frontmatter
+        fm = {}
+
+    stem = file_path.stem
+    agent_id = stem
+
+    name = fm.get("name", stem.replace("-", " ").replace("_", " ").title())
+    description = fm.get("description", "").replace("\\n", " ").strip()
+    color = _resolve_color(fm.get("color"))
+
+    if not category:
+        category = _infer_category(stem)
+
+    return {
+        "id": agent_id,
+        "name": name,
+        "description": description,
+        "color": color,
+        "category": category,
+        "enabled": enabled,
+        "scope": scope,
+        "file_path": str(file_path),
+        "filename": file_path.name,
+        "base_dir": str(base_dir or file_path.parent),
+    }
+
+
+def _scan_agent_dir(base_dir: Path, scope: str) -> List[Dict[str, Any]]:
+    """Scan an agents directory for enabled and disabled agents."""
+    agents = []
+    if not base_dir.exists():
+        return agents
+
+    # Enabled agents: .md files directly in base_dir
+    for f in sorted(base_dir.glob("*.md")):
+        if f.name.lower() in ("readme.md", "readme.txt"):
+            continue
+        agent = _parse_agent_file(
+            f, enabled=True, scope=scope, base_dir=base_dir
+        )
+        if agent:
+            agents.append(agent)
+
+    # Disabled agents: .md files in _disabled/ subdirectories
+    disabled_dir = base_dir / "_disabled"
+    if disabled_dir.exists():
+        for f in sorted(disabled_dir.rglob("*.md")):
+            if f.name.lower() in ("readme.md", "readme.txt"):
+                continue
+            # Category = parent dir name if it's a subdirectory of _disabled
+            if f.parent != disabled_dir:
+                category = f.parent.name
+            else:
+                category = _infer_category(f.stem)
+            agent = _parse_agent_file(
+                f, enabled=False, scope=scope,
+                category=category, base_dir=base_dir,
+            )
+            if agent:
+                agents.append(agent)
+
+    return agents
+
+
+def discover_agents(project_path: Optional[str] = None) -> List[Dict[str, Any]]:
+    """Discover all agents from global and project directories.
+
+    Returns a list of agent dicts with id, name, description, color,
+    category, enabled, scope, file_path, filename, base_dir.
+    """
+    agents = []
+
+    # 1. Global agents: ~/.claude/agents/
+    agents += _scan_agent_dir(_GLOBAL_AGENTS_DIR, scope="global")
+
+    # 2. Project agents: <project>/.claude/agents/
+    if project_path:
+        project_dir = Path(project_path) / ".claude" / "agents"
+        agents += _scan_agent_dir(project_dir, scope="project")
+
+    return agents
+
+
+def discover_categories(agents: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
+    """Build categories dict from discovered agents."""
+    cats: Dict[str, Dict[str, Any]] = {}
+    for agent in agents:
+        cat_key = agent["category"]
+        if cat_key not in cats:
+            meta = _CATEGORY_META.get(cat_key, {
+                "name": cat_key.replace("-", " ").replace("_", " ").title(),
+                "color": "#8b949e",
+                "icon": "circle",
+            })
+            cats[cat_key] = meta
+    return cats
+
+
+def find_agent_by_id(
+    agent_id: str, project_path: Optional[str] = None
+) -> Optional[Dict[str, Any]]:
+    """Find a specific agent by ID across all directories."""
+    for agent in discover_agents(project_path):
+        if agent["id"] == agent_id:
+            return agent
+    return None
+
+
+def toggle_agent(agent_id: str, enabled: bool, project_path: Optional[str] = None) -> Dict[str, Any]:
+    """Toggle an agent between enabled and disabled state.
+
+    - Enable: move from _disabled/<category>/ to agents/ root
+    - Disable: move from agents/ root to _disabled/<category>/
+
+    Returns the updated agent dict.
+    """
+    agent = find_agent_by_id(agent_id, project_path)
+    if not agent:
+        raise FileNotFoundError(f"Agent '{agent_id}' not found")
+
+    src = Path(agent["file_path"])
+    base = Path(agent["base_dir"])
+
+    if enabled and not agent["enabled"]:
+        # Move from _disabled/<category>/ to agents/ root
+        dest = base / agent["filename"]
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(str(src), str(dest))
+        agent["file_path"] = str(dest)
+        agent["enabled"] = True
+    elif not enabled and agent["enabled"]:
+        # Move from agents/ root to _disabled/<category>/
+        category = agent["category"]
+        category_dir = base / "_disabled" / category
+        category_dir.mkdir(parents=True, exist_ok=True)
+        dest = category_dir / agent["filename"]
+        shutil.move(str(src), str(dest))
+        agent["file_path"] = str(dest)
+        agent["enabled"] = False
+
+    return agent
+
+
+# Backward-compatible: AVAILABLE_AGENTS as lazy discovery result
+AVAILABLE_AGENTS = discover_agents()
+
 
 # ---------------------------------------------------------------------------
 # Helper: derive a hook ID from its command string
@@ -249,10 +498,12 @@ def load_configured_mcps(
 # Helper functions (unchanged interface)
 # ---------------------------------------------------------------------------
 
-def get_agents_by_category():
+def get_agents_by_category(agents: Optional[List[Dict[str, Any]]] = None):
     """Group agents by category."""
+    if agents is None:
+        agents = discover_agents()
     result = {}
-    for agent in AVAILABLE_AGENTS:
+    for agent in agents:
         cat = agent["category"]
         if cat not in result:
             result[cat] = []
@@ -262,7 +513,4 @@ def get_agents_by_category():
 
 def get_agent_by_id(agent_id: str):
     """Get agent by ID."""
-    for agent in AVAILABLE_AGENTS:
-        if agent["id"] == agent_id:
-            return agent
-    return None
+    return find_agent_by_id(agent_id)

package/services/database.py

@@ -1234,6 +1234,43 @@ class DatabaseService:
         cursor.execute("CREATE INDEX IF NOT EXISTS idx_session_activity_project ON session_activity(project_path, timestamp DESC)")
         cursor.execute("CREATE INDEX IF NOT EXISTS idx_session_activity_session ON session_activity(session_id, timestamp DESC)")
 
+        # ============================================================
+        # SOUL LAYER TABLES
+        # ============================================================
+
+        # Soul state - persistent synthesized understanding per project
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS soul_state (
+                project_path TEXT PRIMARY KEY,
+                soul_brief TEXT DEFAULT '',
+                user_model TEXT DEFAULT '{}',
+                project_understanding TEXT DEFAULT '{}',
+                success_journal TEXT DEFAULT '[]',
+                blind_spots TEXT DEFAULT '[]',
+                tool_preferences TEXT DEFAULT '{}',
+                last_integrated_at TEXT,
+                integration_count INTEGER DEFAULT 0,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        # Soul fragments - staging area for session observations
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS soul_fragments (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                session_id TEXT NOT NULL,
+                project_path TEXT NOT NULL,
+                fragment_type TEXT NOT NULL,
+                content TEXT NOT NULL,
+                raw_source TEXT DEFAULT '',
+                captured_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                integrated BOOLEAN DEFAULT 0
+            )
+        """)
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_soul_fragments_session ON soul_fragments(session_id)")
+        cursor.execute("CREATE INDEX IF NOT EXISTS idx_soul_fragments_project ON soul_fragments(project_path, integrated)")
+
         self.conn.commit()
 
     def _serialize_embedding(self, embedding: List[float]) -> str:
@@ -3347,6 +3384,155 @@ class DatabaseService:
             for row in cursor.fetchall()
         ]
 
+    # ============================================================
+    # SOUL LAYER METHODS
+    # ============================================================
+
+    async def get_soul_state(self, project_path: str) -> Optional[Dict[str, Any]]:
+        """Get the soul state for a project. Returns None if no state exists."""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    "SELECT * FROM soul_state WHERE project_path = ?",
+                    (project_path,)
+                )
+                row = cursor.fetchone()
+                return dict(row) if row else None
+        except Exception as e:
+            logger.warning(f"Failed to get soul state: {e}")
+            return None
+
+    async def upsert_soul_state(self, project_path: str, updates: Dict[str, Any]) -> bool:
+        """Create or update the soul state for a project.
+
+        Args:
+            project_path: Project path (primary key)
+            updates: Dict of column->value pairs to set
+        """
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                # Check if exists
+                cursor.execute(
+                    "SELECT 1 FROM soul_state WHERE project_path = ?",
+                    (project_path,)
+                )
+                exists = cursor.fetchone() is not None
+
+                if exists:
+                    # Build UPDATE dynamically from updates dict
+                    allowed_cols = {
+                        'soul_brief', 'user_model', 'project_understanding',
+                        'success_journal', 'blind_spots', 'tool_preferences',
+                        'last_integrated_at', 'integration_count'
+                    }
+                    set_parts = []
+                    values = []
+                    for col, val in updates.items():
+                        if col in allowed_cols:
+                            set_parts.append(f"{col} = ?")
+                            values.append(val)
+                    if not set_parts:
+                        return True
+                    set_parts.append("updated_at = CURRENT_TIMESTAMP")
+                    values.append(project_path)
+                    sql = f"UPDATE soul_state SET {', '.join(set_parts)} WHERE project_path = ?"
+                    cursor.execute(sql, values)
+                else:
+                    # INSERT with defaults + any provided updates
+                    cols = ['project_path']
+                    vals = [project_path]
+                    allowed_cols = {
+                        'soul_brief', 'user_model', 'project_understanding',
+                        'success_journal', 'blind_spots', 'tool_preferences',
+                        'last_integrated_at', 'integration_count'
+                    }
+                    for col, val in updates.items():
+                        if col in allowed_cols:
+                            cols.append(col)
+                            vals.append(val)
+                    placeholders = ', '.join(['?'] * len(cols))
+                    sql = f"INSERT INTO soul_state ({', '.join(cols)}) VALUES ({placeholders})"
+                    cursor.execute(sql, vals)
+
+                conn.commit()
+                return True
+        except Exception as e:
+            logger.error(f"Failed to upsert soul state: {e}")
+            return False
+
+    async def insert_soul_fragment(
+        self, session_id: str, project_path: str,
+        fragment_type: str, content: str, raw_source: str = ""
+    ) -> Optional[int]:
+        """Insert a soul fragment into the staging table."""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    """INSERT INTO soul_fragments
+                       (session_id, project_path, fragment_type, content, raw_source)
+                       VALUES (?, ?, ?, ?, ?)""",
+                    (session_id, project_path, fragment_type, content, raw_source)
+                )
+                conn.commit()
+                return cursor.lastrowid
+        except Exception as e:
+            logger.warning(f"Failed to insert soul fragment: {e}")
+            return None
+
+    async def get_session_fragments(
+        self, session_id: str, integrated: Optional[bool] = None
+    ) -> List[Dict[str, Any]]:
+        """Get all soul fragments for a session."""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                if integrated is not None:
+                    cursor.execute(
+                        "SELECT * FROM soul_fragments WHERE session_id = ? AND integrated = ? ORDER BY captured_at",
+                        (session_id, int(integrated))
+                    )
+                else:
+                    cursor.execute(
+                        "SELECT * FROM soul_fragments WHERE session_id = ? ORDER BY captured_at",
+                        (session_id,)
+                    )
+                return [dict(row) for row in cursor.fetchall()]
+        except Exception as e:
+            logger.warning(f"Failed to get session fragments: {e}")
+            return []
+
+    async def get_unintegrated_fragments(self, project_path: str) -> List[Dict[str, Any]]:
+        """Get all unintegrated soul fragments for a project."""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    "SELECT * FROM soul_fragments WHERE project_path = ? AND integrated = 0 ORDER BY captured_at",
+                    (project_path,)
+                )
+                return [dict(row) for row in cursor.fetchall()]
+        except Exception as e:
+            logger.warning(f"Failed to get unintegrated fragments: {e}")
+            return []
+
+    async def mark_fragments_integrated(self, session_id: str) -> int:
+        """Mark all fragments for a session as integrated. Returns count."""
+        try:
+            with self.get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    "UPDATE soul_fragments SET integrated = 1 WHERE session_id = ? AND integrated = 0",
+                    (session_id,)
+                )
+                conn.commit()
+                return cursor.rowcount
+        except Exception as e:
+            logger.warning(f"Failed to mark fragments integrated: {e}")
+            return 0
+
     # ============================================================
     # GENERIC QUERY METHOD
     # ============================================================