@jaguilar87/gaia 5.0.0-rc1

package/tools/memory/search_store.py

@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+"""
+FTS5-Backed Search Store for GAIA-OPS
+
+Provides full-text search over episodic memory using SQLite FTS5 (default)
+with an optional Chroma vector-search backend.
+
+Zero external dependencies for core operation — stdlib only (sqlite3, shutil,
+os, pathlib, typing, abc).  chromadb is imported lazily and only used if
+available and not suppressed via GAIA_TEST_NO_CHROMA.
+
+Architecture:
+- SearchProvider ABC defines the interface
+- FTS5Provider   wraps SQLite FTS5 (always available)
+- ChromaProvider stub that activates only when chromadb is importable and
+                 GAIA_TEST_NO_CHROMA is not set
+- Module-level get_backend() / index_episode() / search() / count()
+  delegate to the active provider (resolved once at import time)
+- Lazy init: DB/table created on first FTS5 call
+- Fail-safe: all public functions wrapped in try/except
+
+Environment:
+    GAIA_SEARCH_DB_PATH:    Override the default SQLite DB path.
+    GAIA_TEST_NO_CHROMA:    Set to any non-empty value to force FTS5 backend.
+
+Functions:
+    index_episode   -- Insert or ignore an episode into the active backend
+    search          -- Query backend, returns ranked results
+    count           -- Count indexed episodes
+    get_backend     -- Returns "chroma" or "fts5"
+    has_engram      -- Returns True if engram binary is on PATH
+"""
+
+import abc
+import os
+import shutil
+import sqlite3
+from pathlib import Path
+from typing import Dict, List, Optional
+
+
+# ---------------------------------------------------------------------------
+# DB path resolution (used by FTS5Provider)
+# ---------------------------------------------------------------------------
+
+_DEFAULT_RELATIVE_PATH = ".claude/project-context/episodic-memory/search.db"
+
+
+def _resolve_db_path() -> Path:
+    """Resolve the search DB path.
+
+    Priority:
+        1. GAIA_SEARCH_DB_PATH environment variable
+        2. Highest ancestor with a .claude/ directory (closest to HOME),
+           so that a nested .claude/ in a sub-repository or dev checkout
+           never shadows the real Gaia instance.
+        3. Bare relative path fallback (last resort, same as before).
+    """
+    env_path = os.environ.get("GAIA_SEARCH_DB_PATH")
+    if env_path:
+        return Path(env_path)
+
+    try:
+        from memory.paths import find_highest_claude_root
+        root = find_highest_claude_root()
+        if root is not None:
+            return root / _DEFAULT_RELATIVE_PATH
+    except ImportError:
+        pass
+
+    # Fallback: original first-match walk (keeps behaviour if paths.py is
+    # somehow unavailable, e.g. during isolated unit tests that only add
+    # the tools/ root to sys.path after import).
+    current = Path.cwd()
+    for parent in [current, *current.parents]:
+        if (parent / ".claude").is_dir():
+            return parent / _DEFAULT_RELATIVE_PATH
+
+    return Path(_DEFAULT_RELATIVE_PATH)
+
+
+# ---------------------------------------------------------------------------
+# SearchProvider ABC
+# ---------------------------------------------------------------------------
+
+class SearchProvider(abc.ABC):
+    """Abstract base for search backends."""
+
+    @abc.abstractmethod
+    def index(self, episode_id: str, text: str, **kwargs) -> None:
+        """Insert or update an episode in the index."""
+
+    @abc.abstractmethod
+    def search(self, query: str, max_results: int = 10) -> List[Dict]:
+        """Return a ranked list of dicts with at least 'episode_id' key."""
+
+    @abc.abstractmethod
+    def count(self) -> int:
+        """Return the total number of indexed episodes."""
+
+
+# ---------------------------------------------------------------------------
+# FTS5Provider
+# ---------------------------------------------------------------------------
+
+class FTS5Provider(SearchProvider):
+    """SQLite FTS5 backend — always available, zero external dependencies."""
+
+    def __init__(self) -> None:
+        self._connection: Optional[sqlite3.Connection] = None
+
+    # -- internal helpers ---------------------------------------------------
+
+    def _get_connection(self) -> sqlite3.Connection:
+        if self._connection is not None:
+            return self._connection
+
+        db_path = _resolve_db_path()
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+        conn = sqlite3.connect(str(db_path), check_same_thread=False)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute(
+            """
+            CREATE VIRTUAL TABLE IF NOT EXISTS episodes_fts USING fts5(
+                episode_id,
+                prompt,
+                enriched_prompt,
+                tags,
+                title
+            )
+            """
+        )
+        conn.commit()
+        self._connection = conn
+        return self._connection
+
+    @staticmethod
+    def _sanitize_query(query: str) -> str:
+        """Append * wildcard to each word for FTS5 prefix matching.
+
+        Uses prefix matching instead of exact quoted tokens so that
+        "approval" matches "approvals", "approving", etc.
+        Special characters that would break FTS5 syntax are stripped.
+
+        Hyphens are replaced with spaces before tokenisation so that
+        queries like "brief-spec" or "context-v5" are treated as two
+        separate prefix terms ("brief*" "spec*") rather than a single
+        phrase that FTS5 cannot match (FTS5 treats hyphens as token
+        separators at index time, so the stored tokens never contain
+        hyphens).
+        """
+        # Replace hyphens with spaces so "brief-spec" → "brief spec"
+        query = query.replace("-", " ")
+        words = query.split()
+        # Strip characters that break FTS5 syntax, then append wildcard
+        safe = [w.replace('"', '').replace("'", '').strip('*') for w in words if w]
+        return " ".join(w + "*" for w in safe if w)
+
+    # -- SearchProvider interface ------------------------------------------
+
+    def index(self, episode_id: str, text: str, **kwargs) -> None:
+        """Insert episode into FTS5 table (no-op if already present).
+
+        Keyword arguments are mapped to FTS5 columns:
+            enriched_prompt, tags, title
+        The ``text`` positional argument maps to the ``prompt`` column.
+        """
+        enriched_prompt = kwargs.get("enriched_prompt", "")
+        tags = kwargs.get("tags", "")
+        title = kwargs.get("title", "")
+        try:
+            conn = self._get_connection()
+            existing = conn.execute(
+                "SELECT rowid FROM episodes_fts WHERE episode_id = ?",
+                (episode_id,),
+            ).fetchone()
+            if existing is not None:
+                return
+            conn.execute(
+                "INSERT INTO episodes_fts"
+                "(episode_id, prompt, enriched_prompt, tags, title) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (episode_id, text, enriched_prompt, tags, title),
+            )
+            conn.commit()
+        except Exception:  # noqa: BLE001
+            pass
+
+    def search(self, query: str, max_results: int = 10) -> List[Dict]:
+        if not query or not query.strip():
+            return []
+        try:
+            conn = self._get_connection()
+            sanitized = self._sanitize_query(query.strip())
+            rows = conn.execute(
+                "SELECT episode_id, rank FROM episodes_fts "
+                "WHERE episodes_fts MATCH ? "
+                "ORDER BY rank "
+                "LIMIT ?",
+                (sanitized, max_results),
+            ).fetchall()
+            return [{"episode_id": row[0], "rank": row[1]} for row in rows]
+        except Exception:  # noqa: BLE001
+            return []
+
+    def count(self) -> int:
+        try:
+            conn = self._get_connection()
+            row = conn.execute("SELECT COUNT(*) FROM episodes_fts").fetchone()
+            return int(row[0]) if row else 0
+        except Exception:  # noqa: BLE001
+            return 0
+
+
+# ---------------------------------------------------------------------------
+# ChromaProvider (stub)
+# ---------------------------------------------------------------------------
+
+class ChromaProvider(SearchProvider):
+    """Chroma vector-search backend.
+
+    Only instantiated when:
+    - ``chromadb`` is importable, AND
+    - ``GAIA_TEST_NO_CHROMA`` env var is NOT set
+
+    Raises ``ImportError`` in __init__ if either condition is not met so
+    that the factory can fall back to FTS5Provider transparently.
+    """
+
+    def __init__(self) -> None:
+        if os.environ.get("GAIA_TEST_NO_CHROMA"):
+            raise ImportError("GAIA_TEST_NO_CHROMA is set — Chroma disabled")
+        import chromadb  # noqa: F401 -- intentional optional import
+        # Future: initialise a persistent Chroma client and collection here.
+        # For now this is a stub; raise to signal not-yet-implemented.
+        raise NotImplementedError(
+            "ChromaProvider is a stub — not yet fully implemented"
+        )
+
+    def index(self, episode_id: str, text: str, **kwargs) -> None:  # pragma: no cover
+        raise NotImplementedError
+
+    def search(self, query: str, max_results: int = 10) -> List[Dict]:  # pragma: no cover
+        raise NotImplementedError
+
+    def count(self) -> int:  # pragma: no cover
+        raise NotImplementedError
+
+
+# ---------------------------------------------------------------------------
+# Provider resolution (module-level singleton)
+# ---------------------------------------------------------------------------
+
+def _resolve_provider() -> SearchProvider:
+    """Return the best available SearchProvider.
+
+    Tries ChromaProvider first; falls back to FTS5Provider silently on any
+    ImportError or NotImplementedError.
+    """
+    try:
+        return ChromaProvider()
+    except (ImportError, NotImplementedError):
+        return FTS5Provider()
+
+
+_provider: SearchProvider = _resolve_provider()
+
+
+# ---------------------------------------------------------------------------
+# Public API — delegates to active provider
+# ---------------------------------------------------------------------------
+
+def index_episode(
+    episode_id: str,
+    prompt: str,
+    enriched_prompt: str = "",
+    tags: str = "",
+    title: str = "",
+) -> None:
+    """Insert an episode into the active backend (no-op if already present).
+
+    Parameters
+    ----------
+    episode_id:
+        Unique identifier for the episode.
+    prompt:
+        Original user prompt text.
+    enriched_prompt:
+        Expanded / enriched version of the prompt (may be empty).
+    tags:
+        Space- or comma-separated tags string (may be empty).
+    title:
+        Short title for the episode (may be empty).
+    """
+    _provider.index(
+        episode_id,
+        prompt,
+        enriched_prompt=enriched_prompt,
+        tags=tags,
+        title=title,
+    )
+
+
+def search(query: str, max_results: int = 10) -> List[Dict]:
+    """Search the active backend using the configured ranking strategy.
+
+    Parameters
+    ----------
+    query:
+        Free-text search query.
+    max_results:
+        Maximum number of results to return (default: 10).
+
+    Returns
+    -------
+    list of dict
+        Each dict contains at minimum ``{"episode_id": str}``. FTS5 backend
+        also includes ``{"rank": float}``. Returns empty list on error.
+    """
+    return _provider.search(query, max_results)
+
+
+def count() -> int:
+    """Return the total number of episodes indexed in the active backend.
+
+    Returns
+    -------
+    int
+        Row count, or 0 on any error.
+    """
+    return _provider.count()
+
+
+def get_backend() -> str:
+    """Return the active search backend identifier.
+
+    Returns
+    -------
+    str
+        ``"chroma"`` if ChromaProvider is active, ``"fts5"`` otherwise.
+    """
+    if isinstance(_provider, ChromaProvider):
+        return "chroma"
+    return "fts5"
+
+
+def has_engram() -> bool:
+    """Check whether the engram binary is available on PATH.
+
+    Returns
+    -------
+    bool
+        True if ``engram`` is found via shutil.which, False otherwise.
+        Never raises an exception.
+    """
+    try:
+        return shutil.which("engram") is not None
+    except Exception:  # noqa: BLE001
+        return False

package/tools/persist_transcript_analysis.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Retroactive transcript analysis: analyze .output files and persist metrics.
+
+Reads all .output transcript files from a session's tasks directory,
+runs analyze() and compute_compliance_score() from transcript_analyzer,
+and appends non-empty results as JSON lines to the episodic-memory metrics file.
+"""
+
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Ensure the hooks package is importable
+PLUGIN_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PLUGIN_ROOT))
+
+from hooks.modules.agents.transcript_analyzer import analyze, compute_compliance_score
+
+SESSION_ID = "5da6ec0a-5471-4af9-b301-794748b21f62"
+TASKS_DIR = Path(
+    f"/tmp/claude-1000/-home-jaguilar-aaxis-qxo/{SESSION_ID}/tasks"
+)
+METRICS_FILE = Path(
+    "/home/jaguilar/aaxis/qxo/.claude/project-context"
+    "/workflow-episodic-memory/metrics.jsonl"
+)
+
+
+def main() -> None:
+    if not TASKS_DIR.exists():
+        print(f"Tasks directory not found: {TASKS_DIR}")
+        sys.exit(1)
+
+    output_files = sorted(TASKS_DIR.glob("*.output"))
+    if not output_files:
+        print("No .output files found.")
+        sys.exit(0)
+
+    METRICS_FILE.parent.mkdir(parents=True, exist_ok=True)
+
+    persisted = 0
+
+    with open(METRICS_FILE, "a") as f:
+        for output_path in output_files:
+            task_id = output_path.stem  # filename without .output
+
+            analysis = analyze(str(output_path))
+
+            if analysis.api_call_count == 0:
+                continue
+
+            score = compute_compliance_score(analysis, True, False, 0.5)
+
+            timestamp = analysis.first_timestamp or datetime.now(
+                timezone.utc
+            ).isoformat()
+
+            entry = {
+                "timestamp": timestamp,
+                "session_id": SESSION_ID,
+                "task_id": task_id,
+                "agent": "retroactive-analysis",
+                "source": "transcript_analyzer_retroactive",
+                "input_tokens": analysis.input_tokens,
+                "cache_creation_tokens": analysis.cache_creation_tokens,
+                "cache_read_tokens": analysis.cache_read_tokens,
+                "output_tokens": analysis.output_tokens,
+                "duration_ms": analysis.duration_ms,
+                "tool_call_count": analysis.tool_call_count,
+                "skills_injected": analysis.skills_injected,
+                "model_used": analysis.model,
+                "compliance_score": score.total,
+                "compliance_grade": score.grade,
+                "api_call_count": analysis.api_call_count,
+            }
+
+            f.write(json.dumps(entry, separators=(",", ":")) + "\n")
+            persisted += 1
+
+    print(f"Persisted {persisted} entries to {METRICS_FILE}")
+
+
+if __name__ == "__main__":
+    main()

package/tools/review/__init__.py

@@ -0,0 +1 @@
+# Review tools for pending update management

package/tools/review/review_engine.py

@@ -0,0 +1,157 @@
+"""Review engine for managing pending context update suggestions.
+
+This module provides the review logic for pending update suggestions,
+including listing, approving, rejecting, and viewing statistics of
+updates discovered by agents.
+"""
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+# Import PendingUpdateStore from the context module
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from context.pending_updates import PendingUpdateStore
+
+
+def review_pending(
+    action: str,
+    update_id: Optional[str] = None,
+    context_path: Optional[Path] = None,
+    store: Optional["PendingUpdateStore"] = None
+) -> dict:
+    """Execute a review action on pending updates.
+
+    Args:
+        action: One of "list", "approve", "reject", "stats"
+        update_id: Required for approve/reject actions
+        context_path: Optional path to project-context.json (for apply)
+        store: Optional PendingUpdateStore instance (for testing)
+
+    Returns:
+        dict: Action-specific results
+
+    Raises:
+        ValueError: If action is invalid or required parameters are missing
+    """
+    valid_actions = {"list", "approve", "reject", "stats"}
+
+    if action not in valid_actions:
+        raise ValueError(
+            f"Invalid action: {action}. Must be one of: {', '.join(sorted(valid_actions))}"
+        )
+
+    # Validate update_id requirement for approve/reject
+    if action in {"approve", "reject"} and not update_id:
+        raise ValueError(f"update_id is required for {action} action")
+
+    # Use provided store or create default
+    if store is None:
+        store = PendingUpdateStore()
+    
+    # Execute action
+    if action == "list":
+        updates = store.list_pending()
+        # Convert dataclass instances to dicts for JSON serialization
+        updates_dict = [
+            {
+                "update_id": u.update_id,
+                "category": u.category,
+                "target_section": u.target_section,
+                "summary": u.summary,
+                "confidence": u.confidence,
+                "source_agent": u.source_agent,
+                "status": u.status,
+                "created_at": u.created_at,
+                "seen_count": u.seen_count,
+            }
+            for u in updates
+        ]
+        return {
+            "action": "list",
+            "updates": updates_dict,
+            "count": len(updates_dict)
+        }
+    
+    elif action == "approve":
+        # Approve the update
+        store.approve(update_id)
+        
+        # Apply the update to project-context.json
+        result = store.apply(update_id, context_path)
+        
+        return {
+            "action": "approve",
+            "update_id": update_id,
+            "applied": result.get("success", False),
+            "result": result
+        }
+    
+    elif action == "reject":
+        store.reject(update_id)
+        return {
+            "action": "reject",
+            "update_id": update_id,
+            "success": True
+        }
+    
+    elif action == "stats":
+        statistics = store.get_statistics()
+        return {
+            "action": "stats",
+            "statistics": statistics
+        }
+
+
+def main():
+    """CLI entry point for review operations."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(
+        description="Review pending context updates"
+    )
+    parser.add_argument(
+        "action",
+        choices=["list", "approve", "reject", "stats"],
+        help="Action to perform"
+    )
+    parser.add_argument(
+        "--update-id",
+        help="Update ID (required for approve/reject)"
+    )
+    parser.add_argument(
+        "--context-path",
+        help="Path to project-context.json"
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON (default behavior)"
+    )
+    
+    args = parser.parse_args()
+    
+    try:
+        # Execute review action
+        result = review_pending(
+            args.action,
+            args.update_id,
+            Path(args.context_path) if args.context_path else None
+        )
+        
+        # Output result as JSON
+        print(json.dumps(result, indent=2))
+        
+    except Exception as e:
+        # Output error as JSON
+        error_result = {
+            "error": str(e),
+            "action": args.action
+        }
+        print(json.dumps(error_result, indent=2))
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/tools/scan/__init__.py

@@ -0,0 +1,35 @@
+"""
+Scan Module: Modular project scanner for gaia-ops
+
+This module provides a pluggable scanner system that detects project stack,
+infrastructure, git configuration, tools, orchestration, and environment.
+Each scanner is a pure function (filesystem path -> JSON sections) that runs
+independently and in parallel. The system produces an agnostic project-context.json
+schema that works for any project type.
+
+Main components:
+- BaseScanner: Abstract base class for scanner modules
+- ScannerRegistry: Auto-discovery registry for scanner modules
+- ScanOrchestrator: Parallel scanner execution and result aggregation
+- ScanConfig: Scanner configuration and tool definitions
+"""
+
+import json as _json
+from pathlib import Path as _Path
+
+
+def _read_version() -> str:
+    """Read version from package.json (single source of truth)."""
+    try:
+        pkg_path = _Path(__file__).resolve().parent.parent.parent / "package.json"
+        with open(pkg_path) as f:
+            return _json.load(f)["version"]
+    except Exception:
+        return "unknown"
+
+
+__version__ = _read_version()
+
+__all__ = [
+    "__version__",
+]