@jaguilar87/gaia 5.0.0-rc1

package/tools/gaia_simulator/skills_mapper.py

@@ -0,0 +1,264 @@
+"""
+Skills mapper for gaia-ops agent/skill/surface/contract analysis.
+
+Builds a complete map of agents, their skills, the surfaces that route
+to them, and the contract permissions they hold. Flags orphan skills.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+
+_TOOLS_DIR = Path(__file__).resolve().parent.parent
+if str(_TOOLS_DIR) not in sys.path:
+    sys.path.insert(0, str(_TOOLS_DIR))
+
+from gaia_simulator.routing_simulator import _parse_frontmatter
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SkillMapping:
+    """Maps a skill to the agents that use it."""
+
+    skill_name: str
+    skill_path: str
+    used_by_agents: list[str]
+    is_orphan: bool  # no agent uses it
+
+
+@dataclass
+class AgentProfile:
+    """Full profile for an agent."""
+
+    agent_name: str
+    skills: list[str]
+    invocation_count: int  # from metrics/logs
+    surfaces: list[str]  # which surfaces route to this agent
+    read_sections: list[str]
+    write_sections: list[str]
+
+
+# ---------------------------------------------------------------------------
+# SkillsMapper
+# ---------------------------------------------------------------------------
+
+
+class SkillsMapper:
+    """Builds the complete map of agents, skills, surfaces, and contracts."""
+
+    def __init__(self, agents_dir: Path, skills_dir: Path, config_dir: Path):
+        """Initialize the mapper.
+
+        Args:
+            agents_dir: Path to agents/ directory with agent .md files.
+            skills_dir: Path to skills/ directory with skill subdirectories.
+            config_dir: Path to config/ directory with routing and contracts.
+        """
+        self._agents_dir = agents_dir
+        self._skills_dir = skills_dir
+        self._config_dir = config_dir
+
+        # Load agent frontmatter -- skip README.md (not an agent definition)
+        self._agent_frontmatter: dict[str, dict[str, Any]] = {}
+        if agents_dir.is_dir():
+            for md_file in sorted(agents_dir.glob("*.md")):
+                if md_file.name.upper() == "README.MD":
+                    continue
+                content = md_file.read_text(encoding="utf-8", errors="replace")
+                fm = _parse_frontmatter(content)
+                name = fm.get("name", md_file.stem)
+                self._agent_frontmatter[name] = fm
+
+        # Load surface routing config
+        routing_file = config_dir / "surface-routing.json"
+        if routing_file.is_file():
+            self._routing_config = json.loads(
+                routing_file.read_text(encoding="utf-8")
+            )
+        else:
+            self._routing_config = {"surfaces": {}}
+
+        # Load contracts
+        contracts_file = config_dir / "context-contracts.json"
+        if contracts_file.is_file():
+            self._contracts = json.loads(
+                contracts_file.read_text(encoding="utf-8")
+            )
+        else:
+            self._contracts = {"agents": {}}
+
+        # Discover all skill directories
+        self._all_skills: list[str] = []
+        if skills_dir.is_dir():
+            for item in sorted(skills_dir.iterdir()):
+                if item.is_dir() and not item.name.startswith("."):
+                    self._all_skills.append(item.name)
+
+    def _get_surfaces_for_agent(self, agent_name: str) -> list[str]:
+        """Find which surfaces route to a given agent."""
+        surfaces: list[str] = []
+        for surface_name, surface_cfg in self._routing_config.get(
+            "surfaces", {}
+        ).items():
+            if surface_cfg.get("primary_agent") == agent_name:
+                surfaces.append(surface_name)
+        return surfaces
+
+    def get_agent_profiles(self) -> list[AgentProfile]:
+        """Full profile for each agent.
+
+        Returns:
+            List of AgentProfile instances, one per agent.
+        """
+        profiles: list[AgentProfile] = []
+
+        for agent_name, fm in self._agent_frontmatter.items():
+            skills = fm.get("skills", [])
+            surfaces = self._get_surfaces_for_agent(agent_name)
+            contract = self._contracts.get("agents", {}).get(agent_name, {})
+            read_sections = contract.get("read", [])
+            write_sections = contract.get("write", [])
+
+            profiles.append(
+                AgentProfile(
+                    agent_name=agent_name,
+                    skills=skills,
+                    invocation_count=0,
+                    surfaces=surfaces,
+                    read_sections=read_sections,
+                    write_sections=write_sections,
+                )
+            )
+
+        return profiles
+
+    def get_skill_mappings(self) -> list[SkillMapping]:
+        """Which skills are used by which agents. Flag orphans.
+
+        Returns:
+            List of SkillMapping instances, one per discovered skill.
+        """
+        # Build reverse map: skill -> list of agents
+        skill_to_agents: dict[str, list[str]] = {}
+        for agent_name, fm in self._agent_frontmatter.items():
+            for skill in fm.get("skills", []):
+                skill_to_agents.setdefault(skill, []).append(agent_name)
+
+        mappings: list[SkillMapping] = []
+        for skill_name in self._all_skills:
+            skill_path = str(self._skills_dir / skill_name)
+            used_by = skill_to_agents.get(skill_name, [])
+            mappings.append(
+                SkillMapping(
+                    skill_name=skill_name,
+                    skill_path=skill_path,
+                    used_by_agents=used_by,
+                    is_orphan=len(used_by) == 0,
+                )
+            )
+
+        return mappings
+
+    def get_unused_skills(self) -> list[str]:
+        """Skills that no agent references.
+
+        Returns:
+            List of orphan skill names.
+        """
+        return [m.skill_name for m in self.get_skill_mappings() if m.is_orphan]
+
+    def enrich_from_logs(self, metrics_path: Path) -> dict[str, Any]:
+        """Cross-reference with production metrics.
+
+        Reads audit-*.jsonl files to count agent invocations.
+
+        Args:
+            metrics_path: Path to directory containing audit JSONL files.
+
+        Returns:
+            Dict with agent_invocations and skill_loads counts.
+        """
+        agent_counts: dict[str, int] = {}
+        skill_counts: dict[str, int] = {}
+
+        if not metrics_path.is_dir():
+            return {"agent_invocations": agent_counts, "skill_loads": skill_counts}
+
+        for jsonl_file in sorted(metrics_path.glob("audit-*.jsonl")):
+            for line in jsonl_file.read_text(
+                encoding="utf-8", errors="replace"
+            ).splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except (json.JSONDecodeError, ValueError):
+                    continue
+                agent = record.get("agent", record.get("subagent_type", ""))
+                if agent:
+                    agent_counts[agent] = agent_counts.get(agent, 0) + 1
+
+        # Map agent counts to skill counts
+        for agent_name, count in agent_counts.items():
+            fm = self._agent_frontmatter.get(agent_name, {})
+            for skill in fm.get("skills", []):
+                skill_counts[skill] = skill_counts.get(skill, 0) + count
+
+        return {"agent_invocations": agent_counts, "skill_loads": skill_counts}
+
+    def format_report(self) -> str:
+        """Human-readable report of the agent/skill/surface map.
+
+        Returns:
+            Formatted multi-line string.
+        """
+        lines = []
+        lines.append("=" * 60)
+        lines.append("SKILLS MAPPER REPORT")
+        lines.append("=" * 60)
+
+        # Agent -> Skills table
+        lines.append("")
+        lines.append("AGENT -> SKILLS:")
+        lines.append("-" * 40)
+        for profile in self.get_agent_profiles():
+            skills_str = ", ".join(profile.skills) or "(none)"
+            surfaces_str = ", ".join(profile.surfaces) or "(none)"
+            lines.append("  " + profile.agent_name + ":")
+            lines.append("    Skills:   " + skills_str)
+            lines.append("    Surfaces: " + surfaces_str)
+            lines.append("    Read:     " + str(len(profile.read_sections)) + " sections")
+            lines.append("    Write:    " + str(len(profile.write_sections)) + " sections")
+
+        # Skill -> Agents table (reverse)
+        lines.append("")
+        lines.append("SKILL -> AGENTS:")
+        lines.append("-" * 40)
+        for mapping in self.get_skill_mappings():
+            agents_str = ", ".join(mapping.used_by_agents) or "ORPHAN"
+            orphan_tag = " [ORPHAN]" if mapping.is_orphan else ""
+            lines.append("  " + mapping.skill_name + ": " + agents_str + orphan_tag)
+
+        # Orphan skills
+        unused = self.get_unused_skills()
+        if unused:
+            lines.append("")
+            lines.append("ORPHAN SKILLS (" + str(len(unused)) + "):")
+            lines.append("-" * 40)
+            for skill in unused:
+                lines.append("  - " + skill)
+
+        lines.append("=" * 60)
+        return chr(10).join(lines)

package/tools/memory/__init__.py

@@ -0,0 +1,20 @@
+"""
+Episodic Memory Module for GAIA-OPS
+
+This module provides episodic memory functionality for storing and retrieving
+historical context from user interactions.
+"""
+
+from .episodic import (
+    EpisodicMemory,
+    Episode,
+    search_episodic_memory
+)
+
+__all__ = [
+    'EpisodicMemory',
+    'Episode',
+    'search_episodic_memory'
+]
+
+__version__ = '1.0.0'

package/tools/memory/backfill_fts5.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+Backfill FTS5 search index from episodes.jsonl.
+
+Reads all episodes from the episodic memory JSONL file and indexes them
+into the FTS5 search store. Idempotent -- safe to run multiple times.
+
+Usage:
+    python3 tools/memory/backfill_fts5.py
+    python3 -m tools.memory.backfill_fts5
+"""
+
+import json
+import sys
+from pathlib import Path
+
+
+def _find_project_root() -> Path:
+    """Walk up from cwd to find the directory containing .claude/."""
+    current = Path.cwd()
+    for candidate in [current, *current.parents]:
+        if (candidate / ".claude").is_dir():
+            return candidate
+    return current
+
+
+def main() -> int:
+    project_root = _find_project_root()
+    episodes_path = project_root / ".claude" / "project-context" / "episodic-memory" / "episodes.jsonl"
+
+    if not episodes_path.exists():
+        print(f"ERROR: episodes.jsonl not found at {episodes_path}", file=sys.stderr)
+        return 1
+
+    # Import here so the script works when run from the project root
+    # (tools/memory is on the path via the module run, or cwd is project root)
+    try:
+        from tools.memory import search_store
+    except ImportError:
+        # Fallback: add project root to sys.path and retry
+        sys.path.insert(0, str(project_root))
+        from tools.memory import search_store
+
+    indexed = 0
+    skipped_event = 0
+    skipped_malformed = 0
+
+    with open(episodes_path, encoding="utf-8") as fh:
+        for lineno, raw_line in enumerate(fh, start=1):
+            line = raw_line.strip()
+            if not line:
+                continue
+
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError as exc:
+                print(f"WARNING: line {lineno} is malformed JSON, skipping ({exc})", file=sys.stderr)
+                skipped_malformed += 1
+                continue
+
+            # Skip outcome_update / relationship_added events
+            if "event_type" in record:
+                skipped_event += 1
+                continue
+
+            episode_id = record.get("episode_id") or record.get("id", "")
+            if not episode_id:
+                print(f"WARNING: line {lineno} has no episode_id/id, skipping", file=sys.stderr)
+                skipped_malformed += 1
+                continue
+
+            prompt = record.get("prompt", "")
+            enriched_prompt = record.get("enriched_prompt", "")
+            title = record.get("title", "")
+
+            raw_tags = record.get("tags", [])
+            if isinstance(raw_tags, list):
+                tags = " ".join(str(t) for t in raw_tags)
+            else:
+                tags = str(raw_tags)
+
+            search_store.index_episode(
+                episode_id=episode_id,
+                prompt=prompt,
+                enriched_prompt=enriched_prompt,
+                tags=tags,
+                title=title,
+            )
+            indexed += 1
+
+            if indexed % 1000 == 0:
+                print(f"Progress: {indexed} episodes indexed...")
+
+    if indexed == 0 and skipped_event == 0 and skipped_malformed == 0:
+        print("0 episodes indexed (file is empty)")
+    else:
+        print(
+            f"Done: {indexed} episodes indexed"
+            + (f", {skipped_event} event records skipped" if skipped_event else "")
+            + (f", {skipped_malformed} malformed lines skipped" if skipped_malformed else "")
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/memory/conflict_detector.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""
+Conflict Detection Module for GAIA-OPS Memory Files
+
+Scans memory .md files for contradictions by:
+1. Loading all .md files from the memory directory
+2. Computing Jaccard similarity on word sets (after stopword removal)
+3. For similar file pairs (Jaccard > threshold), checking for polarity contradictions
+4. Returning structured conflict reports
+
+Inspired by hippo-memory's conflict detection heuristics.
+"""
+
+import os
+import re
+from pathlib import Path
+from typing import Optional
+
+
+# 50-word stopword list
+STOPWORDS = frozenset([
+    "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
+    "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
+    "being", "have", "has", "had", "do", "does", "did", "will", "would",
+    "could", "should", "may", "might", "shall", "that", "this", "these",
+    "those", "it", "its", "as", "if", "than", "then", "so", "not", "no",
+    "all", "any", "each", "both", "more",
+])
+
+
+def _tokenize(text: str) -> set:
+    """Tokenize text into lowercase words, removing stopwords."""
+    words = re.findall(r"[a-zA-Z0-9]+", text.lower())
+    return {w for w in words if w not in STOPWORDS and len(w) > 1}
+
+
+def _jaccard(set_a: set, set_b: set) -> float:
+    """Compute Jaccard similarity between two word sets."""
+    if not set_a and not set_b:
+        return 0.0
+    intersection = len(set_a & set_b)
+    union = len(set_a | set_b)
+    return intersection / union if union > 0 else 0.0
+
+
+def _extract_lines(text: str) -> list:
+    """Return non-empty, non-heading stripped lines from text."""
+    return [line.strip() for line in text.splitlines() if line.strip()]
+
+
+def _check_polarity_contradictions(lines_a: list, lines_b: list) -> list:
+    """
+    Check for polarity contradictions between two sets of lines.
+
+    Patterns checked:
+    - "use X" vs "do not use X" / "don't use X"
+    - "enabled" vs "disabled"
+    - "always" vs "never"
+    - "Windows" vs "WSL" / "Linux" in same context
+    - Version mismatches (e.g., "v4" vs "v5")
+    """
+    conflicts = []
+
+    # Pattern 1: "use X" vs "do not use X" / "don't use X"
+    use_pattern = re.compile(r"\buse\s+(\w+)", re.IGNORECASE)
+    no_use_pattern = re.compile(r"\b(?:do\s+not|don\'t|never\s+use)\s+(\w+)", re.IGNORECASE)
+
+    uses_a = {m.group(1).lower(): line for line in lines_a for m in [use_pattern.search(line)] if m}
+    uses_b = {m.group(1).lower(): line for line in lines_b for m in [use_pattern.search(line)] if m}
+    no_uses_a = {m.group(1).lower(): line for line in lines_a for m in [no_use_pattern.search(line)] if m}
+    no_uses_b = {m.group(1).lower(): line for line in lines_b for m in [no_use_pattern.search(line)] if m}
+
+    for term, line_a in uses_a.items():
+        if term in no_uses_b:
+            conflicts.append({
+                "line_a": line_a,
+                "line_b": no_uses_b[term],
+                "reason": f"'use {term}' contradicts 'do not use {term}'",
+            })
+
+    for term, line_b in uses_b.items():
+        if term in no_uses_a:
+            conflicts.append({
+                "line_a": no_uses_a[term],
+                "line_b": line_b,
+                "reason": f"'use {term}' contradicts 'do not use {term}'",
+            })
+
+    # Pattern 2: "enabled" vs "disabled"
+    enabled_pattern = re.compile(r"\benabled\b", re.IGNORECASE)
+    disabled_pattern = re.compile(r"\bdisabled\b", re.IGNORECASE)
+
+    enabled_lines_a = [l for l in lines_a if enabled_pattern.search(l)]
+    disabled_lines_a = [l for l in lines_a if disabled_pattern.search(l)]
+    enabled_lines_b = [l for l in lines_b if enabled_pattern.search(l)]
+    disabled_lines_b = [l for l in lines_b if disabled_pattern.search(l)]
+
+    for line_a in enabled_lines_a:
+        for line_b in disabled_lines_b:
+            # Only flag if they share a common significant keyword
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "enabled vs disabled on shared topic",
+                })
+
+    for line_a in disabled_lines_a:
+        for line_b in enabled_lines_b:
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "disabled vs enabled on shared topic",
+                })
+
+    # Pattern 3: "always" vs "never"
+    always_pattern = re.compile(r"\balways\b", re.IGNORECASE)
+    never_pattern = re.compile(r"\bnever\b", re.IGNORECASE)
+
+    always_lines_a = [l for l in lines_a if always_pattern.search(l)]
+    never_lines_a = [l for l in lines_a if never_pattern.search(l)]
+    always_lines_b = [l for l in lines_b if always_pattern.search(l)]
+    never_lines_b = [l for l in lines_b if never_pattern.search(l)]
+
+    for line_a in always_lines_a:
+        for line_b in never_lines_b:
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "always vs never on shared topic",
+                })
+
+    for line_a in never_lines_a:
+        for line_b in always_lines_b:
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "never vs always on shared topic",
+                })
+
+    # Pattern 4: "Windows" vs "WSL/Linux" in same context
+    windows_pattern = re.compile(r"\bWindows\b")
+    wsl_pattern = re.compile(r"\b(?:WSL|Linux)\b")
+
+    windows_lines_a = [l for l in lines_a if windows_pattern.search(l)]
+    wsl_lines_a = [l for l in lines_a if wsl_pattern.search(l)]
+    windows_lines_b = [l for l in lines_b if windows_pattern.search(l)]
+    wsl_lines_b = [l for l in lines_b if wsl_pattern.search(l)]
+
+    # Cross-file: file_a says Windows, file_b says WSL for same context
+    for line_a in windows_lines_a:
+        for line_b in wsl_lines_b:
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "Windows vs WSL/Linux on shared topic",
+                })
+
+    for line_a in wsl_lines_a:
+        for line_b in windows_lines_b:
+            shared = _tokenize(line_a) & _tokenize(line_b)
+            if shared:
+                conflicts.append({
+                    "line_a": line_a,
+                    "line_b": line_b,
+                    "reason": "WSL/Linux vs Windows on shared topic",
+                })
+
+    # Pattern 5: Version mismatches (e.g., "v4" vs "v5")
+    version_pattern = re.compile(r"\bv(\d+)\b", re.IGNORECASE)
+
+    def extract_versions(lines: list) -> dict:
+        """Map version number -> line for lines containing a version."""
+        result = {}
+        for line in lines:
+            for m in version_pattern.finditer(line):
+                v = int(m.group(1))
+                result[v] = line
+        return result
+
+    versions_a = extract_versions(lines_a)
+    versions_b = extract_versions(lines_b)
+
+    for va, line_a in versions_a.items():
+        for vb, line_b in versions_b.items():
+            if va != vb:
+                # Only flag if the context words (excluding the version itself) overlap
+                tokens_a = _tokenize(re.sub(r"\bv\d+\b", "", line_a, flags=re.IGNORECASE))
+                tokens_b = _tokenize(re.sub(r"\bv\d+\b", "", line_b, flags=re.IGNORECASE))
+                shared = tokens_a & tokens_b
+                if shared:
+                    conflicts.append({
+                        "line_a": line_a,
+                        "line_b": line_b,
+                        "reason": f"version mismatch: v{va} vs v{vb} on shared topic",
+                    })
+
+    # Deduplicate: remove identical (line_a, line_b, reason) triples
+    seen = set()
+    deduped = []
+    for c in conflicts:
+        key = (c["line_a"], c["line_b"], c["reason"])
+        if key not in seen:
+            seen.add(key)
+            deduped.append(c)
+
+    return deduped
+
+
+def _load_memory_files(memory_dir: Path) -> dict:
+    """
+    Load all .md files from memory_dir.
+    Returns dict mapping filename -> file content string.
+    """
+    files = {}
+    try:
+        for entry in memory_dir.iterdir():
+            if entry.is_file() and entry.suffix == ".md":
+                try:
+                    content = entry.read_text(encoding="utf-8", errors="replace")
+                    files[str(entry)] = content
+                except (PermissionError, OSError):
+                    pass
+    except (PermissionError, OSError, FileNotFoundError):
+        pass
+    return files
+
+
+def detect_conflicts(
+    memory_dir: Optional[Path] = None,
+    threshold: float = 0.3,
+) -> list:
+    """
+    Scan memory .md files for contradictions.
+
+    Args:
+        memory_dir: Path to directory containing memory .md files.
+                    Defaults to ~/.claude/projects/-home-jorge-ws-me/memory/
+        threshold:  Jaccard similarity threshold above which pairs are checked
+                    for polarity contradictions. Default 0.3.
+
+    Returns:
+        List of dicts with keys:
+            file_a    - absolute path of first file
+            file_b    - absolute path of second file
+            similarity - Jaccard similarity score (float)
+            conflicts  - list of {"line_a", "line_b", "reason"} dicts
+    """
+    if memory_dir is None:
+        default = Path.home() / ".claude" / "projects" / "-home-jorge-ws-me" / "memory"
+        memory_dir = default
+
+    memory_dir = Path(memory_dir)
+
+    file_contents = _load_memory_files(memory_dir)
+    if not file_contents:
+        return []
+
+    file_paths = sorted(file_contents.keys())
+    word_sets = {fp: _tokenize(file_contents[fp]) for fp in file_paths}
+    file_lines = {fp: _extract_lines(file_contents[fp]) for fp in file_paths}
+
+    results = []
+
+    for i in range(len(file_paths)):
+        for j in range(i + 1, len(file_paths)):
+            fp_a = file_paths[i]
+            fp_b = file_paths[j]
+
+            sim = _jaccard(word_sets[fp_a], word_sets[fp_b])
+            if sim <= threshold:
+                continue
+
+            conflicts = _check_polarity_contradictions(
+                file_lines[fp_a],
+                file_lines[fp_b],
+            )
+
+            results.append({
+                "file_a": fp_a,
+                "file_b": fp_b,
+                "similarity": round(sim, 4),
+                "conflicts": conflicts,
+            })
+
+    return results