delimit-cli 3.15.13 → 4.0.0

package/gateway/ai/session_phoenix.py

@@ -1,371 +0,0 @@
-"""
-Session Phoenix — Cross-model session resurrection (LED-218).
-
-When a session dies from rate limits, context overflow, or model switch,
-the user runs `delimit revive` in any model to restore working state.
-
-Architecture:
-  capture_soul() -> ~/.delimit/souls/{project_hash}/{timestamp}.json
-  revive()       -> structured context blob any AI model can read
-
-Complements delimit_session_handoff (ledger state) by saving the
-working context: task, decisions, files, blockers, next steps.
-"""
-
-import hashlib
-import json
-import os
-import subprocess
-import time
-import uuid
-from dataclasses import asdict, dataclass, field
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-MAX_SOULS_PER_PROJECT = 10
-SOULS_BASE_DIR = Path.home() / ".delimit" / "souls"
-_capture_counter = 0  # Monotonic counter for sub-second ordering
-
-
-@dataclass
-class SessionSoul:
-    """Compressed session state that survives death."""
-
-    soul_id: str = ""
-    created_at: str = ""
-    source_model: str = "unknown"
-    project_path: str = ""
-
-    # What was being worked on
-    active_task: str = ""
-    task_status: str = "in_progress"  # in_progress, blocked, almost_done
-
-    # Key decisions made this session
-    decisions: List[str] = field(default_factory=list)
-
-    # Files touched
-    files_modified: List[str] = field(default_factory=list)
-    files_created: List[str] = field(default_factory=list)
-
-    # Context that matters
-    key_context: List[str] = field(default_factory=list)
-    blockers: List[str] = field(default_factory=list)
-    next_steps: List[str] = field(default_factory=list)
-
-    # Technical state
-    git_branch: str = ""
-    git_sha: str = ""
-    uncommitted_changes: int = 0
-
-    # Token stats
-    tokens_used: int = 0
-    context_fullness: float = 0.0
-
-
-def _project_hash(project_path: str) -> str:
-    """Stable hash for a project path, used as directory name."""
-    normalized = os.path.realpath(project_path)
-    return hashlib.sha256(normalized.encode()).hexdigest()[:12]
-
-
-def _project_dir(project_path: str) -> Path:
-    """Return the soul storage directory for a project."""
-    return SOULS_BASE_DIR / _project_hash(project_path)
-
-
-def _run_git(args: List[str], cwd: str = "") -> str:
-    """Run a git command and return stdout, or empty string on failure."""
-    try:
-        result = subprocess.run(
-            ["git"] + args,
-            capture_output=True,
-            text=True,
-            timeout=5,
-            cwd=cwd or None,
-        )
-        if result.returncode == 0:
-            return result.stdout.strip()
-    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
-        pass
-    return ""
-
-
-def _detect_git_state(project_path: str) -> Dict[str, Any]:
-    """Auto-detect git branch, sha, modified/created files, uncommitted count."""
-    cwd = project_path or os.getcwd()
-
-    branch = _run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
-    sha = _run_git(["rev-parse", "--short", "HEAD"], cwd=cwd)
-
-    # Uncommitted changes (staged + unstaged + untracked)
-    porcelain = _run_git(["status", "--porcelain"], cwd=cwd)
-    porcelain_lines = [l for l in porcelain.splitlines() if l.strip()] if porcelain else []
-    uncommitted = len(porcelain_lines)
-
-    # Files modified (tracked, staged or unstaged)
-    diff_names = _run_git(["diff", "--name-only", "HEAD"], cwd=cwd)
-    files_modified = [l.strip() for l in diff_names.splitlines() if l.strip()] if diff_names else []
-
-    # New untracked files
-    untracked_raw = _run_git(["ls-files", "--others", "--exclude-standard"], cwd=cwd)
-    files_created = [l.strip() for l in untracked_raw.splitlines() if l.strip()] if untracked_raw else []
-
-    return {
-        "git_branch": branch,
-        "git_sha": sha,
-        "uncommitted_changes": uncommitted,
-        "files_modified": files_modified,
-        "files_created": files_created,
-    }
-
-
-def capture_soul(
-    active_task: str = "",
-    decisions: Optional[List[str]] = None,
-    key_context: Optional[List[str]] = None,
-    blockers: Optional[List[str]] = None,
-    next_steps: Optional[List[str]] = None,
-    source_model: str = "unknown",
-    project_path: str = "",
-    task_status: str = "in_progress",
-    tokens_used: int = 0,
-    context_fullness: float = 0.0,
-) -> SessionSoul:
-    """Capture current session state as a soul and persist it to disk."""
-    project_path = project_path or os.getcwd()
-    git_state = _detect_git_state(project_path)
-
-    soul = SessionSoul(
-        soul_id=str(uuid.uuid4())[:8],
-        created_at=datetime.now(timezone.utc).isoformat(),
-        source_model=source_model,
-        project_path=project_path,
-        active_task=active_task,
-        task_status=task_status,
-        decisions=decisions or [],
-        files_modified=git_state["files_modified"],
-        files_created=git_state["files_created"],
-        key_context=key_context or [],
-        blockers=blockers or [],
-        next_steps=next_steps or [],
-        git_branch=git_state["git_branch"],
-        git_sha=git_state["git_sha"],
-        uncommitted_changes=git_state["uncommitted_changes"],
-        tokens_used=tokens_used,
-        context_fullness=context_fullness,
-    )
-
-    _store_soul(soul)
-    return soul
-
-
-def _store_soul(soul: SessionSoul) -> Path:
-    """Persist a soul to disk and maintain the latest pointer."""
-    global _capture_counter
-    proj_dir = _project_dir(soul.project_path)
-    proj_dir.mkdir(parents=True, exist_ok=True)
-
-    # Timestamp + monotonic counter for correct ordering within same second
-    _capture_counter += 1
-    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    filename = f"{ts}_{_capture_counter:06d}_{soul.soul_id}.json"
-    filepath = proj_dir / filename
-
-    data = asdict(soul)
-    filepath.write_text(json.dumps(data, indent=2))
-
-    # Update latest.json as a copy (symlinks can be fragile across systems)
-    latest = proj_dir / "latest.json"
-    latest.write_text(json.dumps(data, indent=2))
-
-    # Auto-prune to MAX_SOULS_PER_PROJECT (keep latest N by name sort)
-    _prune_souls(proj_dir)
-
-    return filepath
-
-
-def _prune_souls(proj_dir: Path) -> None:
-    """Keep only the latest MAX_SOULS_PER_PROJECT souls per project."""
-    soul_files = sorted(
-        [f for f in proj_dir.iterdir() if f.name != "latest.json" and f.suffix == ".json"],
-        key=lambda f: f.name,
-    )
-    while len(soul_files) > MAX_SOULS_PER_PROJECT:
-        oldest = soul_files.pop(0)
-        oldest.unlink(missing_ok=True)
-
-
-def _load_soul(path: Path) -> Optional[SessionSoul]:
-    """Load a soul from a JSON file."""
-    try:
-        data = json.loads(path.read_text())
-        return SessionSoul(**{k: v for k, v in data.items() if k in SessionSoul.__dataclass_fields__})
-    except (json.JSONDecodeError, TypeError, KeyError, OSError):
-        return None
-
-
-def list_souls(project_path: str = "") -> List[SessionSoul]:
-    """List all stored souls for a project, newest first."""
-    project_path = project_path or os.getcwd()
-    proj_dir = _project_dir(project_path)
-    if not proj_dir.exists():
-        return []
-
-    soul_files = sorted(
-        [f for f in proj_dir.iterdir() if f.name != "latest.json" and f.suffix == ".json"],
-        key=lambda f: f.name,
-        reverse=True,
-    )
-    souls = []
-    for f in soul_files:
-        soul = _load_soul(f)
-        if soul:
-            souls.append(soul)
-    return souls
-
-
-def get_latest_soul(project_path: str = "") -> Optional[SessionSoul]:
-    """Get the most recent soul for a project."""
-    project_path = project_path or os.getcwd()
-    latest = _project_dir(project_path) / "latest.json"
-    if latest.exists():
-        return _load_soul(latest)
-    return None
-
-
-def _format_revival(soul: SessionSoul) -> str:
-    """Format a soul into a readable context string for any AI model."""
-    lines = []
-    lines.append("=" * 60)
-    lines.append("SESSION PHOENIX -- Revived Session Context")
-    lines.append("=" * 60)
-    lines.append("")
-
-    lines.append(f"Soul ID:      {soul.soul_id}")
-    lines.append(f"Captured:     {soul.created_at}")
-    lines.append(f"Source Model: {soul.source_model}")
-    lines.append(f"Project:      {soul.project_path}")
-    lines.append("")
-
-    # Current task
-    lines.append("--- ACTIVE TASK ---")
-    if soul.active_task:
-        lines.append(f"  {soul.active_task}")
-        lines.append(f"  Status: {soul.task_status}")
-    else:
-        lines.append("  (none recorded)")
-    lines.append("")
-
-    # Decisions
-    if soul.decisions:
-        lines.append("--- KEY DECISIONS ---")
-        for d in soul.decisions:
-            lines.append(f"  - {d}")
-        lines.append("")
-
-    # Files
-    if soul.files_modified or soul.files_created:
-        lines.append("--- FILES CHANGED ---")
-        for f in soul.files_modified:
-            lines.append(f"  M {f}")
-        for f in soul.files_created:
-            lines.append(f"  + {f}")
-        lines.append("")
-
-    # Context
-    if soul.key_context:
-        lines.append("--- KEY CONTEXT ---")
-        for c in soul.key_context:
-            lines.append(f"  - {c}")
-        lines.append("")
-
-    # Blockers
-    if soul.blockers:
-        lines.append("--- BLOCKERS ---")
-        for b in soul.blockers:
-            lines.append(f"  ! {b}")
-        lines.append("")
-
-    # Next steps
-    if soul.next_steps:
-        lines.append("--- NEXT STEPS ---")
-        for i, s in enumerate(soul.next_steps, 1):
-            lines.append(f"  {i}. {s}")
-        lines.append("")
-
-    # Git state
-    lines.append("--- GIT STATE ---")
-    lines.append(f"  Branch: {soul.git_branch or '(unknown)'}")
-    lines.append(f"  SHA:    {soul.git_sha or '(unknown)'}")
-    lines.append(f"  Uncommitted changes: {soul.uncommitted_changes}")
-    lines.append("")
-
-    # Token stats
-    if soul.tokens_used or soul.context_fullness:
-        lines.append("--- SESSION STATS ---")
-        if soul.tokens_used:
-            lines.append(f"  Tokens used: ~{soul.tokens_used:,}")
-        if soul.context_fullness:
-            lines.append(f"  Context fullness: {soul.context_fullness:.0%}")
-        lines.append("")
-
-    lines.append("=" * 60)
-    return "\n".join(lines)
-
-
-def revive(project_path: str = "", soul_id: str = "") -> Dict[str, Any]:
-    """Revive the latest session soul for this project.
-
-    Returns a structured dict with both the raw soul data and a
-    formatted context string that can be injected into any model.
-    """
-    project_path = project_path or os.getcwd()
-
-    if soul_id:
-        # Search for a specific soul by ID
-        for soul in list_souls(project_path):
-            if soul.soul_id == soul_id:
-                return {
-                    "status": "revived",
-                    "soul": asdict(soul),
-                    "context": _format_revival(soul),
-                }
-        return {
-            "status": "not_found",
-            "message": f"No soul with ID '{soul_id}' found for project {project_path}",
-        }
-
-    # Get latest
-    soul = get_latest_soul(project_path)
-    if not soul:
-        return {
-            "status": "no_souls",
-            "message": f"No session souls found for {project_path}. Nothing to revive.",
-            "hint": "Use delimit_soul_capture to save session state before ending.",
-        }
-
-    return {
-        "status": "revived",
-        "soul": asdict(soul),
-        "context": _format_revival(soul),
-    }
-
-
-def should_auto_capture(
-    context_fullness: float = 0.0,
-    session_age_minutes: int = 0,
-    last_capture_minutes_ago: int = -1,
-) -> bool:
-    """Determine if we should auto-capture a soul.
-
-    Triggers:
-    - Context > 70% full
-    - Session > 30 minutes old with no capture in the last 15 minutes
-    - Explicit session end (handled by caller, not this function)
-    """
-    if context_fullness >= 0.7:
-        return True
-    if session_age_minutes >= 30 and (last_capture_minutes_ago < 0 or last_capture_minutes_ago >= 15):
-        return True
-    return False

package/gateway/ai/toolcard_cache.py

@@ -1,327 +0,0 @@
-"""
-Toolcard Delta Cache — LED-219
-
-MCP servers dump full tool definitions every session. GitHub's MCP server
-alone sends 40K+ tokens of tool schemas. This module stores hashed tool
-schemas and only surfaces diffs when schemas change, cutting token waste
-on tool definitions dramatically.
-
-This is a MEASUREMENT tool first — it shows the savings potential. The
-actual MCP protocol optimization to send compressed schemas is a separate
-step.
-
-Architecture:
-  - SHA256 hash of each tool's canonical schema (name + description + parameters)
-  - Persistent JSON cache at ~/.delimit/toolcard_cache.json
-  - Per-session JSONL logs at ~/.delimit/toolcard_sessions/{date}.jsonl
-  - Thread-safe via atomic writes (write to tmp, rename)
-  - No external dependencies — stdlib only
-"""
-
-import hashlib
-import json
-import logging
-import os
-import tempfile
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-logger = logging.getLogger("delimit.toolcard_cache")
-
-CACHE_FILE = Path.home() / ".delimit" / "toolcard_cache.json"
-SESSION_DIR = Path.home() / ".delimit" / "toolcard_sessions"
-
-
-def _canonical_json(obj: Any) -> str:
-    """Produce a deterministic JSON string for hashing."""
-    return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
-
-
-def _hash_schema(tool: Dict[str, Any]) -> str:
-    """SHA256 hash of a tool's canonical schema (name + description + parameters)."""
-    canonical = {
-        "name": tool.get("name", ""),
-        "description": tool.get("description", ""),
-        "parameters": tool.get("parameters", {}),
-    }
-    return hashlib.sha256(_canonical_json(canonical).encode("utf-8")).hexdigest()
-
-
-def _estimate_tokens(obj: Any) -> int:
-    """Estimate token count: len(JSON) / 4 (standard approximation)."""
-    return max(1, len(_canonical_json(obj)) // 4)
-
-
-def _atomic_write_json(path: Path, data: Any) -> None:
-    """Write JSON atomically: write to temp file, then rename."""
-    path.parent.mkdir(parents=True, exist_ok=True)
-    fd, tmp_path = tempfile.mkstemp(
-        dir=str(path.parent), suffix=".tmp", prefix=".toolcard_"
-    )
-    try:
-        with os.fdopen(fd, "w") as f:
-            json.dump(data, f, indent=2, default=str)
-        os.replace(tmp_path, str(path))
-    except Exception:
-        # Clean up temp file on failure
-        try:
-            os.unlink(tmp_path)
-        except OSError:
-            pass
-        raise
-
-
-class ToolcardCache:
-    """Hashed tool schema registry. Sends full schemas on first session, diffs after."""
-
-    def __init__(self, cache_file: Optional[Path] = None, session_dir: Optional[Path] = None):
-        self._cache_file = cache_file or CACHE_FILE
-        self._session_dir = session_dir or SESSION_DIR
-        self.cache: Dict[str, Dict[str, Any]] = self._load()
-        # Per-session tracking
-        self._session_start = datetime.now(timezone.utc).isoformat()
-        self._session_calls: Dict[str, int] = {}  # tool_name -> call count
-        self._session_registered = 0
-        self._session_hits = 0
-        self._session_misses = 0
-
-    def _load(self) -> Dict[str, Dict[str, Any]]:
-        """Load cache from disk. Returns empty dict if missing or corrupt."""
-        try:
-            if self._cache_file.exists():
-                with open(self._cache_file, "r") as f:
-                    data = json.load(f)
-                if isinstance(data, dict):
-                    return data
-        except (json.JSONDecodeError, OSError) as e:
-            logger.warning("Toolcard cache load failed: %s", e)
-        return {}
-
-    def _save(self) -> None:
-        """Persist cache to disk atomically."""
-        _atomic_write_json(self._cache_file, self.cache)
-
-    def register_tools(self, tools: List[Dict[str, Any]]) -> Dict[str, Any]:
-        """Register tool schemas. Returns only NEW or CHANGED tools.
-
-        Args:
-            tools: List of tool schema dicts, each with 'name', 'description', 'parameters'.
-
-        Returns:
-            Dict with:
-                new_tools: list of tool schemas not previously cached
-                changed_tools: list of tool schemas whose hash differs
-                unchanged_tools: list of tool names (no schema, just names)
-                full_tokens: estimated tokens if all schemas were sent
-                delta_tokens: actual tokens for just new/changed
-                savings_pct: percentage reduction
-                saved_tokens: absolute token count saved
-        """
-        new_tools = []
-        changed_tools = []
-        unchanged_names = []
-        now = datetime.now(timezone.utc).isoformat()
-
-        for tool in tools:
-            name = tool.get("name", "")
-            if not name:
-                continue
-            h = _hash_schema(tool)
-
-            if name not in self.cache:
-                # New tool
-                new_tools.append(tool)
-                self.cache[name] = {
-                    "hash": h,
-                    "schema": tool,
-                    "first_seen": now,
-                    "last_changed": now,
-                }
-                self._session_misses += 1
-            elif self.cache[name]["hash"] != h:
-                # Changed tool
-                changed_tools.append(tool)
-                self.cache[name] = {
-                    "hash": h,
-                    "schema": tool,
-                    "first_seen": self.cache[name].get("first_seen", now),
-                    "last_changed": now,
-                }
-                self._session_misses += 1
-            else:
-                # Unchanged — cache hit
-                unchanged_names.append(name)
-                self._session_hits += 1
-
-        self._session_registered = len(tools)
-        self._save()
-
-        # Token calculations
-        full_tokens = sum(_estimate_tokens(t) for t in tools)
-        delta_schemas = new_tools + changed_tools
-        delta_tokens = sum(_estimate_tokens(t) for t in delta_schemas)
-        # Unchanged tools still need their names sent (compact summary)
-        delta_tokens += sum(len(n) // 4 + 1 for n in unchanged_names)
-
-        saved_tokens = max(0, full_tokens - delta_tokens)
-        savings_pct = round((saved_tokens / full_tokens * 100), 1) if full_tokens > 0 else 0.0
-
-        return {
-            "new_tools": new_tools,
-            "changed_tools": changed_tools,
-            "unchanged_tools": unchanged_names,
-            "full_tokens": full_tokens,
-            "delta_tokens": delta_tokens,
-            "savings_pct": savings_pct,
-            "saved_tokens": saved_tokens,
-            "total_registered": len(tools),
-            "cache_size": len(self.cache),
-        }
-
-    def get_delta(self, tool_names: List[str]) -> Dict[str, Any]:
-        """Return only schemas that changed since last check.
-
-        Args:
-            tool_names: List of tool names to check against the cache.
-
-        Returns:
-            Dict with cached (hit) and missing (miss) tools.
-        """
-        cached = []
-        missing = []
-
-        for name in tool_names:
-            if name in self.cache:
-                cached.append(name)
-                self._session_hits += 1
-            else:
-                missing.append(name)
-                self._session_misses += 1
-
-        return {
-            "cached": cached,
-            "missing": missing,
-            "cached_count": len(cached),
-            "missing_count": len(missing),
-            "hit_rate": round(len(cached) / len(tool_names) * 100, 1) if tool_names else 0.0,
-        }
-
-    def record_call(self, tool_name: str) -> None:
-        """Record that a tool was called in the current session."""
-        self._session_calls[tool_name] = self._session_calls.get(tool_name, 0) + 1
-
-    def get_stats(self) -> Dict[str, Any]:
-        """Return cache stats: total tools, cached, cache hit rate, token savings."""
-        total_checks = self._session_hits + self._session_misses
-        hit_rate = round(
-            (self._session_hits / total_checks * 100), 1
-        ) if total_checks > 0 else 0.0
-
-        # Estimate total cached schema tokens
-        cached_tokens = sum(
-            _estimate_tokens(entry.get("schema", {}))
-            for entry in self.cache.values()
-        )
-
-        # Most called tools this session
-        top_tools = sorted(
-            self._session_calls.items(), key=lambda x: x[1], reverse=True
-        )[:10]
-
-        return {
-            "total_cached_tools": len(self.cache),
-            "session_registered": self._session_registered,
-            "session_hits": self._session_hits,
-            "session_misses": self._session_misses,
-            "session_hit_rate": hit_rate,
-            "cached_schema_tokens": cached_tokens,
-            "session_tools_called": dict(top_tools),
-            "session_start": self._session_start,
-            "cache_file": str(self._cache_file),
-        }
-
-    def estimate_savings(self, tools: List[Dict[str, Any]]) -> Dict[str, Any]:
-        """Estimate token savings without modifying the cache.
-
-        Dry-run version of register_tools — shows what WOULD be saved.
-        """
-        hits = 0
-        misses = 0
-
-        for tool in tools:
-            name = tool.get("name", "")
-            if not name:
-                continue
-            h = _hash_schema(tool)
-            if name in self.cache and self.cache[name]["hash"] == h:
-                hits += 1
-            else:
-                misses += 1
-
-        full_tokens = sum(_estimate_tokens(t) for t in tools)
-        # If all cached, only names need sending
-        cached_names_tokens = sum(len(t.get("name", "")) // 4 + 1 for t in tools if t.get("name") in self.cache and self.cache.get(t.get("name"), {}).get("hash") == _hash_schema(t))
-        missed_tokens = sum(
-            _estimate_tokens(t) for t in tools
-            if t.get("name") not in self.cache or self.cache.get(t.get("name"), {}).get("hash") != _hash_schema(t)
-        )
-        delta_tokens = cached_names_tokens + missed_tokens
-        saved_tokens = max(0, full_tokens - delta_tokens)
-        savings_pct = round((saved_tokens / full_tokens * 100), 1) if full_tokens > 0 else 0.0
-
-        return {
-            "total_tools": len(tools),
-            "would_be_cached": hits,
-            "would_need_sending": misses,
-            "full_tokens": full_tokens,
-            "delta_tokens": delta_tokens,
-            "savings_pct": savings_pct,
-            "saved_tokens": saved_tokens,
-        }
-
-    def clear(self) -> Dict[str, Any]:
-        """Clear the cache. Forces full schema send next session."""
-        count = len(self.cache)
-        self.cache = {}
-        self._save()
-        return {
-            "cleared": count,
-            "message": f"Cleared {count} cached tool schemas. Next session will send full schemas.",
-        }
-
-    def flush_session(self) -> Dict[str, Any]:
-        """Write session stats to the per-session JSONL log."""
-        self._session_dir.mkdir(parents=True, exist_ok=True)
-        date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        session_file = self._session_dir / f"{date_str}.jsonl"
-
-        record = {
-            "session_start": self._session_start,
-            "flushed_at": datetime.now(timezone.utc).isoformat(),
-            "stats": self.get_stats(),
-        }
-
-        with open(session_file, "a") as f:
-            f.write(json.dumps(record, default=str) + "\n")
-
-        return {"written_to": str(session_file), "record": record}
-
-
-# Module-level singleton for use by server.py
-_cache_instance: Optional[ToolcardCache] = None
-
-
-def get_cache() -> ToolcardCache:
-    """Get or create the module-level cache singleton."""
-    global _cache_instance
-    if _cache_instance is None:
-        _cache_instance = ToolcardCache()
-    return _cache_instance
-
-
-def reset_cache() -> None:
-    """Reset the singleton (for testing)."""
-    global _cache_instance
-    _cache_instance = None