@jaguilar87/gaia-ops 4.4.0 → 4.7.2

package/dist/gaia-security/hooks/modules/audit/workflow_recorder.py

@@ -0,0 +1,296 @@
+"""
+Workflow metrics capture and persistence.
+
+Renamed from metrics_recorder.py for clarity.
+
+Provides:
+    - get_workflow_memory_dir(): Resolve workflow memory directory
+    - record(): Build metrics dict, write to JSONL
+"""
+
+import json
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..context.context_injector import build_context_telemetry_snapshot
+from ..core.paths import get_plugin_data_dir
+
+logger = logging.getLogger(__name__)
+
+
+def get_workflow_memory_dir() -> Path:
+    """
+    Get workflow memory directory path.
+
+    Resolution order:
+    1. WORKFLOW_MEMORY_BASE_PATH env var (testing override)
+    2. CLAUDE_PLUGIN_DATA / project-context / workflow-episodic-memory
+    3. .claude/project-context/workflow-episodic-memory (backward compat)
+    """
+    base_path = os.environ.get("WORKFLOW_MEMORY_BASE_PATH")
+    if base_path:
+        return Path(base_path) / "project-context" / "workflow-episodic-memory"
+    return get_plugin_data_dir() / "project-context" / "workflow-episodic-memory"
+
+
+def _append_jsonl(path: Path, payload: Dict[str, Any]) -> None:
+    """Append one JSON record per line."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "a") as f:
+        f.write(json.dumps(payload) + "\n")
+
+
+def _parse_frontmatter(text: str) -> Dict[str, Any]:
+    """Parse simple markdown frontmatter without external dependencies."""
+    if not text.startswith("---"):
+        return {}
+
+    try:
+        end = text.index("---", 3)
+    except ValueError:
+        return {}
+
+    frontmatter = text[3:end]
+    result: Dict[str, Any] = {}
+    current_key: Optional[str] = None
+    current_list: Optional[List[str]] = None
+
+    for line in frontmatter.splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+
+        if stripped.startswith("- ") and current_key and current_list is not None:
+            current_list.append(stripped[2:].strip())
+            continue
+
+        if ":" in stripped:
+            if current_key and current_list is not None:
+                result[current_key] = current_list
+
+            key, _, value = stripped.partition(":")
+            key = key.strip()
+            value = value.strip()
+
+            if value:
+                result[key] = value
+                current_key = key
+                current_list = None
+            else:
+                current_key = key
+                current_list = []
+
+    if current_key and current_list is not None:
+        result[current_key] = current_list
+
+    return result
+
+
+def _parse_tools_field(value: Any) -> List[str]:
+    """Normalize frontmatter tools into a list."""
+    if isinstance(value, list):
+        return [item for item in value if isinstance(item, str) and item.strip()]
+    if isinstance(value, str):
+        return [item.strip() for item in value.split(",") if item.strip()]
+    return []
+
+
+def _resolve_agent_file(agent_type: str) -> Optional[Path]:
+    """Resolve the markdown definition for a project agent."""
+    if not agent_type:
+        return None
+
+    candidates = [
+        Path(".claude/agents") / f"{agent_type}.md",
+        Path(__file__).resolve().parents[3] / "agents" / f"{agent_type}.md",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def load_agent_runtime_profile(agent_type: str) -> Dict[str, Any]:
+    """Load runtime-default metadata from an agent definition file."""
+    agent_file = _resolve_agent_file(agent_type)
+    if not agent_file:
+        return {
+            "agent": agent_type or "unknown",
+            "model": "",
+            "tools": [],
+            "skills": [],
+            "skills_count": 0,
+        }
+
+    frontmatter = _parse_frontmatter(agent_file.read_text())
+    skills = frontmatter.get("skills", [])
+    if not isinstance(skills, list):
+        skills = []
+
+    return {
+        "agent": agent_type or "unknown",
+        "model": frontmatter.get("model", ""),
+        "tools": _parse_tools_field(frontmatter.get("tools", [])),
+        "skills": skills,
+        "skills_count": len(skills),
+    }
+
+
+def record_agent_skill_snapshot(
+    agent_type: str,
+    session_context: Optional[Dict[str, Any]] = None,
+    task_description: str = "",
+) -> Dict[str, Any]:
+    """Persist a historical snapshot of an agent's runtime defaults."""
+    session_context = session_context or {}
+    profile = load_agent_runtime_profile(agent_type)
+    snapshot = {
+        "timestamp": session_context.get("timestamp", datetime.now().isoformat()),
+        "session_id": session_context.get("session_id", ""),
+        "agent": profile.get("agent", agent_type or "unknown"),
+        "task_description": task_description[:200],
+        "model": profile.get("model", ""),
+        "tools": profile.get("tools", []),
+        "skills": profile.get("skills", []),
+        "skills_count": profile.get("skills_count", 0),
+    }
+
+    try:
+        _append_jsonl(get_workflow_memory_dir() / "agent-skills.jsonl", snapshot)
+    except Exception as exc:
+        logger.debug("Could not persist agent skill snapshot: %s", exc)
+
+    return snapshot
+
+
+def record(
+    task_info: Dict[str, Any],
+    agent_output: str,
+    session_context: Dict[str, Any],
+    commands_executed: Optional[List[str]] = None,
+    context_update_result: Optional[Dict[str, Any]] = None,
+    anchor_hits: Optional[Dict[str, Any]] = None,
+    transcript_analysis: Optional[Any] = None,
+    compliance_result: Optional[Any] = None,
+) -> Dict[str, Any]:
+    """
+    Capture workflow execution metrics for analysis.
+
+    Args:
+        task_info: Task metadata
+        agent_output: Output from agent execution
+        session_context: Current session context
+        commands_executed: List of commands the agent executed.
+        context_update_result: Result of context update processing.
+        anchor_hits: Context anchor hit data.
+        transcript_analysis: Optional TranscriptAnalysis from transcript_analyzer.
+            When provided, real token counts and tool metrics are added to the
+            metrics dict alongside the existing output_tokens_approx.
+        compliance_result: Optional ComplianceScore from transcript_analyzer.
+            When provided, compliance_score and compliance_grade are added.
+
+    Returns:
+        Dict with duration, exit_code, agent, tier, etc.
+    """
+    # Duration cannot be reliably measured from within this hook because
+    # it fires only at agent completion (no start timestamp available).
+    duration_ms = None
+
+    # Use exit_code from task_info (derived from AGENT_STATUS block) instead
+    # of naive text matching which gives false positives on "No errors found".
+    exit_code = task_info.get("exit_code", 0)
+
+    # Approximate token count: 4 chars per token is a reliable heuristic for LLM output
+    output_tokens_approx = len(agent_output) // 4
+
+    commands_executed = commands_executed or []
+    context_update_result = context_update_result or {}
+    context_snapshot = build_context_telemetry_snapshot(
+        task_info.get("injected_context") or {}
+    )
+    default_skills_snapshot = load_agent_runtime_profile(task_info.get("agent", "unknown"))
+
+    metrics = {
+        "timestamp": session_context["timestamp"],
+        "session_id": session_context["session_id"],
+        "task_id": task_info.get("task_id", "unknown"),
+        "agent_id": task_info.get("agent_id", "unknown"),
+        "agent": task_info.get("agent", "unknown"),
+        "tier": task_info.get("tier", "T0"),
+        "duration_ms": duration_ms,
+        "exit_code": exit_code,
+        "plan_status": task_info.get("plan_status", ""),
+        "output_length": len(agent_output),
+        "output_tokens_approx": output_tokens_approx,
+        "tags": task_info.get("tags", []),
+        "prompt": task_info.get("description", ""),  # Store for episodic
+        "commands_executed": commands_executed,
+        "commands_executed_count": len(commands_executed),
+        "context_snapshot": context_snapshot,
+        "context_updated": bool(context_update_result.get("updated", False)),
+        "context_sections_updated": context_update_result.get("sections_updated", []),
+        "context_rejected_sections": context_update_result.get("rejected", []),
+        "default_skills_snapshot": default_skills_snapshot,
+        "context_anchor_hits": anchor_hits,
+        "context_anchor_hit_rate": anchor_hits.get("hit_rate") if anchor_hits else None,
+    }
+
+    # --- Transcript-analysis enrichment (T010) ---
+    if transcript_analysis is not None:
+        metrics["input_tokens"] = transcript_analysis.input_tokens
+        metrics["cache_creation_tokens"] = transcript_analysis.cache_creation_tokens
+        metrics["cache_read_tokens"] = transcript_analysis.cache_read_tokens
+        metrics["output_tokens_real"] = transcript_analysis.output_tokens
+        metrics["duration_ms"] = transcript_analysis.duration_ms
+        metrics["tool_call_count"] = transcript_analysis.tool_call_count
+        metrics["skills_injected"] = transcript_analysis.skills_injected
+        metrics["model_used"] = transcript_analysis.model
+        metrics["api_call_count"] = transcript_analysis.api_call_count
+
+    # --- Compliance enrichment (T010) ---
+    if compliance_result is not None:
+        metrics["compliance_score"] = compliance_result.total
+        metrics["compliance_grade"] = compliance_result.grade
+
+    run_snapshot = {
+        "timestamp": metrics["timestamp"],
+        "session_id": metrics["session_id"],
+        "task_id": metrics["task_id"],
+        "agent_id": metrics["agent_id"],
+        "agent": metrics["agent"],
+        "tier": metrics["tier"],
+        "plan_status": metrics["plan_status"],
+        "context_snapshot": context_snapshot,
+        "context_updated": metrics["context_updated"],
+        "context_sections_updated": metrics["context_sections_updated"],
+        "context_rejected_sections": metrics["context_rejected_sections"],
+        "default_skills_snapshot": default_skills_snapshot,
+        "context_anchor_hits": anchor_hits,
+        "context_anchor_hit_rate": anchor_hits.get("hit_rate") if anchor_hits else None,
+    }
+
+    try:
+        workflow_memory_dir = get_workflow_memory_dir()
+        workflow_memory_dir.mkdir(parents=True, exist_ok=True)
+        _append_jsonl(workflow_memory_dir / "run-snapshots.jsonl", run_snapshot)
+    except Exception as exc:
+        logger.debug("Could not persist run telemetry snapshot: %s", exc)
+
+    # Save to workflow memory (gated behind env var; default: no write)
+    if os.environ.get("GAIA_WRITE_WORKFLOW_METRICS") == "1":
+        workflow_memory_dir = get_workflow_memory_dir()
+        workflow_memory_dir.mkdir(parents=True, exist_ok=True)
+
+        metrics_file = workflow_memory_dir / "metrics.jsonl"
+        with open(metrics_file, "a") as f:
+            f.write(json.dumps(metrics) + "\n")
+
+    logger.debug(
+        "Captured workflow metrics: %s (duration: %sms, exit: %s, commands: %s)",
+        metrics["agent"], duration_ms, exit_code, len(commands_executed),
+    )
+
+    return metrics

package/dist/gaia-security/hooks/modules/context/__init__.py

@@ -0,0 +1,11 @@
+"""
+Context Management Module
+
+This module provides tools for managing context in agent conversations:
+- context_writer: Progressive enrichment of project-context.json via CONTEXT_UPDATE blocks
+- contracts_loader: Load context contracts, detect cloud provider, merge agent permissions
+- context_injector: Core context injection subsystem for project agents
+- context_freshness: Check staleness of project-context.json for SessionStart
+"""
+
+__all__ = []

package/dist/gaia-security/hooks/modules/context/anchor_tracker.py

@@ -0,0 +1,317 @@
+"""
+Context anchor hit tracking for project context effectiveness measurement.
+
+Extracts "anchors" (paths, names, IDs) from injected project context and checks
+whether the agent's early tool calls reference them. This measures whether agents
+use injected context as search anchors versus discovering on their own.
+
+Provides:
+    - extract_anchors(): Extract searchable anchors from a context payload
+    - save_anchors(): Persist anchors to a session-scoped temp file
+    - load_anchors(): Load persisted anchors for a session
+    - extract_tool_calls_from_transcript(): Parse early tool calls from JSONL transcript
+    - compute_anchor_hits(): Compare tool call args against anchors
+"""
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+# How many early tool calls to check
+MAX_TOOL_CALLS_TO_CHECK = 5
+
+# Tool types that have inspectable path/keyword arguments
+TRACKABLE_TOOLS = {"Glob", "Grep", "Read", "Bash"}
+
+# Minimum anchor length to avoid false-positive matches on short strings
+MIN_ANCHOR_LENGTH = 4
+
+
+def _anchors_dir() -> Path:
+    """Return the directory for anchor temp files."""
+    return Path("/tmp/gaia-context-anchors")
+
+
+def extract_anchors(context_payload: Dict[str, Any]) -> Set[str]:
+    """Extract searchable anchor strings from a context payload.
+
+    Walks the project knowledge sections and collects values from fields that
+    are likely to appear in agent tool calls: paths, names, IDs, clusters,
+    regions, namespaces, service accounts.
+
+    Args:
+        context_payload: The full context JSON payload injected into agent prompt.
+
+    Returns:
+        Set of anchor strings (paths, names, identifiers).
+    """
+    anchors: Set[str] = set()
+    contract = context_payload.get("project_knowledge", {})
+
+    # Anchor-worthy field name patterns
+    anchor_fields = re.compile(
+        r"(path|name|cluster|project|region|namespace|service|image|"
+        r"base_path|config_path|module_path|repository|bucket|sa$|"
+        r"service_account|pod_name|terragrunt_path)",
+        re.IGNORECASE,
+    )
+
+    def _walk(obj: Any, depth: int = 0) -> None:
+        if depth > 10:
+            return
+        if isinstance(obj, dict):
+            for key, value in obj.items():
+                if isinstance(value, str) and value and anchor_fields.search(key):
+                    # Normalize: strip leading ./ for path matching
+                    clean = value.lstrip("./")
+                    if len(clean) >= MIN_ANCHOR_LENGTH:
+                        anchors.add(clean)
+                elif isinstance(value, (dict, list)):
+                    _walk(value, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _walk(item, depth + 1)
+
+    _walk(contract)
+
+    # Also extract from top-level metadata
+    metadata = context_payload.get("metadata", {})
+    for key in ("project_id", "cluster_name", "region"):
+        val = metadata.get(key)
+        if isinstance(val, str) and len(val) >= MIN_ANCHOR_LENGTH:
+            anchors.add(val)
+
+    return anchors
+
+
+def save_anchors(session_id: str, agent_type: str, anchors: Set[str]) -> Optional[Path]:
+    """Persist anchors to a session+agent-scoped temp file.
+
+    Args:
+        session_id: Current session identifier.
+        agent_type: Agent name (e.g. "terraform-architect").
+        anchors: Set of anchor strings to save.
+
+    Returns:
+        Path to the saved file, or None on failure.
+    """
+    if not anchors:
+        return None
+
+    try:
+        anchor_dir = _anchors_dir()
+        anchor_dir.mkdir(parents=True, exist_ok=True)
+
+        safe_session = re.sub(r"[^a-zA-Z0-9_-]", "_", session_id or "unknown")[:32]
+        safe_agent = re.sub(r"[^a-zA-Z0-9_-]", "_", agent_type or "unknown")[:32]
+        anchor_file = anchor_dir / f"{safe_session}-{safe_agent}.json"
+
+        anchor_file.write_text(json.dumps(sorted(anchors)))
+        logger.debug(
+            "Saved %d anchors for %s/%s -> %s",
+            len(anchors), session_id, agent_type, anchor_file,
+        )
+        return anchor_file
+    except Exception as e:
+        logger.debug("Failed to save anchors: %s", e)
+        return None
+
+
+def load_anchors(session_id: str, agent_type: str) -> Set[str]:
+    """Load persisted anchors for a session+agent.
+
+    Args:
+        session_id: Current session identifier.
+        agent_type: Agent name.
+
+    Returns:
+        Set of anchor strings, or empty set if not found.
+    """
+    try:
+        safe_session = re.sub(r"[^a-zA-Z0-9_-]", "_", session_id or "unknown")[:32]
+        safe_agent = re.sub(r"[^a-zA-Z0-9_-]", "_", agent_type or "unknown")[:32]
+        anchor_file = _anchors_dir() / f"{safe_session}-{safe_agent}.json"
+
+        if not anchor_file.exists():
+            return set()
+
+        data = json.loads(anchor_file.read_text())
+        return set(data) if isinstance(data, list) else set()
+    except Exception as e:
+        logger.debug("Failed to load anchors: %s", e)
+        return set()
+
+
+def extract_tool_calls_from_transcript(
+    transcript_path: str,
+    max_calls: int = MAX_TOOL_CALLS_TO_CHECK,
+) -> List[Dict[str, Any]]:
+    """Extract the first N trackable tool calls from a Claude Code transcript JSONL.
+
+    Claude Code transcripts contain tool_use entries in the assistant messages
+    (content blocks with type "tool_use").
+
+    Args:
+        transcript_path: Path to the transcript JSONL file.
+        max_calls: Maximum number of tool calls to extract.
+
+    Returns:
+        List of dicts with keys: tool_name, arguments (dict), call_index (1-based).
+    """
+    if not transcript_path:
+        return []
+
+    try:
+        path = Path(transcript_path).expanduser()
+        if not path.exists():
+            return []
+
+        tool_calls: List[Dict[str, Any]] = []
+        call_index = 0
+
+        for line in path.read_text().strip().splitlines():
+            if not line.strip():
+                continue
+            if call_index >= max_calls:
+                break
+
+            try:
+                entry = json.loads(line)
+                msg = entry.get("message", entry)
+
+                if msg.get("role") != "assistant":
+                    continue
+
+                content = msg.get("content", [])
+                if not isinstance(content, list):
+                    continue
+
+                for block in content:
+                    if call_index >= max_calls:
+                        break
+                    if not isinstance(block, dict):
+                        continue
+                    if block.get("type") != "tool_use":
+                        continue
+
+                    tool_name = block.get("name", "")
+                    if tool_name not in TRACKABLE_TOOLS:
+                        continue
+
+                    call_index += 1
+                    tool_calls.append({
+                        "tool_name": tool_name,
+                        "arguments": block.get("input", {}),
+                        "call_index": call_index,
+                    })
+
+            except (json.JSONDecodeError, TypeError):
+                continue
+
+        return tool_calls
+
+    except Exception as e:
+        logger.debug("Failed to extract tool calls from transcript: %s", e)
+        return []
+
+
+def _extract_searchable_text(tool_name: str, arguments: Dict[str, Any]) -> str:
+    """Extract the searchable text from a tool call's arguments.
+
+    Returns a single string containing all path/keyword-relevant arguments
+    concatenated for substring matching.
+    """
+    parts: List[str] = []
+
+    if tool_name == "Glob":
+        parts.append(arguments.get("pattern", ""))
+        parts.append(arguments.get("path", ""))
+    elif tool_name == "Grep":
+        parts.append(arguments.get("pattern", ""))
+        parts.append(arguments.get("path", ""))
+        parts.append(arguments.get("glob", ""))
+    elif tool_name == "Read":
+        parts.append(arguments.get("file_path", ""))
+    elif tool_name == "Bash":
+        parts.append(arguments.get("command", ""))
+
+    return " ".join(p for p in parts if p)
+
+
+def compute_anchor_hits(
+    tool_calls: List[Dict[str, Any]],
+    anchors: Set[str],
+) -> Dict[str, Any]:
+    """Compare tool call arguments against known anchors.
+
+    For each tool call, checks if any anchor appears as a substring in the
+    tool's searchable arguments. This is a lightweight prefix/substring match.
+
+    Args:
+        tool_calls: List from extract_tool_calls_from_transcript().
+        anchors: Set of anchor strings from extract_anchors().
+
+    Returns:
+        Dict with hit tracking data.
+    """
+    if not tool_calls or not anchors:
+        return {
+            "total_checked": len(tool_calls),
+            "hits": 0,
+            "hit_rate": 0.0,
+            "details": [],
+        }
+
+    details: List[Dict[str, Any]] = []
+    hits = 0
+
+    for call in tool_calls:
+        searchable = _extract_searchable_text(call["tool_name"], call["arguments"])
+        matched_anchor: Optional[str] = None
+
+        if searchable:
+            for anchor in anchors:
+                if anchor in searchable:
+                    matched_anchor = anchor
+                    break
+
+        is_hit = matched_anchor is not None
+        if is_hit:
+            hits += 1
+
+        details.append({
+            "call_index": call["call_index"],
+            "tool": call["tool_name"],
+            "anchor": matched_anchor,
+            "hit": is_hit,
+        })
+
+    total = len(tool_calls)
+    return {
+        "total_checked": total,
+        "hits": hits,
+        "hit_rate": round(hits / total, 2) if total > 0 else 0.0,
+        "details": details,
+    }
+
+
+def cleanup_anchors(session_id: str, agent_type: str) -> None:
+    """Remove the anchor temp file after use.
+
+    Args:
+        session_id: Current session identifier.
+        agent_type: Agent name.
+    """
+    try:
+        safe_session = re.sub(r"[^a-zA-Z0-9_-]", "_", session_id or "unknown")[:32]
+        safe_agent = re.sub(r"[^a-zA-Z0-9_-]", "_", agent_type or "unknown")[:32]
+        anchor_file = _anchors_dir() / f"{safe_session}-{safe_agent}.json"
+        if anchor_file.exists():
+            anchor_file.unlink()
+            logger.debug("Cleaned up anchor file: %s", anchor_file)
+    except Exception as e:
+        logger.debug("Failed to cleanup anchors: %s", e)