@trac3er/oh-my-god 2.1.9 → 2.2.2

package/build/lib/runtime/tool_plan_gate.py

@@ -9,10 +9,14 @@ from uuid import uuid4
 import json
 
 from runtime.context_engine import ContextEngine
-from runtime.compliance_governor import evaluate_tool_compliance
+from runtime.compliance_governor import classify_bash_command_mode, evaluate_tool_compliance
 from runtime.complexity_scorer import score_complexity
-from runtime.release_run_coordinator import resolve_current_run_id as resolve_coordinator_run_id
+from runtime.release_run_coordinator import (
+    get_active_coordinator_run_id,
+    resolve_current_run_id as resolve_coordinator_run_id,
+)
 from runtime.runtime_contracts import read_run_state
+from runtime.test_intent_lock import verify_done_when, verify_lock
 
 
 _TOOL_KEYWORDS: dict[str, tuple[str, ...]] = {
@@ -21,6 +25,7 @@ _TOOL_KEYWORDS: dict[str, tuple[str, ...]] = {
     "omg_security_check": ("security", "scan", "vulnerability", "audit", "auth", "token", "secret"),
     "omg-control": ("policy", "governance", "release", "proof", "control plane"),
 }
+_MUTATION_TOOLS = frozenset({"write", "edit", "multiedit", "bash"})
 
 def build_tool_plan(
     goal: str,
@@ -102,33 +107,85 @@ def has_tool_plan_for_run(project_dir: str, run_id: str | None) -> bool:
     return any(plans_dir.glob(f"{run_prefix}-plan-*.json"))
 
 
-def tool_plan_gate_check(project_dir: str, run_id: str | None, tool: str) -> dict[str, object]:
-    if not run_id:
+def tool_plan_gate_check(
+    project_dir: str,
+    run_id: str | None,
+    tool: str,
+    tool_input: dict[str, object] | None = None,
+) -> dict[str, object]:
+    effective_run_id = get_active_coordinator_run_id(project_dir=project_dir) or run_id
+    if not effective_run_id:
         return {"status": "allowed", "reason": "run_id unavailable; skip tool plan gate"}
 
+    tool_input_obj = tool_input if isinstance(tool_input, dict) else {}
+    has_plan = has_tool_plan_for_run(project_dir, effective_run_id)
+
     context_packet = _resolve_bounded_context_packet(
         project_dir=project_dir,
-        run_id=run_id,
+        run_id=effective_run_id,
         fallback_context=None,
     )
     clarification = _clarification_status(context_packet)
     decision = evaluate_tool_compliance(
         project_dir=project_dir,
-        run_id=run_id,
+        run_id=effective_run_id,
         tool=tool,
-        has_tool_plan=has_tool_plan_for_run(project_dir, run_id),
+        has_tool_plan=has_plan,
         clarification_status=clarification,
+        tool_input=tool_input,
     )
     reason = str(decision.get("reason", ""))
     if decision.get("status") == "blocked":
-        _persist_gate_error(project_dir, run_id, tool, reason)
+        _persist_gate_error(project_dir, effective_run_id, tool, reason)
+        return {
+            **decision,
+            "run_id": effective_run_id,
+            "tool": tool,
+        }
+
+    if _is_mutation_capable_tool(tool, tool_input_obj) and not _is_docs_exemption(tool_input_obj):
+        lock_id = _extract_lock_id(tool_input_obj)
+        lock_verdict = verify_lock(project_dir, run_id=effective_run_id, lock_id=lock_id)
+        if lock_verdict.get("status") != "ok":
+            return {
+                "status": "blocked",
+                "authority": "test_intent_lock",
+                "reason": "test_intent_lock_required_before_mutation",
+                "run_id": effective_run_id,
+                "tool": tool,
+            }
+
+        metadata = _extract_metadata(tool_input_obj)
+        done_when_verdict = verify_done_when(metadata, run_id=effective_run_id)
+        if done_when_verdict.get("status") != "ok":
+            return {
+                "status": "blocked",
+                "authority": "done_when",
+                "reason": "done_when_required_before_mutation",
+                "run_id": effective_run_id,
+                "tool": tool,
+            }
+
     return {
         **decision,
-        "run_id": run_id,
+        "run_id": effective_run_id,
         "tool": tool,
     }
 
 
+def governed_tool_plan_gate_check(
+    project_dir: str,
+    run_id: str | None,
+    lane_name: str,
+    tool: str,
+) -> dict[str, object]:
+    decision = tool_plan_gate_check(project_dir=project_dir, run_id=run_id, tool=tool)
+    return {
+        **decision,
+        "lane_name": str(lane_name).strip().lower(),
+    }
+
+
 def _normalize_tools(available_tools: list[str]) -> list[str]:
     output: list[str] = []
     seen: set[str] = set()
@@ -334,3 +391,38 @@ def _clarification_status(context_packet: dict[str, object]) -> dict[str, object
         "clarification_prompt": prompt,
         "confidence": round(max(0.0, min(1.0, confidence)), 2),
     }
+
+
+def _is_mutation_capable_tool(tool: str, tool_input: dict[str, object]) -> bool:
+    token = str(tool or "").strip().lower()
+    if token == "bash":
+        return classify_bash_command_mode(str(tool_input.get("command", ""))) == "mutation"
+    return token in _MUTATION_TOOLS
+
+
+def _extract_metadata(tool_input: dict[str, object]) -> dict[str, object]:
+    metadata = tool_input.get("metadata")
+    if isinstance(metadata, dict):
+        return dict(metadata)
+    return {}
+
+
+def _extract_lock_id(tool_input: dict[str, object]) -> str | None:
+    lock_id = tool_input.get("lock_id")
+    if isinstance(lock_id, str) and lock_id.strip():
+        return lock_id.strip()
+    metadata = _extract_metadata(tool_input)
+    metadata_lock_id = metadata.get("lock_id")
+    if isinstance(metadata_lock_id, str) and metadata_lock_id.strip():
+        return metadata_lock_id.strip()
+    return None
+
+
+def _is_docs_exemption(tool_input: dict[str, object]) -> bool:
+    exemption = str(tool_input.get("exemption", "")).strip().lower()
+    if exemption == "docs":
+        return True
+    metadata = _extract_metadata(tool_input)
+    if metadata.get("exempt") is True and str(metadata.get("exemption", "")).strip().lower() == "docs":
+        return True
+    return False

package/build/lib/runtime/untrusted_content.py

@@ -133,7 +133,7 @@ def write_trust_evidence(inputs: list[dict[str, Any]], output_dir: str | Path) -
     normalized_inputs: list[dict[str, Any]] = []
     for item in inputs:
         data = dict(item)
-        trust_tier = str(data.get("_trust_tier", "research"))
+        trust_tier = str(data.get("trust_tier", data.get("_trust_tier", "research")))
         trust_label = str(data.get("_trust_label", "UNTRUSTED_EXTERNAL_CONTENT"))
         try:
             trust_score = float(data.get("_trust_score", 0.0))
@@ -203,20 +203,24 @@ def mark_untrusted_content(
     resolved_tier = normalize_trust_tier(tier) if tier is not None else trust_tier_for_source(source_type)
     entry = tag_content(
         {
-        "source_type": source_type,
-        "source_ref": source_ref,
-        "quarantined_fragments": quarantined,
-        "sanitized_content": sanitized,
+            "source_type": source_type,
+            "source_ref": source_ref,
+            "quarantined_fragments": quarantined,
+            "sanitized_content": sanitized,
         },
         resolved_tier,
     )
+    entry["trust_tier"] = resolved_tier.value
     entry["trust_score"] = entry.get("_trust_score", 0.0)
-    provenance = list(state.get("provenance", []))
-    provenance.append(entry)
+
     evidence_path = write_trust_evidence(
         [entry],
         output_dir=Path(project_dir) / EVIDENCE_REL_DIR,
     )
+    entry["evidence_artifact"] = evidence_path
+
+    provenance = list(state.get("provenance", []))
+    provenance.append(entry)
 
     prior_scores = state.get("trust_scores", {})
     next_external_score = min(
@@ -232,6 +236,7 @@ def mark_untrusted_content(
         "last_trust_tier": resolved_tier.value,
         "last_trust_label": entry.get("_trust_label"),
         "last_trust_score": entry.get("_trust_score"),
+        "last_evidence_artifact": evidence_path,
         "sanitized_content": sanitized,
         "quarantined_fragments": quarantined,
         "provenance": provenance[-20:],

package/build/lib/runtime/worker_watchdog.py

@@ -0,0 +1,482 @@
+"""Worker lifecycle watchdog — heartbeats, stall detection, termination, and replay evidence.
+
+Monitors workers by run_id, detects stalls, terminates unresponsive workers,
+cleans up stale worktrees, and emits replayable evidence for post-mortem analysis.
+
+State paths:
+- Heartbeats: .omg/state/worker-heartbeats/<run_id>.json
+- Replay evidence: .omg/evidence/subagents/<run_id>-replay.json
+"""
+from __future__ import annotations
+
+import json
+import os
+import signal
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+_DEFAULT_STALL_THRESHOLD_SECONDS = 60
+_SIGKILL_GRACE_SECONDS = 5
+
+
+def _project_dir(project_dir: str | None = None) -> Path:
+    if project_dir:
+        return Path(project_dir)
+    return Path(os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd()))
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _write_atomic_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_name(f"{path.name}.tmp")
+    tmp_path.write_text(
+        json.dumps(payload, indent=2, ensure_ascii=True) + "\n",
+        encoding="utf-8",
+    )
+    os.replace(tmp_path, path)
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _active_coordinator_run_id(project_dir: Path) -> str:
+    shadow_path = project_dir / ".omg" / "shadow" / "active-run"
+    if shadow_path.exists():
+        try:
+            shadow = shadow_path.read_text(encoding="utf-8").strip()
+        except OSError:
+            shadow = ""
+        if shadow:
+            return shadow
+
+    state_dir = project_dir / ".omg" / "state" / "release_run_coordinator"
+    if not state_dir.exists():
+        return ""
+    candidates = sorted(state_dir.glob("*.json"), key=lambda p: p.stat().st_mtime)
+    for path in reversed(candidates):
+        payload = _read_json(path)
+        run_id = str(payload.get("run_id", "")).strip()
+        if run_id:
+            return run_id
+    return ""
+
+
+class WorkerWatchdog:
+    """Monitor workers by run_id with heartbeats, stall detection, and termination."""
+
+    def __init__(self, project_dir: str | None = None):
+        self.project_dir = _project_dir(project_dir)
+
+    # --- Paths ---
+
+    def _heartbeat_dir(self) -> Path:
+        return self.project_dir / ".omg" / "state" / "worker-heartbeats"
+
+    def _heartbeat_path(self, run_id: str) -> Path:
+        return self._heartbeat_dir() / f"{run_id}.json"
+
+    def _replay_evidence_path(self, run_id: str) -> Path:
+        return self.project_dir / ".omg" / "evidence" / "subagents" / f"{run_id}-replay.json"
+
+    def _worktree_dir(self, run_id: str) -> Path:
+        return self.project_dir / ".omg" / "worktrees" / run_id
+
+    def _ownership_metadata(self, run_id: str) -> dict[str, Any]:
+        from runtime.merge_writer import get_merge_writer
+
+        active_run_id = _active_coordinator_run_id(self.project_dir)
+        merge_writer = get_merge_writer(str(self.project_dir))
+        merge_writer_details = merge_writer.check_authorization_details(run_id)
+        return {
+            "run_id": run_id,
+            "active_run_id": active_run_id,
+            "merge_writer": merge_writer_details,
+        }
+
+    # --- Heartbeats ---
+
+    def record_heartbeat(
+        self,
+        run_id: str,
+        worker_pid: int | None = None,
+        *,
+        status: str = "alive",
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Write heartbeat to .omg/state/worker-heartbeats/<run_id>.json.
+
+        Args:
+            run_id: The run identifier for this worker.
+            worker_pid: Optional PID of the worker subprocess.
+            status: Worker status (alive, stalled, terminated, etc.).
+            metadata: Optional extra metadata to include.
+
+        Returns:
+            The heartbeat record that was persisted.
+        """
+        path = self._heartbeat_path(run_id)
+        existing = _read_json(path)
+
+        heartbeat_count = int(existing.get("heartbeat_count", 0)) + 1
+
+        record: dict[str, Any] = {
+            "schema": "WorkerHeartbeat",
+            "run_id": run_id,
+            "worker_pid": worker_pid or existing.get("worker_pid"),
+            "status": status,
+            "heartbeat_count": heartbeat_count,
+            "last_heartbeat_at": _now_iso(),
+            "first_heartbeat_at": existing.get("first_heartbeat_at", _now_iso()),
+            "metadata": metadata or existing.get("metadata", {}),
+            "ownership": self._ownership_metadata(run_id),
+        }
+
+        _write_atomic_json(path, record)
+        return record
+
+    def read_heartbeat(self, run_id: str) -> dict[str, Any]:
+        """Read heartbeat state for a given run_id.
+
+        Returns:
+            Heartbeat record dict, or empty dict if not found.
+        """
+        return _read_json(self._heartbeat_path(run_id))
+
+    def list_heartbeats(self) -> list[dict[str, Any]]:
+        """List all heartbeat records."""
+        hb_dir = self._heartbeat_dir()
+        if not hb_dir.exists():
+            return []
+        results: list[dict[str, Any]] = []
+        try:
+            for path in sorted(hb_dir.glob("*.json")):
+                if path.name.endswith(".tmp"):
+                    continue
+                record = _read_json(path)
+                if record:
+                    results.append(record)
+        except OSError:
+            pass
+        return results
+
+    # --- Stall Detection ---
+
+    def check_stall(
+        self,
+        run_id: str,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> bool:
+        """Return True if worker is stalled (no heartbeat within threshold).
+
+        A worker is stalled when:
+        - It has a heartbeat record with status 'alive'
+        - Its last heartbeat timestamp exceeds the stall threshold
+
+        Returns False if no heartbeat exists (worker may not have started).
+        """
+        record = self.read_heartbeat(run_id)
+        if not record:
+            return False
+
+        status = str(record.get("status", ""))
+        if status in ("terminated", "completed", "failed"):
+            return False
+
+        last_heartbeat = record.get("last_heartbeat_at", "")
+        if not last_heartbeat:
+            return False
+
+        try:
+            last_ts = datetime.fromisoformat(last_heartbeat)
+            if last_ts.tzinfo is None:
+                last_ts = last_ts.replace(tzinfo=timezone.utc)
+            now = datetime.now(timezone.utc)
+            elapsed = (now - last_ts).total_seconds()
+            return elapsed > stall_threshold_seconds
+        except (ValueError, TypeError):
+            return False
+
+    def get_stalled_workers(
+        self,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> list[dict[str, Any]]:
+        """Return all heartbeat records for stalled workers."""
+        stalled: list[dict[str, Any]] = []
+        for record in self.list_heartbeats():
+            run_id = str(record.get("run_id", ""))
+            if run_id and self.check_stall(run_id, stall_threshold_seconds):
+                stalled.append(record)
+        return stalled
+
+    # --- Worker Termination ---
+
+    def terminate_worker(
+        self,
+        run_id: str,
+        worker_pid: int,
+        *,
+        grace_seconds: float = _SIGKILL_GRACE_SECONDS,
+    ) -> dict[str, Any]:
+        """Terminate a worker subprocess by PID.
+
+        Sends SIGTERM first, waits grace_seconds, then SIGKILL if still alive.
+
+        Args:
+            run_id: Run identifier for evidence recording.
+            worker_pid: PID to terminate.
+            grace_seconds: Seconds to wait after SIGTERM before SIGKILL.
+
+        Returns:
+            Termination result dict with status and details.
+        """
+        result: dict[str, Any] = {
+            "run_id": run_id,
+            "worker_pid": worker_pid,
+            "terminated_at": _now_iso(),
+            "sigterm_sent": False,
+            "sigkill_sent": False,
+            "process_exited": False,
+            "error": None,
+        }
+
+        try:
+            # Check if process exists first
+            os.kill(worker_pid, 0)
+        except (ProcessLookupError, PermissionError):
+            result["process_exited"] = True
+            result["error"] = "process_not_found_or_no_permission"
+            self.record_heartbeat(run_id, worker_pid, status="terminated")
+            return result
+
+        # Send SIGTERM
+        try:
+            os.kill(worker_pid, signal.SIGTERM)
+            result["sigterm_sent"] = True
+        except (ProcessLookupError, PermissionError) as exc:
+            result["process_exited"] = True
+            result["error"] = f"sigterm_failed: {exc}"
+            self.record_heartbeat(run_id, worker_pid, status="terminated")
+            return result
+
+        # Wait grace period for clean exit
+        deadline = time.monotonic() + grace_seconds
+        while time.monotonic() < deadline:
+            try:
+                os.kill(worker_pid, 0)
+            except ProcessLookupError:
+                result["process_exited"] = True
+                self.record_heartbeat(run_id, worker_pid, status="terminated")
+                return result
+            except PermissionError:
+                result["process_exited"] = True
+                self.record_heartbeat(run_id, worker_pid, status="terminated")
+                return result
+            time.sleep(0.1)
+
+        # SIGKILL if still alive
+        try:
+            os.kill(worker_pid, signal.SIGKILL)
+            result["sigkill_sent"] = True
+            result["process_exited"] = True
+        except (ProcessLookupError, PermissionError):
+            result["process_exited"] = True
+
+        self.record_heartbeat(run_id, worker_pid, status="terminated")
+        return result
+
+    # --- Worktree Cleanup ---
+
+    def cleanup_stale_worktree(self, run_id: str) -> dict[str, Any]:
+        """Remove stale worktree for a dead/terminated worker.
+
+        Uses git worktree remove first, then falls back to shutil.rmtree.
+
+        Returns:
+            Cleanup result dict.
+        """
+        import shutil
+        import subprocess
+
+        worktree = self._worktree_dir(run_id)
+        result: dict[str, Any] = {
+            "run_id": run_id,
+            "worktree_path": str(worktree),
+            "existed": worktree.is_dir(),
+            "cleaned": False,
+            "method": None,
+            "error": None,
+        }
+
+        if not worktree.is_dir():
+            return result
+
+        # Try git worktree remove first
+        try:
+            subprocess.run(
+                ["git", "worktree", "remove", "--force", str(worktree)],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=15,
+                cwd=str(self.project_dir),
+            )
+            if not worktree.is_dir():
+                result["cleaned"] = True
+                result["method"] = "git_worktree_remove"
+                return result
+        except (subprocess.TimeoutExpired, OSError):
+            pass
+
+        # Fallback: rmtree
+        try:
+            shutil.rmtree(str(worktree), ignore_errors=True)
+            result["cleaned"] = not worktree.is_dir()
+            result["method"] = "shutil_rmtree"
+        except OSError as exc:
+            result["error"] = str(exc)
+
+        return result
+
+    # --- Replay Evidence ---
+
+    def emit_replay_evidence(
+        self,
+        run_id: str,
+        reason: str,
+        *,
+        heartbeat: dict[str, Any] | None = None,
+        termination: dict[str, Any] | None = None,
+        cleanup: dict[str, Any] | None = None,
+        extra: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Write replay evidence to .omg/evidence/subagents/<run_id>-replay.json.
+
+        Captures all lifecycle data for post-mortem analysis.
+
+        Returns:
+            The evidence payload that was written.
+        """
+        path = self._replay_evidence_path(run_id)
+
+        payload: dict[str, Any] = {
+            "schema": "WorkerReplayEvidence",
+            "run_id": run_id,
+            "reason": reason,
+            "generated_at": _now_iso(),
+            "heartbeat_snapshot": heartbeat or self.read_heartbeat(run_id),
+            "termination": termination,
+            "cleanup": cleanup,
+            "extra": {
+                **(extra or {}),
+                "ownership": self._ownership_metadata(run_id),
+            },
+        }
+
+        _write_atomic_json(path, payload)
+        payload["evidence_path"] = str(path.relative_to(self.project_dir)).replace("\\", "/")
+        return payload
+
+    # --- Composite Operations ---
+
+    def cancel_and_cleanup(
+        self,
+        run_id: str,
+        worker_pid: int | None = None,
+        *,
+        reason: str = "cancelled",
+        grace_seconds: float = _SIGKILL_GRACE_SECONDS,
+    ) -> dict[str, Any]:
+        """Full cancel flow: terminate subprocess, cleanup worktree, emit evidence.
+
+        Args:
+            run_id: Run identifier.
+            worker_pid: PID of the worker (if known).
+            reason: Reason for cancellation.
+            grace_seconds: SIGTERM → SIGKILL grace period.
+
+        Returns:
+            Composite result with termination, cleanup, and evidence details.
+        """
+        termination: dict[str, Any] | None = None
+        if worker_pid is not None:
+            termination = self.terminate_worker(
+                run_id, worker_pid, grace_seconds=grace_seconds,
+            )
+
+        cleanup = self.cleanup_stale_worktree(run_id)
+
+        evidence = self.emit_replay_evidence(
+            run_id,
+            reason,
+            termination=termination,
+            cleanup=cleanup,
+        )
+
+        # Mark heartbeat as terminal
+        self.record_heartbeat(
+            run_id,
+            worker_pid,
+            status="terminated",
+            metadata={"reason": reason},
+        )
+
+        return {
+            "run_id": run_id,
+            "termination": termination,
+            "cleanup": cleanup,
+            "evidence": evidence,
+            "status": "cancelled",
+        }
+
+    def escalate_stall(
+        self,
+        run_id: str,
+        *,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> dict[str, Any] | None:
+        """Check for stall and escalate if detected.
+
+        Returns escalation result if stalled, None otherwise.
+        """
+        if not self.check_stall(run_id, stall_threshold_seconds):
+            return None
+
+        heartbeat = self.read_heartbeat(run_id)
+        worker_pid = heartbeat.get("worker_pid")
+
+        # Mark as stalled
+        self.record_heartbeat(run_id, worker_pid, status="stalled")
+
+        # Emit replay evidence for the stall
+        evidence = self.emit_replay_evidence(
+            run_id,
+            f"stall_detected: no heartbeat within {stall_threshold_seconds}s",
+            heartbeat=heartbeat,
+        )
+
+        return {
+            "run_id": run_id,
+            "worker_pid": worker_pid,
+            "stall_threshold_seconds": stall_threshold_seconds,
+            "heartbeat": heartbeat,
+            "evidence": evidence,
+            "status": "stalled",
+        }
+
+
+def get_worker_watchdog(project_dir: str | None = None) -> WorkerWatchdog:
+    """Factory for WorkerWatchdog instances."""
+    return WorkerWatchdog(project_dir)