@trac3er/oh-my-god 2.1.9 → 2.2.3

package/build/lib/runtime/worker_watchdog.py

@@ -0,0 +1,482 @@
+"""Worker lifecycle watchdog — heartbeats, stall detection, termination, and replay evidence.
+
+Monitors workers by run_id, detects stalls, terminates unresponsive workers,
+cleans up stale worktrees, and emits replayable evidence for post-mortem analysis.
+
+State paths:
+- Heartbeats: .omg/state/worker-heartbeats/<run_id>.json
+- Replay evidence: .omg/evidence/subagents/<run_id>-replay.json
+"""
+from __future__ import annotations
+
+import json
+import os
+import signal
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+_DEFAULT_STALL_THRESHOLD_SECONDS = 60
+_SIGKILL_GRACE_SECONDS = 5
+
+
+def _project_dir(project_dir: str | None = None) -> Path:
+    if project_dir:
+        return Path(project_dir)
+    return Path(os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd()))
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _write_atomic_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_name(f"{path.name}.tmp")
+    tmp_path.write_text(
+        json.dumps(payload, indent=2, ensure_ascii=True) + "\n",
+        encoding="utf-8",
+    )
+    os.replace(tmp_path, path)
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _active_coordinator_run_id(project_dir: Path) -> str:
+    shadow_path = project_dir / ".omg" / "shadow" / "active-run"
+    if shadow_path.exists():
+        try:
+            shadow = shadow_path.read_text(encoding="utf-8").strip()
+        except OSError:
+            shadow = ""
+        if shadow:
+            return shadow
+
+    state_dir = project_dir / ".omg" / "state" / "release_run_coordinator"
+    if not state_dir.exists():
+        return ""
+    candidates = sorted(state_dir.glob("*.json"), key=lambda p: p.stat().st_mtime)
+    for path in reversed(candidates):
+        payload = _read_json(path)
+        run_id = str(payload.get("run_id", "")).strip()
+        if run_id:
+            return run_id
+    return ""
+
+
+class WorkerWatchdog:
+    """Monitor workers by run_id with heartbeats, stall detection, and termination."""
+
+    def __init__(self, project_dir: str | None = None):
+        self.project_dir = _project_dir(project_dir)
+
+    # --- Paths ---
+
+    def _heartbeat_dir(self) -> Path:
+        return self.project_dir / ".omg" / "state" / "worker-heartbeats"
+
+    def _heartbeat_path(self, run_id: str) -> Path:
+        return self._heartbeat_dir() / f"{run_id}.json"
+
+    def _replay_evidence_path(self, run_id: str) -> Path:
+        return self.project_dir / ".omg" / "evidence" / "subagents" / f"{run_id}-replay.json"
+
+    def _worktree_dir(self, run_id: str) -> Path:
+        return self.project_dir / ".omg" / "worktrees" / run_id
+
+    def _ownership_metadata(self, run_id: str) -> dict[str, Any]:
+        from runtime.merge_writer import get_merge_writer
+
+        active_run_id = _active_coordinator_run_id(self.project_dir)
+        merge_writer = get_merge_writer(str(self.project_dir))
+        merge_writer_details = merge_writer.check_authorization_details(run_id)
+        return {
+            "run_id": run_id,
+            "active_run_id": active_run_id,
+            "merge_writer": merge_writer_details,
+        }
+
+    # --- Heartbeats ---
+
+    def record_heartbeat(
+        self,
+        run_id: str,
+        worker_pid: int | None = None,
+        *,
+        status: str = "alive",
+        metadata: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Write heartbeat to .omg/state/worker-heartbeats/<run_id>.json.
+
+        Args:
+            run_id: The run identifier for this worker.
+            worker_pid: Optional PID of the worker subprocess.
+            status: Worker status (alive, stalled, terminated, etc.).
+            metadata: Optional extra metadata to include.
+
+        Returns:
+            The heartbeat record that was persisted.
+        """
+        path = self._heartbeat_path(run_id)
+        existing = _read_json(path)
+
+        heartbeat_count = int(existing.get("heartbeat_count", 0)) + 1
+
+        record: dict[str, Any] = {
+            "schema": "WorkerHeartbeat",
+            "run_id": run_id,
+            "worker_pid": worker_pid or existing.get("worker_pid"),
+            "status": status,
+            "heartbeat_count": heartbeat_count,
+            "last_heartbeat_at": _now_iso(),
+            "first_heartbeat_at": existing.get("first_heartbeat_at", _now_iso()),
+            "metadata": metadata or existing.get("metadata", {}),
+            "ownership": self._ownership_metadata(run_id),
+        }
+
+        _write_atomic_json(path, record)
+        return record
+
+    def read_heartbeat(self, run_id: str) -> dict[str, Any]:
+        """Read heartbeat state for a given run_id.
+
+        Returns:
+            Heartbeat record dict, or empty dict if not found.
+        """
+        return _read_json(self._heartbeat_path(run_id))
+
+    def list_heartbeats(self) -> list[dict[str, Any]]:
+        """List all heartbeat records."""
+        hb_dir = self._heartbeat_dir()
+        if not hb_dir.exists():
+            return []
+        results: list[dict[str, Any]] = []
+        try:
+            for path in sorted(hb_dir.glob("*.json")):
+                if path.name.endswith(".tmp"):
+                    continue
+                record = _read_json(path)
+                if record:
+                    results.append(record)
+        except OSError:
+            pass
+        return results
+
+    # --- Stall Detection ---
+
+    def check_stall(
+        self,
+        run_id: str,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> bool:
+        """Return True if worker is stalled (no heartbeat within threshold).
+
+        A worker is stalled when:
+        - It has a heartbeat record with status 'alive'
+        - Its last heartbeat timestamp exceeds the stall threshold
+
+        Returns False if no heartbeat exists (worker may not have started).
+        """
+        record = self.read_heartbeat(run_id)
+        if not record:
+            return False
+
+        status = str(record.get("status", ""))
+        if status in ("terminated", "completed", "failed"):
+            return False
+
+        last_heartbeat = record.get("last_heartbeat_at", "")
+        if not last_heartbeat:
+            return False
+
+        try:
+            last_ts = datetime.fromisoformat(last_heartbeat)
+            if last_ts.tzinfo is None:
+                last_ts = last_ts.replace(tzinfo=timezone.utc)
+            now = datetime.now(timezone.utc)
+            elapsed = (now - last_ts).total_seconds()
+            return elapsed > stall_threshold_seconds
+        except (ValueError, TypeError):
+            return False
+
+    def get_stalled_workers(
+        self,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> list[dict[str, Any]]:
+        """Return all heartbeat records for stalled workers."""
+        stalled: list[dict[str, Any]] = []
+        for record in self.list_heartbeats():
+            run_id = str(record.get("run_id", ""))
+            if run_id and self.check_stall(run_id, stall_threshold_seconds):
+                stalled.append(record)
+        return stalled
+
+    # --- Worker Termination ---
+
+    def terminate_worker(
+        self,
+        run_id: str,
+        worker_pid: int,
+        *,
+        grace_seconds: float = _SIGKILL_GRACE_SECONDS,
+    ) -> dict[str, Any]:
+        """Terminate a worker subprocess by PID.
+
+        Sends SIGTERM first, waits grace_seconds, then SIGKILL if still alive.
+
+        Args:
+            run_id: Run identifier for evidence recording.
+            worker_pid: PID to terminate.
+            grace_seconds: Seconds to wait after SIGTERM before SIGKILL.
+
+        Returns:
+            Termination result dict with status and details.
+        """
+        result: dict[str, Any] = {
+            "run_id": run_id,
+            "worker_pid": worker_pid,
+            "terminated_at": _now_iso(),
+            "sigterm_sent": False,
+            "sigkill_sent": False,
+            "process_exited": False,
+            "error": None,
+        }
+
+        try:
+            # Check if process exists first
+            os.kill(worker_pid, 0)
+        except (ProcessLookupError, PermissionError):
+            result["process_exited"] = True
+            result["error"] = "process_not_found_or_no_permission"
+            self.record_heartbeat(run_id, worker_pid, status="terminated")
+            return result
+
+        # Send SIGTERM
+        try:
+            os.kill(worker_pid, signal.SIGTERM)
+            result["sigterm_sent"] = True
+        except (ProcessLookupError, PermissionError) as exc:
+            result["process_exited"] = True
+            result["error"] = f"sigterm_failed: {exc}"
+            self.record_heartbeat(run_id, worker_pid, status="terminated")
+            return result
+
+        # Wait grace period for clean exit
+        deadline = time.monotonic() + grace_seconds
+        while time.monotonic() < deadline:
+            try:
+                os.kill(worker_pid, 0)
+            except ProcessLookupError:
+                result["process_exited"] = True
+                self.record_heartbeat(run_id, worker_pid, status="terminated")
+                return result
+            except PermissionError:
+                result["process_exited"] = True
+                self.record_heartbeat(run_id, worker_pid, status="terminated")
+                return result
+            time.sleep(0.1)
+
+        # SIGKILL if still alive
+        try:
+            os.kill(worker_pid, signal.SIGKILL)
+            result["sigkill_sent"] = True
+            result["process_exited"] = True
+        except (ProcessLookupError, PermissionError):
+            result["process_exited"] = True
+
+        self.record_heartbeat(run_id, worker_pid, status="terminated")
+        return result
+
+    # --- Worktree Cleanup ---
+
+    def cleanup_stale_worktree(self, run_id: str) -> dict[str, Any]:
+        """Remove stale worktree for a dead/terminated worker.
+
+        Uses git worktree remove first, then falls back to shutil.rmtree.
+
+        Returns:
+            Cleanup result dict.
+        """
+        import shutil
+        import subprocess
+
+        worktree = self._worktree_dir(run_id)
+        result: dict[str, Any] = {
+            "run_id": run_id,
+            "worktree_path": str(worktree),
+            "existed": worktree.is_dir(),
+            "cleaned": False,
+            "method": None,
+            "error": None,
+        }
+
+        if not worktree.is_dir():
+            return result
+
+        # Try git worktree remove first
+        try:
+            subprocess.run(
+                ["git", "worktree", "remove", "--force", str(worktree)],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=15,
+                cwd=str(self.project_dir),
+            )
+            if not worktree.is_dir():
+                result["cleaned"] = True
+                result["method"] = "git_worktree_remove"
+                return result
+        except (subprocess.TimeoutExpired, OSError):
+            pass
+
+        # Fallback: rmtree
+        try:
+            shutil.rmtree(str(worktree), ignore_errors=True)
+            result["cleaned"] = not worktree.is_dir()
+            result["method"] = "shutil_rmtree"
+        except OSError as exc:
+            result["error"] = str(exc)
+
+        return result
+
+    # --- Replay Evidence ---
+
+    def emit_replay_evidence(
+        self,
+        run_id: str,
+        reason: str,
+        *,
+        heartbeat: dict[str, Any] | None = None,
+        termination: dict[str, Any] | None = None,
+        cleanup: dict[str, Any] | None = None,
+        extra: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Write replay evidence to .omg/evidence/subagents/<run_id>-replay.json.
+
+        Captures all lifecycle data for post-mortem analysis.
+
+        Returns:
+            The evidence payload that was written.
+        """
+        path = self._replay_evidence_path(run_id)
+
+        payload: dict[str, Any] = {
+            "schema": "WorkerReplayEvidence",
+            "run_id": run_id,
+            "reason": reason,
+            "generated_at": _now_iso(),
+            "heartbeat_snapshot": heartbeat or self.read_heartbeat(run_id),
+            "termination": termination,
+            "cleanup": cleanup,
+            "extra": {
+                **(extra or {}),
+                "ownership": self._ownership_metadata(run_id),
+            },
+        }
+
+        _write_atomic_json(path, payload)
+        payload["evidence_path"] = str(path.relative_to(self.project_dir)).replace("\\", "/")
+        return payload
+
+    # --- Composite Operations ---
+
+    def cancel_and_cleanup(
+        self,
+        run_id: str,
+        worker_pid: int | None = None,
+        *,
+        reason: str = "cancelled",
+        grace_seconds: float = _SIGKILL_GRACE_SECONDS,
+    ) -> dict[str, Any]:
+        """Full cancel flow: terminate subprocess, cleanup worktree, emit evidence.
+
+        Args:
+            run_id: Run identifier.
+            worker_pid: PID of the worker (if known).
+            reason: Reason for cancellation.
+            grace_seconds: SIGTERM → SIGKILL grace period.
+
+        Returns:
+            Composite result with termination, cleanup, and evidence details.
+        """
+        termination: dict[str, Any] | None = None
+        if worker_pid is not None:
+            termination = self.terminate_worker(
+                run_id, worker_pid, grace_seconds=grace_seconds,
+            )
+
+        cleanup = self.cleanup_stale_worktree(run_id)
+
+        evidence = self.emit_replay_evidence(
+            run_id,
+            reason,
+            termination=termination,
+            cleanup=cleanup,
+        )
+
+        # Mark heartbeat as terminal
+        self.record_heartbeat(
+            run_id,
+            worker_pid,
+            status="terminated",
+            metadata={"reason": reason},
+        )
+
+        return {
+            "run_id": run_id,
+            "termination": termination,
+            "cleanup": cleanup,
+            "evidence": evidence,
+            "status": "cancelled",
+        }
+
+    def escalate_stall(
+        self,
+        run_id: str,
+        *,
+        stall_threshold_seconds: float = _DEFAULT_STALL_THRESHOLD_SECONDS,
+    ) -> dict[str, Any] | None:
+        """Check for stall and escalate if detected.
+
+        Returns escalation result if stalled, None otherwise.
+        """
+        if not self.check_stall(run_id, stall_threshold_seconds):
+            return None
+
+        heartbeat = self.read_heartbeat(run_id)
+        worker_pid = heartbeat.get("worker_pid")
+
+        # Mark as stalled
+        self.record_heartbeat(run_id, worker_pid, status="stalled")
+
+        # Emit replay evidence for the stall
+        evidence = self.emit_replay_evidence(
+            run_id,
+            f"stall_detected: no heartbeat within {stall_threshold_seconds}s",
+            heartbeat=heartbeat,
+        )
+
+        return {
+            "run_id": run_id,
+            "worker_pid": worker_pid,
+            "stall_threshold_seconds": stall_threshold_seconds,
+            "heartbeat": heartbeat,
+            "evidence": evidence,
+            "status": "stalled",
+        }
+
+
+def get_worker_watchdog(project_dir: str | None = None) -> WorkerWatchdog:
+    """Factory for WorkerWatchdog instances."""
+    return WorkerWatchdog(project_dir)

package/commands/OMG:issue.md

@@ -0,0 +1,56 @@
+---
+description: "Active red-team diagnostics and simulation surface for ranked issue triage."
+allowed-tools: Read, Grep, Glob, Bash(python3 scripts/omg.py issue:*), Bash(pytest:*)
+argument-hint: "[optional: --surfaces <csv> --simulate-surface <name> --simulate-scenario <name>]"
+---
+
+# /OMG:issue
+
+Run active, read-only diagnostics across live and governed OMG surfaces, then emit ranked machine-readable issues.
+
+## Command Roles
+
+- `/OMG:issue` is the diagnostic and red-team simulation surface.
+- `omg fix --issue` remains the issue-driven fix surface.
+- Both share the same backend: `runtime/issue_surface.py`.
+- `/OMG:issue` does not mutate targets.
+
+## Surfaces Scanned
+
+- `live_session`
+- `forge_runs`
+- `hooks`
+- `skills`
+- `mcps`
+- `plugin_interop`
+- `governed_tools`
+- `domain_pipelines`
+
+## Severity and Approval
+
+- Severities: `critical`, `high`, `medium`, `low`, `info`
+- High/critical issues must declare signed-approval requirements before fix execution.
+- Output includes fix guidance plus evidence links for replay and audit.
+
+## Usage
+
+```text
+/OMG:issue
+/OMG:issue --surfaces plugin_interop,governed_tools
+/OMG:issue --simulate-surface hooks --simulate-scenario "delete protected lock"
+```
+
+## Backend Output
+
+- Evidence file: `.omg/evidence/issues/<run_id>.json`
+- Report schema: `IssueReport`
+- Each issue includes:
+  - `id`
+  - `severity`
+  - `surface`
+  - `title`
+  - `description`
+  - `fix_guidance`
+  - `evidence_links`
+  - `approval_required`
+  - `approval_reason`

package/commands/OMG:teams.md

@@ -1,11 +1,14 @@
 ---
-description: OMG internal team routing (standalone). Standalone team routing.
+description: OMG internal staged team routing (standalone). /OMG:team canonical, /OMG:teams compatibility alias.
 allowed-tools: Read, Grep, Glob, Bash(git:*), Bash(rg:*), Bash(find:*), Bash(cat:*), Bash(python3:*)
 argument-hint: "[codex|gemini|ccg|auto] 'problem statement'"
 ---
 
 # /OMG:teams — Standalone Internal Router
 
+Canonical alias: `/OMG:team`
+Compatibility alias: `/OMG:teams`
+
 Use OMG's internal router using internal routing.
 
 ## Input contract
@@ -16,20 +19,32 @@ Use OMG's internal router using internal routing.
 - expected_outcome: optional acceptance target
 
 ## Execution
-Use internal CLI router:
+Use internal CLI router (canonical):
 
 ```bash
 OMG_CLI="${OMG_CLI_PATH:-$HOME/.claude/omg-runtime/scripts/omg.py}"
 if [ ! -f "$OMG_CLI" ] && [ -f "scripts/omg.py" ]; then OMG_CLI="scripts/omg.py"; fi
+python3 "$OMG_CLI" team --target auto --problem "[problem]"
+```
+
+Compatibility alias (legacy):
+
+```bash
 python3 "$OMG_CLI" teams --target auto --problem "[problem]"
 ```
 
 For explicit target:
 
 ```bash
-python3 "$OMG_CLI" teams --target codex --problem "[problem]"
+python3 "$OMG_CLI" team --target codex --problem "[problem]"
 ```
 
+## Staged flow
+- `team-plan`: resolve context packet and clarification/council gate before dispatch
+- `team-exec`: launch workers only when context is resolved
+- `team-verify`: run verification and critic synthesis
+- `team-fix`: surface blockers/next actions for follow-up iteration
+
 ## Output schema
 `TeamDispatchResult { status, findings[], actions[], evidence{} }`
 

package/commands/OMG:validate.md

@@ -36,7 +36,7 @@ python3 scripts/omg.py validate --format json
       "required": true | false
     }
   ],
-  "version": "2.1.9"
+  "version": "2.2.3"
 }
 ```
 

package/control_plane/service.py

@@ -24,6 +24,7 @@ from runtime.mutation_gate import check_mutation_allowed
 from runtime.runtime_contracts import read_run_state
 from runtime.security_check import run_security_check
 from runtime.test_intent_lock import lock_intent, verify_intent
+from runtime.tool_fabric import ToolFabric
 
 
 class ControlPlaneService:
@@ -348,6 +349,76 @@ class ControlPlaneService:
         )
         return 200, result
 
+    def tool_fabric_request(self, payload: dict[str, Any]) -> tuple[int, dict[str, Any]]:
+        if not isinstance(payload, dict):
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": "payload must be an object",
+            }
+
+        lane_name = str(payload.get("lane_name", "")).strip()
+        tool_name = str(payload.get("tool_name", "")).strip()
+        run_id = str(payload.get("run_id", "")).strip()
+        context = payload.get("context")
+
+        if not lane_name:
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": "lane_name is required",
+            }
+        if not tool_name:
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": "tool_name is required",
+            }
+        if not run_id:
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": "run_id is required",
+            }
+        if context is not None and not isinstance(context, dict):
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": "context must be an object when provided",
+            }
+
+        fabric = ToolFabric(project_dir=self.project_dir)
+        bundle_map = {
+            "lsp-pack": "registry/bundles/lsp-pack.yaml",
+            "hash-edit": "registry/bundles/hash-edit.yaml",
+            "ast-pack": "registry/bundles/ast-pack.yaml",
+            "terminal-lane": "registry/bundles/terminal-lane.yaml",
+        }
+        for lane, bundle_path in bundle_map.items():
+            fabric.register_lane(lane, bundle_path)
+
+        try:
+            result = fabric.request_tool(
+                lane_name=lane_name,
+                tool_name=tool_name,
+                run_id=run_id,
+                context=context if isinstance(context, dict) else None,
+            )
+        except ValueError as exc:
+            return 400, {
+                "status": "error",
+                "error_code": "INVALID_TOOL_FABRIC_INPUT",
+                "message": str(exc),
+            }
+
+        payload_out = {
+            "status": "allowed" if result.allowed else "blocked",
+            "reason": result.reason,
+            "evidence_path": result.evidence_path,
+            "ledger_entry": result.ledger_entry,
+        }
+        return (200 if result.allowed else 403), payload_out
+
     def session_health(self, payload: dict[str, Any]) -> tuple[int, dict[str, Any]]:
         run_id = payload.get("run_id")
         if isinstance(run_id, str) and run_id.strip():

package/dist/enterprise/attestations/bundle/.agents/skills/omg/AGENTS.fragment.md.minisig

@@ -1,4 +1,4 @@
 untrusted comment: signature from omg local signing key
-kKKtJIRU6V/zDNlq4KSdhrTekKfHCN7859D8rExVR4+9sCGiVdXbZi9Ay3ybETtgeopVF7ZJqzxHOesjGfY/AQ==
-trusted comment: type=omg-slsa-v1 keyid=01810c85c0a1ea2c issued_at=2026-03-13T00:09:44.258294+00:00 artifact=bundle/.agents/skills/omg/AGENTS.fragment.md sha256=19234c26a265b0256b44087986992ee8f40591a9b0af603ee7ba4e55f34d3ea5
-mzIapCCOPIrPMVUrQXLwGE1hLXA7T9CcHlGhJboF3z6YRLcLgZst4P2QcWXJ/DHPgiNOrGoQJLbyZCYVSeDBDg==
+t3Hj+kr0JheIUNTTALI6VBMayNDqIaAauCYbCQFmfKVUxjlc9RUDd4S57aBg94UnoqYB7bbwYUfsg5rbqZ1XAQ==
+trusted comment: type=omg-slsa-v1 keyid=913c3701a7ac8076 issued_at=2026-03-14T19:07:28.521813+00:00 artifact=bundle/.agents/skills/omg/AGENTS.fragment.md sha256=19234c26a265b0256b44087986992ee8f40591a9b0af603ee7ba4e55f34d3ea5
+isGvKXxn/JeHQiAXShBZmeAjfV8sg6FCM1o5e3CAMIner8BZzyP/f+RzCPAdySgoFBRnHZ5U55s2SNxpKE8jAw==