@trac3er/oh-my-god 2.1.8 → 2.2.2

package/runtime/host_parity.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from runtime.canonical_surface import get_canonical_hosts
+
+
+_IGNORED_SEMANTIC_KEYS = {
+    "host",
+    "model",
+    "model_name",
+    "provider",
+    "timestamp",
+    "run_id",
+    "id",
+    "latency_ms",
+    "duration_ms",
+    "token_usage",
+    "usage",
+    "raw",
+}
+
+_TOKEN_RE = re.compile(r"[a-z0-9_\-]+")
+_KV_LINE_RE = re.compile(r"^\s*([A-Za-z0-9_\- ]+):\s*(.+?)\s*$")
+
+
+@dataclass
+class ParityCheckResult:
+    passed: bool
+    host_results: dict[str, dict[str, Any]]
+    drift_detected: bool
+    drift_details: list[str]
+
+
+@dataclass
+class HostParityReport:
+    run_id: str
+    timestamp: str
+    canonical_hosts: list[str]
+    parity_results: dict[str, Any]
+    overall_status: str
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host).strip().lower()
+
+
+def _canonicalize(value: Any) -> Any:
+    if isinstance(value, dict):
+        out: dict[str, Any] = {}
+        for key in sorted(value.keys()):
+            key_str = str(key)
+            if key_str in _IGNORED_SEMANTIC_KEYS:
+                continue
+            out[key_str] = _canonicalize(value[key])
+        return out
+    if isinstance(value, list):
+        canonical_items = [_canonicalize(item) for item in value]
+        if all(isinstance(item, (str, int, float, bool, type(None))) for item in canonical_items):
+            return sorted(canonical_items, key=lambda item: repr(item))
+        return canonical_items
+    if isinstance(value, str):
+        return value.strip()
+    return value
+
+
+def _tokenize(text: str) -> list[str]:
+    return sorted(set(_TOKEN_RE.findall(text.lower())))
+
+
+def _parse_json_blob(text: str) -> Any | None:
+    stripped = text.strip()
+    if not stripped:
+        return None
+    try:
+        return json.loads(stripped)
+    except json.JSONDecodeError:
+        pass
+
+    fence = re.search(r"```(?:json)?\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*```", stripped, flags=re.IGNORECASE)
+    if fence:
+        candidate = fence.group(1)
+        try:
+            return json.loads(candidate)
+        except json.JSONDecodeError:
+            pass
+
+    decoder = json.JSONDecoder()
+    for start in range(len(stripped)):
+        if stripped[start] not in "[{":
+            continue
+        try:
+            parsed, _ = decoder.raw_decode(stripped[start:])
+            return parsed
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+def _coerce_scalar(value: str) -> Any:
+    lowered = value.strip().lower()
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+    if lowered in {"null", "none"}:
+        return None
+    if re.fullmatch(r"-?\d+", value.strip()):
+        return int(value.strip())
+    if re.fullmatch(r"-?\d+\.\d+", value.strip()):
+        return float(value.strip())
+    return value.strip()
+
+
+def _parse_structured_text(text: str) -> dict[str, Any]:
+    structured: dict[str, Any] = {}
+    for line in text.splitlines():
+        match = _KV_LINE_RE.match(line)
+        if not match:
+            continue
+        key = match.group(1).strip().lower().replace(" ", "_")
+        value = match.group(2).strip()
+        if "," in value:
+            structured[key] = [_coerce_scalar(part) for part in value.split(",") if part.strip()]
+        else:
+            structured[key] = _coerce_scalar(value)
+    return structured
+
+
+def _build_semantic_content(host: str, output_value: Any) -> dict[str, Any]:
+    if isinstance(output_value, (dict, list)):
+        return {"structured": _canonicalize(output_value)}
+
+    output_text = str(output_value or "").strip()
+    parsed = _parse_json_blob(output_text)
+    if parsed is not None:
+        return {"structured": _canonicalize(parsed)}
+
+    if host == "gemini":
+        structured = _parse_structured_text(output_text)
+        if structured:
+            return {"structured": _canonicalize(structured)}
+
+    tokens = _tokenize(output_text)
+    return {
+        "text": {
+            "tokens": tokens[:64],
+            "token_count": len(tokens),
+        }
+    }
+
+
+def _jaccard_similarity(a_tokens: list[str], b_tokens: list[str]) -> float:
+    a_set = set(a_tokens)
+    b_set = set(b_tokens)
+    if not a_set and not b_set:
+        return 1.0
+    if not a_set or not b_set:
+        return 0.0
+    return len(a_set & b_set) / len(a_set | b_set)
+
+
+class HostParityNormalizer:
+    def __init__(self, canonical_hosts: list[str] | None = None, text_similarity_threshold: float = 0.55):
+        self.canonical_hosts = [_normalize_host(host) for host in (canonical_hosts or get_canonical_hosts())]
+        self.text_similarity_threshold = max(0.0, min(1.0, float(text_similarity_threshold)))
+
+    def normalize_output(self, host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+        _ = context
+        host_name = _normalize_host(host)
+        payload = raw_output if isinstance(raw_output, dict) else {"output": raw_output}
+
+        exit_code_raw = payload.get("exit_code", 0)
+        try:
+            exit_code = int(exit_code_raw)
+        except Exception:
+            exit_code = 0
+
+        error = str(payload.get("error", "")).strip()
+        status = "error" if error or exit_code != 0 else "ok"
+        fallback = str(payload.get("fallback", "")).strip()
+        semantic_content = _build_semantic_content(host_name, payload.get("output", ""))
+
+        normalized = {
+            "host": host_name,
+            "status": status,
+            "exit_code": exit_code,
+            "exit_code_class": "success" if exit_code == 0 else "error",
+            "fallback": fallback,
+            "error": error,
+            "error_class": "none" if not error else "provider-error",
+            "semantic": semantic_content,
+        }
+        return normalized
+
+    def _is_semantically_equivalent(self, left: dict[str, Any], right: dict[str, Any]) -> tuple[bool, str]:
+        if left.get("status") != right.get("status"):
+            return False, "status mismatch"
+
+        left_semantic = left.get("semantic", {}) if isinstance(left.get("semantic"), dict) else {}
+        right_semantic = right.get("semantic", {}) if isinstance(right.get("semantic"), dict) else {}
+
+        left_structured = left_semantic.get("structured")
+        right_structured = right_semantic.get("structured")
+        if left_structured is not None and right_structured is not None:
+            if left_structured == right_structured:
+                return True, "structured-equivalent"
+            return False, "structured content mismatch"
+
+        left_tokens = left_semantic.get("text", {}).get("tokens", [])
+        right_tokens = right_semantic.get("text", {}).get("tokens", [])
+        if isinstance(left_tokens, list) and isinstance(right_tokens, list):
+            score = _jaccard_similarity(left_tokens, right_tokens)
+            if score >= self.text_similarity_threshold:
+                return True, f"text-equivalent score={score:.2f}"
+            return False, f"text drift score={score:.2f}"
+        return False, "semantic payload mismatch"
+
+    def check_parity(self, outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+        host_results: dict[str, dict[str, Any]] = {}
+        drift_details: list[str] = []
+
+        baseline_host = ""
+        baseline_output: dict[str, Any] | None = None
+
+        for host in self.canonical_hosts:
+            if host not in outputs_by_host:
+                host_results[host] = {
+                    "present": False,
+                    "passed": False,
+                    "reason": "missing host output",
+                    "normalized": {},
+                }
+                drift_details.append(f"missing host output: {host}")
+                continue
+
+            normalized = self.normalize_output(host, outputs_by_host[host], context)
+            host_results[host] = {
+                "present": True,
+                "passed": True,
+                "reason": "baseline",
+                "normalized": normalized,
+            }
+            if baseline_output is None:
+                baseline_host = host
+                baseline_output = normalized
+
+        if baseline_output is not None:
+            for host in self.canonical_hosts:
+                current = host_results.get(host, {})
+                if not current.get("present"):
+                    continue
+                normalized = current.get("normalized")
+                if not isinstance(normalized, dict):
+                    continue
+                equivalent, reason = self._is_semantically_equivalent(baseline_output, normalized)
+                current["passed"] = equivalent
+                current["reason"] = reason if host != baseline_host else "baseline"
+                if not equivalent:
+                    drift_details.append(f"semantic drift: {host} vs {baseline_host} ({reason})")
+
+        drift_detected = bool(drift_details)
+        passed = not drift_detected and all(
+            isinstance(host_results.get(host), dict)
+            and bool(host_results[host].get("present"))
+            and bool(host_results[host].get("passed"))
+            for host in self.canonical_hosts
+        )
+        return ParityCheckResult(
+            passed=passed,
+            host_results=host_results,
+            drift_detected=drift_detected,
+            drift_details=drift_details,
+        )
+
+
+def normalize_output(host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+    return HostParityNormalizer().normalize_output(host, raw_output, context)
+
+
+def check_parity(outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+    return HostParityNormalizer().check_parity(outputs_by_host, context)
+
+
+def emit_parity_report(
+    run_id: str,
+    results: ParityCheckResult,
+    *,
+    project_dir: str | None = None,
+) -> str:
+    base_dir = Path(project_dir or os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd()))
+    out_path = base_dir / ".omg" / "evidence" / f"host-parity-{run_id}.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    report = HostParityReport(
+        run_id=str(run_id),
+        timestamp=_now_iso(),
+        canonical_hosts=get_canonical_hosts(),
+        parity_results=asdict(results),
+        overall_status="ok" if results.passed else "drift",
+    )
+    out_path.write_text(json.dumps(asdict(report), indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    return str(out_path)
+
+
+__all__ = [
+    "HostParityNormalizer",
+    "ParityCheckResult",
+    "HostParityReport",
+    "normalize_output",
+    "check_parity",
+    "emit_parity_report",
+]

package/runtime/incident_replay.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 from datetime import datetime, timezone
 import json
 from pathlib import Path
+import re
 from typing import Any
 from uuid import uuid4
 
@@ -12,6 +13,32 @@ def _now() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def _write_pack(project_dir: str, incident_id: str, payload: dict[str, Any]) -> dict[str, Any]:
+    rel_path = Path(".omg") / "incidents" / f"{incident_id}.json"
+    path = Path(project_dir) / rel_path
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    payload["path"] = rel_path.as_posix()
+    return payload
+
+
+def _slug(value: str) -> str:
+    normalized = re.sub(r"[^a-z0-9]+", "-", str(value).strip().lower())
+    normalized = normalized.strip("-")
+    return normalized or "chaos"
+
+
+def _to_blockers(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    blockers: list[str] = []
+    for item in value:
+        cleaned = str(item).strip()
+        if cleaned:
+            blockers.append(cleaned)
+    return blockers
+
+
 def build_incident_pack(
     project_dir: str,
     *,
@@ -39,9 +66,245 @@ def build_incident_pack(
         "regression_guards": failing_tests,
     }
 
-    rel_path = Path(".omg") / "incidents" / f"{incident_id}.json"
-    path = Path(project_dir) / rel_path
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(payload, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
-    payload["path"] = rel_path.as_posix()
-    return payload
+    return _write_pack(project_dir, incident_id, payload)
+
+
+def build_worker_lifecycle_pack(
+    project_dir: str,
+    *,
+    run_id: str,
+    event: str,
+    heartbeat: dict[str, Any] | None = None,
+    termination: dict[str, Any] | None = None,
+    cleanup: dict[str, Any] | None = None,
+    trace_id: str | None = None,
+) -> dict[str, Any]:
+    incident_id = f"worker-lifecycle-{uuid4().hex[:12]}"
+    payload: dict[str, Any] = {
+        "schema": "WorkerLifecycleReplayPack",
+        "incident_id": incident_id,
+        "run_id": run_id,
+        "event": event,
+        "generated_at": _now(),
+        "trace_id": trace_id or "",
+        "heartbeat": heartbeat or {},
+        "termination": termination or {},
+        "cleanup": cleanup or {},
+        "reproduction_steps": [
+            f"Review heartbeat state for run_id={run_id}.",
+            f"Inspect worker lifecycle event: {event}.",
+            "Check .omg/evidence/subagents/ for replay evidence.",
+        ],
+    }
+
+    return _write_pack(project_dir, incident_id, payload)
+
+
+def build_chaos_replay_pack(
+    project_dir: str,
+    *,
+    run_id: str,
+    scenario: str,
+    fixture: str,
+    fault: str,
+    expected_outcome: dict[str, Any],
+    observed: dict[str, Any],
+    blockers: list[str] | None = None,
+    trace_id: str | None = None,
+    deterministic_seed: str | None = None,
+    evidence_freshness_max_age_seconds: float | None = None,
+) -> dict[str, Any]:
+    scenario_slug = _slug(scenario)
+    incident_id = f"chaos-{scenario_slug}-{uuid4().hex[:12]}"
+    expected_blockers = _to_blockers(expected_outcome.get("blockers"))
+    observed_blockers = _to_blockers(observed.get("blockers"))
+    generated_at = _now()
+
+    evidence_freshness: dict[str, Any] = {
+        "generated_at": generated_at,
+        "fixture_id": fixture,
+        "trace_id": trace_id or "",
+    }
+    if evidence_freshness_max_age_seconds is not None:
+        evidence_freshness["max_age_seconds"] = evidence_freshness_max_age_seconds
+
+    payload: dict[str, Any] = {
+        "schema": "ChaosReplayPack",
+        "incident_id": incident_id,
+        "run_id": run_id,
+        "scenario": scenario,
+        "fixture": fixture,
+        "fault": fault,
+        "generated_at": generated_at,
+        "trace_id": trace_id or "",
+        "deterministic_seed": deterministic_seed or "",
+        "expected_outcome": expected_outcome,
+        "observed": observed,
+        "blockers": blockers or expected_blockers or observed_blockers,
+        "evidence_freshness": evidence_freshness,
+        "reproduction_steps": [
+            f"Load fixture '{fixture}' and inject fault '{fault}'.",
+            "Execute the deterministic chaos scenario with bounded timeout.",
+            "Replay this pack and verify the blocker status reproduces.",
+        ],
+    }
+    return _write_pack(project_dir, incident_id, payload)
+
+
+def replay_chaos_pack(project_dir: str, pack_path: str) -> dict[str, Any]:
+    root = Path(project_dir)
+    source_path = Path(pack_path)
+    if not source_path.is_absolute():
+        source_path = root / source_path
+
+    if not source_path.exists():
+        return {
+            "schema": "ChaosReplayResult",
+            "status": "error",
+            "reproduced": False,
+            "reason": "pack_not_found",
+            "pack_path": str(pack_path),
+        }
+
+    try:
+        payload = json.loads(source_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {
+            "schema": "ChaosReplayResult",
+            "status": "error",
+            "reproduced": False,
+            "reason": "pack_unreadable",
+            "pack_path": str(pack_path),
+        }
+
+    if not isinstance(payload, dict) or payload.get("schema") != "ChaosReplayPack":
+        return {
+            "schema": "ChaosReplayResult",
+            "status": "error",
+            "reproduced": False,
+            "reason": "invalid_pack_schema",
+            "pack_path": str(pack_path),
+        }
+
+    expected = payload.get("expected_outcome")
+    observed = payload.get("observed")
+    if not isinstance(expected, dict) or not isinstance(observed, dict):
+        return {
+            "schema": "ChaosReplayResult",
+            "status": "error",
+            "reproduced": False,
+            "reason": "missing_outcomes",
+            "pack_path": str(source_path.relative_to(root)).replace("\\", "/"),
+        }
+
+    expected_status = str(expected.get("status", "")).strip().lower()
+    observed_status = str(observed.get("status", "")).strip().lower()
+    expected_blockers = set(_to_blockers(expected.get("blockers")))
+    observed_blockers = set(_to_blockers(observed.get("blockers")))
+    status_match = not expected_status or expected_status == observed_status
+    blocker_match = expected_blockers.issubset(observed_blockers)
+    reproduced = status_match and blocker_match
+
+    return {
+        "schema": "ChaosReplayResult",
+        "status": "blocked" if reproduced and observed_status == "blocked" else ("ok" if reproduced else "mismatch"),
+        "reproduced": reproduced,
+        "run_id": str(payload.get("run_id", "")).strip(),
+        "scenario": str(payload.get("scenario", "")).strip(),
+        "pack_path": str(source_path.relative_to(root)).replace("\\", "/"),
+        "expected_status": expected_status,
+        "observed_status": observed_status,
+        "expected_blockers": sorted(expected_blockers),
+        "observed_blockers": sorted(observed_blockers),
+    }
+
+
+def collect_incident_signals(project_dir: str, run_id: str) -> dict[str, object]:
+    root = Path(project_dir)
+    incident_dir = root / ".omg" / "incidents"
+    ledger_path = root / ".omg" / "state" / "ledger" / "tool-ledger.jsonl"
+
+    incident_count = 0
+    issue_payloads: list[dict[str, object]] = []
+    if incident_dir.is_dir():
+        for path in sorted(incident_dir.glob("*.json")):
+            incident_count += 1
+            try:
+                payload = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError):
+                continue
+            if not isinstance(payload, dict):
+                continue
+            title = str(payload.get("title") or payload.get("event") or payload.get("scenario") or "incident")
+            issue_payloads.append(
+                {
+                    "severity": "low",
+                    "surface": "hooks",
+                    "title": f"Incident replay artifact present: {title}",
+                    "description": "Incident artifact available for replay and triage verification.",
+                    "fix_guidance": "Link this artifact to issue remediation if related to current run.",
+                    "evidence_links": [str(path.relative_to(root)).replace("\\", "/")],
+                    "approval_required": False,
+                    "approval_reason": "",
+                }
+            )
+
+    hook_issues: list[dict[str, object]] = list(issue_payloads)
+    governed_tool_issues: list[dict[str, object]] = []
+    if ledger_path.exists():
+        try:
+            with ledger_path.open("r", encoding="utf-8") as handle:
+                for raw in handle:
+                    line = raw.strip()
+                    if not line:
+                        continue
+                    try:
+                        entry = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    if not isinstance(entry, dict):
+                        continue
+                    entry_run_id = str(entry.get("run_id", "")).strip()
+                    if run_id and entry_run_id and entry_run_id != run_id:
+                        continue
+                    lane = str(entry.get("lane", "")).strip()
+                    governed_tool = str(entry.get("governed_tool", "")).strip()
+                    if lane or governed_tool:
+                        evidence = str(entry.get("evidence_path", "")).strip()
+                        if not evidence:
+                            governed_tool_issues.append(
+                                {
+                                    "severity": "high",
+                                    "surface": "governed_tools",
+                                    "title": "Governed tool execution missing evidence link",
+                                    "description": (
+                                        f"Governed tool '{governed_tool or entry.get('tool', '')}' ran on lane '{lane}' without evidence_path."
+                                    ),
+                                    "fix_guidance": "Attach signed evidence artifacts before approving governed tool output.",
+                                    "evidence_links": [".omg/state/ledger/tool-ledger.jsonl"],
+                                    "approval_required": True,
+                                    "approval_reason": "signed approval required when governed tools lack evidence linkage",
+                                }
+                            )
+                    exit_code = entry.get("exit_code")
+                    if isinstance(exit_code, int) and exit_code != 0:
+                        hook_issues.append(
+                            {
+                                "severity": "medium",
+                                "surface": "hooks",
+                                "title": "Hook/tool execution returned non-zero exit",
+                                "description": f"Tool '{entry.get('tool', '')}' exited with code {exit_code}.",
+                                "fix_guidance": "Inspect the tool ledger entry and replay with bounded diagnostics.",
+                                "evidence_links": [".omg/state/ledger/tool-ledger.jsonl"],
+                                "approval_required": False,
+                                "approval_reason": "",
+                            }
+                        )
+        except OSError:
+            pass
+
+    return {
+        "incident_count": incident_count,
+        "hook_issues": hook_issues,
+        "governed_tool_issues": governed_tool_issues,
+    }

package/runtime/interaction_journal.py

@@ -310,12 +310,15 @@ class InteractionJournal:
         normalized_run_id = run_id.strip()
         if not normalized_run_id:
             return None
+        fallback = f".omg/evidence/repro-pack-{normalized_run_id}.json"
         try:
             result = build_repro_pack(str(self.project_dir), normalized_run_id)
         except Exception:
-            return None
+            evidence_path = self.project_dir / ".omg" / "evidence" / f"{normalized_run_id}.json"
+            return fallback if evidence_path.exists() else None
         if result.get("status") != "ok":
-            return None
+            evidence_path = self.project_dir / ".omg" / "evidence" / f"{normalized_run_id}.json"
+            return fallback if evidence_path.exists() else None
         path = result.get("path")
         return str(path) if isinstance(path, str) and path else None