@trac3r/oh-my-god 2.2.11

package/runtime/guide_assert.py

@@ -0,0 +1,135 @@
+"""Rule-based output assertions for OMG guide checks."""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def guide_assert(candidate: str, rules: dict[str, Any]) -> dict[str, Any]:
+    text = candidate or ""
+    lowered = text.lower()
+    violations: list[dict[str, str]] = []
+
+    terms_rules = rules.get("terms_guard") if isinstance(rules, dict) else {}
+    terms_verdict = terms_guard(text, terms_rules if isinstance(terms_rules, dict) else {})
+    if terms_verdict.get("status") != "ok":
+        violations.append(
+            {
+                "rule_type": "terms_guard",
+                "rule": "terms_guard",
+                "reason": str(terms_verdict.get("reason", "terms_guard_rejected")),
+            }
+        )
+
+    for goal in _as_list(rules.get("goals")):
+        if "todo" in goal.lower() and "todo" in lowered:
+            violations.append({"rule_type": "goal", "rule": goal, "reason": "candidate still includes TODO markers"})
+
+    for non_goal in _as_list(rules.get("non_goals")):
+        if _mentions(lowered, non_goal):
+            violations.append({"rule_type": "non_goal", "rule": non_goal, "reason": "candidate mentions an explicit non-goal"})
+
+    for criterion in _as_list(rules.get("acceptance_criteria")):
+        if "production-ready" in criterion.lower() and any(token in lowered for token in ("todo", "insecure", "placeholder")):
+            violations.append({"rule_type": "acceptance_criteria", "rule": criterion, "reason": "candidate contains non-production wording"})
+
+    return {
+        "schema": "GuideAssertionResult",
+        "verdict": "fail" if violations else "pass",
+        "violations": violations,
+        "terms_guard": terms_verdict,
+        "summary": {
+            "rule_count": sum(len(_as_list(rules.get(key))) for key in ("goals", "non_goals", "acceptance_criteria", "architecture_constraints", "style_rules", "risk_appetite")),
+            "violation_count": len(violations),
+        },
+    }
+
+
+def terms_guard(input_text: str, rules: dict[str, Any]) -> dict[str, Any]:
+    text = str(input_text or "")
+    config = rules if isinstance(rules, dict) else {}
+
+    min_length = _as_int(config.get("min_length"))
+    max_length = _as_int(config.get("max_length"))
+    if min_length is not None and min_length < 0:
+        return {"status": "rejected", "reason": "terms_guard_invalid_min_length"}
+    if max_length is not None and max_length < 0:
+        return {"status": "rejected", "reason": "terms_guard_invalid_max_length"}
+    if min_length is not None and max_length is not None and min_length > max_length:
+        return {"status": "rejected", "reason": "terms_guard_invalid_length_bounds"}
+
+    allowlist = _as_exact_terms(config.get("allowlist"))
+    if config.get("allowlist") is not None and allowlist is None:
+        return {"status": "rejected", "reason": "terms_guard_invalid_allowlist"}
+
+    pattern = config.get("pattern")
+    if pattern is not None and not isinstance(pattern, str):
+        return {"status": "rejected", "reason": "terms_guard_invalid_pattern"}
+
+    deny_patterns = _as_exact_terms(config.get("deny_patterns"))
+    if config.get("deny_patterns") is not None and deny_patterns is None:
+        return {"status": "rejected", "reason": "terms_guard_invalid_deny_patterns"}
+
+    if min_length is not None and len(text) < min_length:
+        return {"status": "rejected", "reason": "terms_guard_too_short"}
+    if max_length is not None and len(text) > max_length:
+        return {"status": "rejected", "reason": "terms_guard_too_long"}
+
+    if allowlist is not None and text not in allowlist:
+        return {"status": "rejected", "reason": "terms_guard_not_in_allowlist"}
+
+    if isinstance(pattern, str) and pattern:
+        try:
+            if re.fullmatch(pattern, text) is None:
+                return {"status": "rejected", "reason": "terms_guard_pattern_mismatch"}
+        except re.error:
+            return {"status": "rejected", "reason": "terms_guard_invalid_pattern"}
+
+    if deny_patterns:
+        for blocked in deny_patterns:
+            try:
+                if re.search(blocked, text):
+                    return {"status": "rejected", "reason": "terms_guard_disallowed_pattern"}
+            except re.error:
+                return {"status": "rejected", "reason": "terms_guard_invalid_deny_pattern"}
+
+    return {"status": "ok"}
+
+
+def _as_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if str(item).strip()]
+
+
+def _mentions(lowered_candidate: str, rule: str) -> bool:
+    tokens = [token.lower() for token in rule.split() if len(token) >= 4]
+    if not tokens:
+        return False
+    return all(token in lowered_candidate for token in tokens)
+
+
+def _as_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, str) and value.strip().isdigit():
+        return int(value.strip())
+    return None
+
+
+def _as_exact_terms(value: Any) -> list[str] | None:
+    if not isinstance(value, list):
+        return None
+    output: list[str] = []
+    for item in value:
+        if not isinstance(item, str):
+            return None
+        token = item.strip()
+        if not token:
+            return None
+        output.append(token)
+    return output

package/runtime/hook_governor.py

@@ -0,0 +1,156 @@
+"""Hook-order validation using the canonical hook-governor bundle."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import re
+from typing import Literal, TypedDict, cast
+
+try:
+    import yaml
+except Exception:  # pragma: no cover - json fallback path
+    yaml = None
+
+
+DEFAULT_BUNDLE_PATH = Path("registry") / "bundles" / "hook-governor.yaml"
+_MODULE_DIR = Path(__file__).resolve().parent.parent  # project root (runtime/ → project root)
+SECURITY_REQUIRED_BY_EVENT: dict[str, tuple[str, ...]] = {
+    "PreToolUse": ("firewall", "secret-guard"),
+}
+
+
+class ValidationResult(TypedDict):
+    status: Literal["ok", "blocked"]
+    blockers: list[str]
+
+
+def _resolve_project_dir(project_dir: str | None) -> Path:
+    if project_dir:
+        return Path(project_dir)
+    env_project_dir = os.environ.get("CLAUDE_PROJECT_DIR")
+    if env_project_dir:
+        return Path(env_project_dir)
+    return Path.cwd()
+
+
+def _extract_hook_name(command: str) -> str:
+    matches = cast(list[str], re.findall(r"([A-Za-z0-9_.-]+\.py)", command))
+    if matches:
+        return Path(matches[-1]).stem
+    tokens = command.strip().split()
+    if not tokens:
+        return ""
+    return Path(tokens[-1].strip("\"'")).stem
+
+
+def _load_compiled_hooks(
+    project_dir: str | None = None,
+    bundle_path: str | None = None,
+) -> tuple[dict[str, list[str]], str | None]:
+    root = _resolve_project_dir(project_dir)
+    candidate = Path(bundle_path) if bundle_path else (root / DEFAULT_BUNDLE_PATH)
+    if not bundle_path and not candidate.exists():
+        # Fallback: resolve relative to this module's installed location (robust to cwd changes
+        # and CLAUDE_PROJECT_DIR pointing to temp dirs in parallel test runs).
+        module_relative = _MODULE_DIR / DEFAULT_BUNDLE_PATH
+        if module_relative.exists():
+            candidate = module_relative
+    try:
+        raw = candidate.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        return {}, f"canonical hook bundle missing: {candidate}"
+    except Exception as exc:
+        return {}, f"failed reading canonical hook bundle: {exc}"
+
+    try:
+        parsed_obj: object
+        if yaml is not None:
+            parsed_obj = cast(object, yaml.safe_load(raw))
+            if parsed_obj is None:
+                parsed_obj = {}
+        else:
+            parsed_obj = cast(object, json.loads(raw))
+    except Exception as exc:
+        return {}, f"failed parsing canonical hook bundle: {exc}"
+
+    if not isinstance(parsed_obj, dict):
+        return {}, "invalid canonical hook bundle payload"
+
+    payload = cast(dict[object, object], parsed_obj)
+    compiled_hooks_obj = payload.get("compiled_hooks")
+    if not isinstance(compiled_hooks_obj, dict):
+        return {}, "compiled_hooks missing from canonical hook bundle"
+
+    compiled_hooks = cast(dict[object, object], compiled_hooks_obj)
+    canonical: dict[str, list[str]] = {}
+    for event_key, hooks_obj in compiled_hooks.items():
+        if not isinstance(event_key, str) or not isinstance(hooks_obj, list):
+            continue
+        names: list[str] = []
+        for hook_entry in cast(list[object], hooks_obj):
+            if not isinstance(hook_entry, dict):
+                continue
+            hook_map = cast(dict[object, object], hook_entry)
+            command_obj = hook_map.get("command")
+            if not isinstance(command_obj, str):
+                continue
+            hook_name = _extract_hook_name(command_obj)
+            if hook_name:
+                names.append(hook_name)
+        canonical[event_key] = names
+    return canonical, None
+
+
+def get_canonical_order(
+    event: str,
+    *,
+    project_dir: str | None = None,
+    bundle_path: str | None = None,
+) -> list[str]:
+    compiled_hooks, _ = _load_compiled_hooks(project_dir=project_dir, bundle_path=bundle_path)
+    return list(compiled_hooks.get(event, ()))
+
+
+def validate_order(
+    event: str,
+    hooks_list: list[str],
+    *,
+    project_dir: str | None = None,
+    bundle_path: str | None = None,
+) -> ValidationResult:
+    compiled_hooks, load_error = _load_compiled_hooks(project_dir=project_dir, bundle_path=bundle_path)
+    if load_error:
+        return {"status": "blocked", "blockers": [load_error]}
+
+    canonical = compiled_hooks.get(event, [])
+    if not canonical:
+        return {"status": "blocked", "blockers": [f"no canonical hook order for event {event}"]}
+
+    blockers: list[str] = []
+    expected_positions = {name: idx for idx, name in enumerate(canonical)}
+
+    required_security_hooks = SECURITY_REQUIRED_BY_EVENT.get(event, ())
+    for required in required_security_hooks:
+        if required not in hooks_list:
+            blockers.append(f"missing required security hook: {required}")
+
+    if required_security_hooks:
+        for expected_index, required in enumerate(required_security_hooks):
+            if required not in hooks_list:
+                continue
+            actual_index = hooks_list.index(required)
+            if actual_index != expected_index:
+                blockers.append(
+                    f"hook order violation for {event}: {required} must run before non-security hooks"
+                )
+
+    filtered_hooks = [hook for hook in hooks_list if hook in expected_positions]
+    for left, right in zip(filtered_hooks, filtered_hooks[1:]):
+        if expected_positions[left] > expected_positions[right]:
+            blockers.append(
+                f"hook order violation for {event}: {left} must run after {right} in canonical order"
+            )
+
+    status: Literal["ok", "blocked"] = "blocked" if blockers else "ok"
+    return {"status": status, "blockers": blockers}

package/runtime/host_parity.py

@@ -0,0 +1,373 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from runtime.canonical_surface import get_canonical_hosts
+
+
+_IGNORED_SEMANTIC_KEYS = {
+    "host",
+    "model",
+    "model_name",
+    "provider",
+    "timestamp",
+    "run_id",
+    "id",
+    "latency_ms",
+    "duration_ms",
+    "token_usage",
+    "usage",
+    "raw",
+}
+
+_TOKEN_RE = re.compile(r"[a-z0-9_\-]+")
+_KV_LINE_RE = re.compile(r"^\s*([A-Za-z0-9_\- ]+):\s*(.+?)\s*$")
+_REAL_PARITY_SOURCE_KINDS = {
+    "compiled_artifact",
+    "compiled_output",
+    "replayed_output",
+    "replayed_artifact",
+}
+
+
+@dataclass
+class ParityCheckResult:
+    passed: bool
+    host_results: dict[str, dict[str, Any]]
+    drift_detected: bool
+    drift_details: list[str]
+
+
+@dataclass
+class HostParityReport:
+    run_id: str
+    timestamp: str
+    canonical_hosts: list[str]
+    parity_results: dict[str, Any]
+    overall_status: str
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host).strip().lower()
+
+
+def _source_metadata(payload: dict[str, Any]) -> tuple[str, str]:
+    raw_source = payload.get("source")
+    if not isinstance(raw_source, dict):
+        return "", ""
+
+    source: dict[str, Any] = raw_source
+    source_kind = str(source.get("kind", "")).strip().lower()
+    source_path = ""
+    for key in ("artifact_path", "replay_path", "compiled_path", "output_path", "path"):
+        value = str(source.get(key, "")).strip()
+        if value:
+            source_path = value
+            break
+    return source_kind, source_path
+
+
+def is_synthetic(payload: dict[str, Any]) -> bool:
+    source_kind, source_path = _source_metadata(payload)
+    if not source_kind:
+        return True
+    if source_kind not in _REAL_PARITY_SOURCE_KINDS:
+        return True
+    if not source_path:
+        return True
+    return False
+
+
+def _canonicalize(value: Any) -> Any:
+    if isinstance(value, dict):
+        out: dict[str, Any] = {}
+        for key in sorted(value.keys()):
+            key_str = str(key)
+            if key_str in _IGNORED_SEMANTIC_KEYS:
+                continue
+            out[key_str] = _canonicalize(value[key])
+        return out
+    if isinstance(value, list):
+        canonical_items = [_canonicalize(item) for item in value]
+        if all(isinstance(item, (str, int, float, bool, type(None))) for item in canonical_items):
+            return sorted(canonical_items, key=lambda item: repr(item))
+        return canonical_items
+    if isinstance(value, str):
+        return value.strip()
+    return value
+
+
+def _tokenize(text: str) -> list[str]:
+    return sorted(set(_TOKEN_RE.findall(text.lower())))
+
+
+def _parse_json_blob(text: str) -> Any | None:
+    stripped = text.strip()
+    if not stripped:
+        return None
+    try:
+        return json.loads(stripped)
+    except json.JSONDecodeError:
+        pass
+
+    fence = re.search(r"```(?:json)?\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*```", stripped, flags=re.IGNORECASE)
+    if fence:
+        candidate = fence.group(1)
+        try:
+            return json.loads(candidate)
+        except json.JSONDecodeError:
+            pass
+
+    decoder = json.JSONDecoder()
+    for start in range(len(stripped)):
+        if stripped[start] not in "[{":
+            continue
+        try:
+            parsed, _ = decoder.raw_decode(stripped[start:])
+            return parsed
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+def _coerce_scalar(value: str) -> Any:
+    lowered = value.strip().lower()
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+    if lowered in {"null", "none"}:
+        return None
+    if re.fullmatch(r"-?\d+", value.strip()):
+        return int(value.strip())
+    if re.fullmatch(r"-?\d+\.\d+", value.strip()):
+        return float(value.strip())
+    return value.strip()
+
+
+def _parse_structured_text(text: str) -> dict[str, Any]:
+    structured: dict[str, Any] = {}
+    for line in text.splitlines():
+        match = _KV_LINE_RE.match(line)
+        if not match:
+            continue
+        key = match.group(1).strip().lower().replace(" ", "_")
+        value = match.group(2).strip()
+        if "," in value:
+            structured[key] = [_coerce_scalar(part) for part in value.split(",") if part.strip()]
+        else:
+            structured[key] = _coerce_scalar(value)
+    return structured
+
+
+def _build_semantic_content(host: str, output_value: Any) -> dict[str, Any]:
+    if isinstance(output_value, (dict, list)):
+        return {"structured": _canonicalize(output_value)}
+
+    output_text = str(output_value or "").strip()
+    parsed = _parse_json_blob(output_text)
+    if parsed is not None:
+        return {"structured": _canonicalize(parsed)}
+
+    if host == "gemini":
+        structured = _parse_structured_text(output_text)
+        if structured:
+            return {"structured": _canonicalize(structured)}
+
+    tokens = _tokenize(output_text)
+    return {
+        "text": {
+            "tokens": tokens[:64],
+            "token_count": len(tokens),
+        }
+    }
+
+
+def _jaccard_similarity(a_tokens: list[str], b_tokens: list[str]) -> float:
+    a_set = set(a_tokens)
+    b_set = set(b_tokens)
+    if not a_set and not b_set:
+        return 1.0
+    if not a_set or not b_set:
+        return 0.0
+    return len(a_set & b_set) / len(a_set | b_set)
+
+
+class HostParityNormalizer:
+    def __init__(self, canonical_hosts: list[str] | None = None, text_similarity_threshold: float = 0.55):
+        self.canonical_hosts = [_normalize_host(host) for host in (canonical_hosts or get_canonical_hosts())]
+        self.text_similarity_threshold = max(0.0, min(1.0, float(text_similarity_threshold)))
+
+    def normalize_output(self, host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+        _ = context
+        host_name = _normalize_host(host)
+        payload = raw_output if isinstance(raw_output, dict) else {"output": raw_output}
+        source_kind, source_path = _source_metadata(payload)
+
+        exit_code_raw = payload.get("exit_code", 0)
+        try:
+            exit_code = int(exit_code_raw)
+        except Exception:
+            exit_code = 0
+
+        error = str(payload.get("error", "")).strip()
+        status = "error" if error or exit_code != 0 else "ok"
+        fallback = str(payload.get("fallback", "")).strip()
+        semantic_content = _build_semantic_content(host_name, payload.get("output", ""))
+
+        normalized = {
+            "host": host_name,
+            "status": status,
+            "exit_code": exit_code,
+            "exit_code_class": "success" if exit_code == 0 else "error",
+            "fallback": fallback,
+            "error": error,
+            "error_class": "none" if not error else "provider-error",
+            "source_kind": source_kind or "synthetic",
+            "source_path": source_path,
+            "source_class": "compiled_or_replayed" if not is_synthetic(payload) else "synthetic",
+            "semantic": semantic_content,
+        }
+        return normalized
+
+    def _is_semantically_equivalent(self, left: dict[str, Any], right: dict[str, Any]) -> tuple[bool, str]:
+        if left.get("status") != right.get("status"):
+            return False, "status mismatch"
+
+        left_semantic = left.get("semantic", {}) if isinstance(left.get("semantic"), dict) else {}
+        right_semantic = right.get("semantic", {}) if isinstance(right.get("semantic"), dict) else {}
+
+        left_structured = left_semantic.get("structured")
+        right_structured = right_semantic.get("structured")
+        if left_structured is not None and right_structured is not None:
+            if left_structured == right_structured:
+                return True, "structured-equivalent"
+            return False, "structured content mismatch"
+
+        left_tokens = left_semantic.get("text", {}).get("tokens", [])
+        right_tokens = right_semantic.get("text", {}).get("tokens", [])
+        if isinstance(left_tokens, list) and isinstance(right_tokens, list):
+            score = _jaccard_similarity(left_tokens, right_tokens)
+            if score >= self.text_similarity_threshold:
+                return True, f"text-equivalent score={score:.2f}"
+            return False, f"text drift score={score:.2f}"
+        return False, "semantic payload mismatch"
+
+    def check_parity(self, outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+        host_results: dict[str, dict[str, Any]] = {}
+        drift_details: list[str] = []
+
+        baseline_host = ""
+        baseline_output: dict[str, Any] | None = None
+
+        for host in self.canonical_hosts:
+            if host not in outputs_by_host:
+                host_results[host] = {
+                    "present": False,
+                    "passed": False,
+                    "reason": "missing host output",
+                    "normalized": {},
+                }
+                drift_details.append(f"missing host output: {host}")
+                continue
+
+            raw_payload = outputs_by_host[host]
+            payload = raw_payload if isinstance(raw_payload, dict) else {"output": raw_payload}
+            normalized = self.normalize_output(host, payload, context)
+            if is_synthetic(payload):
+                host_results[host] = {
+                    "present": True,
+                    "passed": False,
+                    "reason": "synthetic payload rejected",
+                    "normalized": normalized,
+                }
+                drift_details.append(f"synthetic payload rejected: {host}")
+                continue
+
+            host_results[host] = {
+                "present": True,
+                "passed": True,
+                "reason": "baseline",
+                "normalized": normalized,
+            }
+            if baseline_output is None:
+                baseline_host = host
+                baseline_output = normalized
+
+        if baseline_output is not None:
+            for host in self.canonical_hosts:
+                current = host_results.get(host, {})
+                if not current.get("present"):
+                    continue
+                if current.get("reason") == "synthetic payload rejected":
+                    continue
+                normalized = current.get("normalized")
+                if not isinstance(normalized, dict):
+                    continue
+                equivalent, reason = self._is_semantically_equivalent(baseline_output, normalized)
+                current["passed"] = equivalent
+                current["reason"] = reason if host != baseline_host else "baseline"
+                if not equivalent:
+                    drift_details.append(f"semantic drift: {host} vs {baseline_host} ({reason})")
+
+        drift_detected = bool(drift_details)
+        passed = not drift_detected and all(
+            isinstance(host_results.get(host), dict)
+            and bool(host_results[host].get("present"))
+            and bool(host_results[host].get("passed"))
+            for host in self.canonical_hosts
+        )
+        return ParityCheckResult(
+            passed=passed,
+            host_results=host_results,
+            drift_detected=drift_detected,
+            drift_details=drift_details,
+        )
+
+
+def normalize_output(host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+    return HostParityNormalizer().normalize_output(host, raw_output, context)
+
+
+def check_parity(outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+    return HostParityNormalizer().check_parity(outputs_by_host, context)
+
+
+def emit_parity_report(
+    run_id: str,
+    results: ParityCheckResult,
+    *,
+    project_dir: str | None = None,
+) -> str:
+    base_dir = Path(project_dir or os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd()))
+    out_path = base_dir / ".omg" / "evidence" / f"host-parity-{run_id}.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    report = HostParityReport(
+        run_id=str(run_id),
+        timestamp=_now_iso(),
+        canonical_hosts=get_canonical_hosts(),
+        parity_results=asdict(results),
+        overall_status="ok" if results.passed else "drift",
+    )
+    out_path.write_text(json.dumps(asdict(report), indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    return str(out_path)
+
+
+__all__ = [
+    "HostParityNormalizer",
+    "ParityCheckResult",
+    "HostParityReport",
+    "is_synthetic",
+    "normalize_output",
+    "check_parity",
+    "emit_parity_report",
+]