@trac3er/oh-my-god 2.1.8 → 2.2.2

package/build/lib/runtime/forge_contracts.py

@@ -30,6 +30,52 @@ DEFAULT_STAGE_TIMEOUT_MS: dict[str, int] = {
     "regression_test": 5_000,
 }
 
+AXOLOTL_LIVE_MODES: tuple[str, ...] = (
+    "preflight",
+    "live_sft",
+    "live_grpo",
+    "live_gdpo",
+)
+
+AXOLOTL_SEARCH_POLICY: dict[str, object] = {
+    "max_trials": 6,
+    "space": {
+        "learning_rate": [1e-5, 3e-5, 1e-4],
+        "batch_size": [4, 8],
+        "lora_rank": [8, 16, 32],
+        "grad_accum": [1, 2, 4],
+    },
+}
+
+OPERATION_ORCHESTRATION_PROFILES: dict[str, dict[str, object]] = {
+    "add": {
+        "mode": "incremental_extension",
+        "priority": "scope_safeguards",
+        "required_checks": ["schema_alignment", "backward_compatibility"],
+    },
+    "edit": {
+        "mode": "targeted_refinement",
+        "priority": "change_precision",
+        "required_checks": ["delta_scope", "regression_surface"],
+    },
+    "delete": {
+        "mode": "controlled_removal",
+        "priority": "safety_first",
+        "required_checks": ["dependency_impact", "rollback_path"],
+    },
+    "unknown": {
+        "mode": "bounded_execution",
+        "priority": "contract_defaults",
+        "required_checks": ["domain_contract"],
+    },
+}
+
+
+def _normalize_required_checks(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(check) for check in value if isinstance(check, str)]
+
 
 ADAPTER_REGISTRY: dict[str, dict[str, object]] = {
     "axolotl": {
@@ -37,6 +83,13 @@ ADAPTER_REGISTRY: dict[str, dict[str, object]] = {
         "module": "axolotl",
         "hook": "lab.axolotl_adapter.run",
         "specialist": "training-architect",
+        "modes": list(AXOLOTL_LIVE_MODES),
+        "default_mode": "live_sft",
+        "rl_sidecar_required": {
+            "live_grpo": True,
+            "live_gdpo": True,
+        },
+        "search_policy": dict(AXOLOTL_SEARCH_POLICY),
     },
     "pybullet": {
         "kind": "simulator",
@@ -87,11 +140,19 @@ def load_forge_mvp() -> dict[str, object]:
             "security_review": "regression_test",
             "threat_model": "evaluate",
         },
+        "operation_orchestration": {
+            intent: {
+                "mode": str(profile.get("mode", "bounded_execution")),
+                "priority": str(profile.get("priority", "contract_defaults")),
+                "required_checks": _normalize_required_checks(profile.get("required_checks")),
+            }
+            for intent, profile in OPERATION_ORCHESTRATION_PROFILES.items()
+        },
         "specialist_contracts": {
             "forge-cybersecurity": {
                 "labs_only": True,
                 "allowed_domains": ["cybersecurity"],
-                "evidence_profile": "forge-run",
+                "evidence_profile": "security-audit",
                 "proof_backed_dispatch": True,
                 "stage_alias": "regression_test",
                 "evidence_outputs": {
@@ -102,8 +163,27 @@ def load_forge_mvp() -> dict[str, object]:
                 },
                 "reuses_scanner": "runtime.security_check",
             },
+            "music-omr": {
+                "labs_only": True,
+                "allowed_domains": ["music-omr"],
+                "evidence_profile": "music-omr",
+                "proof_backed_dispatch": True,
+                "stage_alias": "regression_test",
+                "evidence_outputs": {
+                    "omr_json": ".omg/evidence/music-omr-*.json",
+                },
+                "reuses_scanner": "runtime.music_omr_testbed",
+            },
+        },
+        "domain_evidence_profiles": {
+            "vision": "forge-vision",
+            "robotics": "forge-run",
+            "algorithms": "transposition-flow",
+            "health": "health-flow",
+            "cybersecurity": "security-audit",
         },
         "adapter_registry": dict(ADAPTER_REGISTRY),
+        "axolotl_search_policy": dict(AXOLOTL_SEARCH_POLICY),
         "canonical_domains": get_all_canonical_domains(),
         "evidence_output_path": ".omg/evidence/forge-<run_id>.json",
         "starter_templates": {
@@ -277,8 +357,46 @@ def build_forge_evidence(
             ]
 
     job_dict = dict(job)
+    base_model_name = "unknown"
+    base_model_raw = job.get("base_model")
+    if isinstance(base_model_raw, Mapping):
+        base_model_name = str(base_model_raw.get("name", "unknown"))
+
+    evaluation_metric = 0.0
+    evaluation_report = result.get("evaluation_report")
+    if isinstance(evaluation_report, Mapping):
+        metric_raw = evaluation_report.get("metric", 0.0)
+        if isinstance(metric_raw, bool):
+            evaluation_metric = 0.0
+        elif isinstance(metric_raw, (int, float)):
+            evaluation_metric = float(metric_raw)
+        elif isinstance(metric_raw, str):
+            try:
+                evaluation_metric = float(metric_raw)
+            except ValueError:
+                evaluation_metric = 0.0
+
+    target_metric = 0.0
+    target_metric_raw = job.get("target_metric", 0.0)
+    if isinstance(target_metric_raw, bool):
+        target_metric = 0.0
+    elif isinstance(target_metric_raw, (int, float)):
+        target_metric = float(target_metric_raw)
+    elif isinstance(target_metric_raw, str):
+        try:
+            target_metric = float(target_metric_raw)
+        except ValueError:
+            target_metric = 0.0
+
     context_checksum = hashlib.sha256(json.dumps(job_dict, sort_keys=True).encode()).hexdigest()
     domain_pack = get_domain_pack_contract(domain) if domain in DOMAIN_PACKS else {}
+    artifact_contracts = _resolve_artifact_contracts(
+        run_id=run_id,
+        result=result,
+        evaluation_metric=evaluation_metric,
+        target_metric=target_metric,
+        base_model_name=base_model_name,
+    )
     payload = {
         "schema": "ForgeMVPEvidence",
         "schema_version": "1.0.0",
@@ -305,37 +423,7 @@ def build_forge_evidence(
         "profile_version": "forge-run-v1",
         "intent_gate_version": "1.0.0",
         "domain_pack": domain_pack,
-        "artifact_contracts": {
-            "dataset_lineage": {
-                "standard": "Croissant-1.1",
-                "path": f".omg/evidence/forge-lineage-{run_id}.json",
-                "status": "placeholder",
-                "reason": "dataset lineage artifact not yet generated",
-            },
-            "model_card": {
-                "standard": "HuggingFace-ModelCard",
-                "path": f".omg/evidence/forge-model-card-{run_id}.md",
-                "status": "placeholder",
-                "reason": "model card artifact not yet generated",
-            },
-            "checkpoint_hash": {
-                "standard": "OpenSSF-OMS",
-                "path": f".omg/evidence/forge-checkpoint-{run_id}.json",
-                "status": "placeholder",
-                "reason": "checkpoint hash artifact not yet generated",
-            },
-            "regression_scoreboard": {
-                "standard": "lm-eval",
-                "path": f".omg/evidence/forge-scoreboard-{run_id}.json",
-                "status": "placeholder",
-                "reason": "regression scoreboard artifact not yet generated",
-            },
-            "promotion_decision": {
-                "status": "pending",
-                "reason": "promotion requires passing regression scoreboard and human review for health domain",
-                "replay_required": True,
-            },
-        },
+        "artifact_contracts": artifact_contracts,
     }
 
     if domain == "cybersecurity":
@@ -354,3 +442,78 @@ def build_forge_evidence(
     _ = tmp_path.write_text(json.dumps(payload, indent=2, ensure_ascii=True), encoding="utf-8")
     _ = os.rename(tmp_path, out_path)
     return str(out_path)
+
+
+def _resolve_artifact_contracts(
+    *,
+    run_id: str,
+    result: Mapping[str, object],
+    evaluation_metric: float,
+    target_metric: float,
+    base_model_name: str,
+) -> dict[str, object]:
+    result_contracts = result.get("artifact_contracts")
+    if isinstance(result_contracts, Mapping):
+        normalized = {str(key): value for key, value in result_contracts.items()}
+        if "dataset_lineage" not in normalized:
+            normalized["dataset_lineage"] = {
+                "standard": "Croissant-1.1",
+                "path": f".omg/evidence/forge-lineage-{run_id}.json",
+                "status": "pending_verification",
+                "lineage_hash": hashlib.sha256(f"lineage-{run_id}".encode()).hexdigest(),
+                "deterministic_metadata": True,
+            }
+        if "model_card" not in normalized:
+            normalized["model_card"] = {
+                "standard": "HuggingFace-ModelCard",
+                "path": f".omg/evidence/forge-model-card-{run_id}.md",
+                "status": "pending_verification",
+                "model_id": f"forge-model-{run_id}",
+                "base_model": base_model_name,
+            }
+        if "checkpoint_hash" not in normalized:
+            normalized["checkpoint_hash"] = {
+                "standard": "OpenSSF-OMS",
+                "path": f".omg/evidence/forge-checkpoint-{run_id}.json",
+                "status": "pending_verification",
+                "sha256": hashlib.sha256(f"checkpoint-{run_id}".encode()).hexdigest(),
+                "algorithm": "sha256",
+            }
+        return normalized
+
+    return {
+        "dataset_lineage": {
+            "standard": "Croissant-1.1",
+            "path": f".omg/evidence/forge-lineage-{run_id}.json",
+            "status": "pending_verification",
+            "lineage_hash": hashlib.sha256(f"lineage-{run_id}".encode()).hexdigest(),
+            "deterministic_metadata": True,
+        },
+        "model_card": {
+            "standard": "HuggingFace-ModelCard",
+            "path": f".omg/evidence/forge-model-card-{run_id}.md",
+            "status": "pending_verification",
+            "model_id": f"forge-model-{run_id}",
+            "base_model": base_model_name,
+        },
+        "checkpoint_hash": {
+            "standard": "OpenSSF-OMS",
+            "path": f".omg/evidence/forge-checkpoint-{run_id}.json",
+            "status": "pending_verification",
+            "sha256": hashlib.sha256(f"checkpoint-{run_id}".encode()).hexdigest(),
+            "algorithm": "sha256",
+        },
+        "regression_scoreboard": {
+            "standard": "lm-eval",
+            "path": f".omg/evidence/forge-scoreboard-{run_id}.json",
+            "status": "insufficient_evidence",
+            "score": evaluation_metric,
+            "target": target_metric,
+        },
+        "promotion_decision": {
+            "status": "pending",
+            "decision_id": f"dec-{run_id}",
+            "reason": "promotion requires passing regression scoreboard and human review for health domain",
+            "replay_required": True,
+        },
+    }

package/build/lib/runtime/forge_run_id.py

@@ -2,6 +2,15 @@
 from __future__ import annotations
 
 from datetime import datetime, timezone
+import hashlib
+
+
+DETERMINISM_VERSION = "forge-determinism-v1"
+DETERMINISM_SCOPE = "same-hardware"
+_TEMPERATURE_LOCK = {
+    "critical_model_paths": 0.0,
+    "critical_tool_paths": 0.0,
+}
 
 
 def generate_run_id() -> str:
@@ -15,6 +24,24 @@ def generate_run_id() -> str:
     return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
 
 
+def derive_run_seed(run_id: str) -> int:
+    is_valid, reason = validate_run_id(run_id)
+    if not is_valid:
+        raise ValueError(f"invalid run_id for deterministic seed derivation: {reason}")
+
+    digest = hashlib.sha256(run_id.encode("utf-8")).digest()
+    return int.from_bytes(digest[:8], byteorder="big", signed=False)
+
+
+def build_deterministic_contract(run_id: str) -> dict[str, object]:
+    return {
+        "seed": derive_run_seed(run_id),
+        "temperature_lock": dict(_TEMPERATURE_LOCK),
+        "determinism_version": DETERMINISM_VERSION,
+        "determinism_scope": DETERMINISM_SCOPE,
+    }
+
+
 def validate_run_id(run_id: str) -> tuple[bool, str]:
     """Validate run ID format and constraints.
     

package/build/lib/runtime/host_parity.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from runtime.canonical_surface import get_canonical_hosts
+
+
+_IGNORED_SEMANTIC_KEYS = {
+    "host",
+    "model",
+    "model_name",
+    "provider",
+    "timestamp",
+    "run_id",
+    "id",
+    "latency_ms",
+    "duration_ms",
+    "token_usage",
+    "usage",
+    "raw",
+}
+
+_TOKEN_RE = re.compile(r"[a-z0-9_\-]+")
+_KV_LINE_RE = re.compile(r"^\s*([A-Za-z0-9_\- ]+):\s*(.+?)\s*$")
+
+
+@dataclass
+class ParityCheckResult:
+    passed: bool
+    host_results: dict[str, dict[str, Any]]
+    drift_detected: bool
+    drift_details: list[str]
+
+
+@dataclass
+class HostParityReport:
+    run_id: str
+    timestamp: str
+    canonical_hosts: list[str]
+    parity_results: dict[str, Any]
+    overall_status: str
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _normalize_host(host: str) -> str:
+    return str(host).strip().lower()
+
+
+def _canonicalize(value: Any) -> Any:
+    if isinstance(value, dict):
+        out: dict[str, Any] = {}
+        for key in sorted(value.keys()):
+            key_str = str(key)
+            if key_str in _IGNORED_SEMANTIC_KEYS:
+                continue
+            out[key_str] = _canonicalize(value[key])
+        return out
+    if isinstance(value, list):
+        canonical_items = [_canonicalize(item) for item in value]
+        if all(isinstance(item, (str, int, float, bool, type(None))) for item in canonical_items):
+            return sorted(canonical_items, key=lambda item: repr(item))
+        return canonical_items
+    if isinstance(value, str):
+        return value.strip()
+    return value
+
+
+def _tokenize(text: str) -> list[str]:
+    return sorted(set(_TOKEN_RE.findall(text.lower())))
+
+
+def _parse_json_blob(text: str) -> Any | None:
+    stripped = text.strip()
+    if not stripped:
+        return None
+    try:
+        return json.loads(stripped)
+    except json.JSONDecodeError:
+        pass
+
+    fence = re.search(r"```(?:json)?\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*```", stripped, flags=re.IGNORECASE)
+    if fence:
+        candidate = fence.group(1)
+        try:
+            return json.loads(candidate)
+        except json.JSONDecodeError:
+            pass
+
+    decoder = json.JSONDecoder()
+    for start in range(len(stripped)):
+        if stripped[start] not in "[{":
+            continue
+        try:
+            parsed, _ = decoder.raw_decode(stripped[start:])
+            return parsed
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+def _coerce_scalar(value: str) -> Any:
+    lowered = value.strip().lower()
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+    if lowered in {"null", "none"}:
+        return None
+    if re.fullmatch(r"-?\d+", value.strip()):
+        return int(value.strip())
+    if re.fullmatch(r"-?\d+\.\d+", value.strip()):
+        return float(value.strip())
+    return value.strip()
+
+
+def _parse_structured_text(text: str) -> dict[str, Any]:
+    structured: dict[str, Any] = {}
+    for line in text.splitlines():
+        match = _KV_LINE_RE.match(line)
+        if not match:
+            continue
+        key = match.group(1).strip().lower().replace(" ", "_")
+        value = match.group(2).strip()
+        if "," in value:
+            structured[key] = [_coerce_scalar(part) for part in value.split(",") if part.strip()]
+        else:
+            structured[key] = _coerce_scalar(value)
+    return structured
+
+
+def _build_semantic_content(host: str, output_value: Any) -> dict[str, Any]:
+    if isinstance(output_value, (dict, list)):
+        return {"structured": _canonicalize(output_value)}
+
+    output_text = str(output_value or "").strip()
+    parsed = _parse_json_blob(output_text)
+    if parsed is not None:
+        return {"structured": _canonicalize(parsed)}
+
+    if host == "gemini":
+        structured = _parse_structured_text(output_text)
+        if structured:
+            return {"structured": _canonicalize(structured)}
+
+    tokens = _tokenize(output_text)
+    return {
+        "text": {
+            "tokens": tokens[:64],
+            "token_count": len(tokens),
+        }
+    }
+
+
+def _jaccard_similarity(a_tokens: list[str], b_tokens: list[str]) -> float:
+    a_set = set(a_tokens)
+    b_set = set(b_tokens)
+    if not a_set and not b_set:
+        return 1.0
+    if not a_set or not b_set:
+        return 0.0
+    return len(a_set & b_set) / len(a_set | b_set)
+
+
+class HostParityNormalizer:
+    def __init__(self, canonical_hosts: list[str] | None = None, text_similarity_threshold: float = 0.55):
+        self.canonical_hosts = [_normalize_host(host) for host in (canonical_hosts or get_canonical_hosts())]
+        self.text_similarity_threshold = max(0.0, min(1.0, float(text_similarity_threshold)))
+
+    def normalize_output(self, host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+        _ = context
+        host_name = _normalize_host(host)
+        payload = raw_output if isinstance(raw_output, dict) else {"output": raw_output}
+
+        exit_code_raw = payload.get("exit_code", 0)
+        try:
+            exit_code = int(exit_code_raw)
+        except Exception:
+            exit_code = 0
+
+        error = str(payload.get("error", "")).strip()
+        status = "error" if error or exit_code != 0 else "ok"
+        fallback = str(payload.get("fallback", "")).strip()
+        semantic_content = _build_semantic_content(host_name, payload.get("output", ""))
+
+        normalized = {
+            "host": host_name,
+            "status": status,
+            "exit_code": exit_code,
+            "exit_code_class": "success" if exit_code == 0 else "error",
+            "fallback": fallback,
+            "error": error,
+            "error_class": "none" if not error else "provider-error",
+            "semantic": semantic_content,
+        }
+        return normalized
+
+    def _is_semantically_equivalent(self, left: dict[str, Any], right: dict[str, Any]) -> tuple[bool, str]:
+        if left.get("status") != right.get("status"):
+            return False, "status mismatch"
+
+        left_semantic = left.get("semantic", {}) if isinstance(left.get("semantic"), dict) else {}
+        right_semantic = right.get("semantic", {}) if isinstance(right.get("semantic"), dict) else {}
+
+        left_structured = left_semantic.get("structured")
+        right_structured = right_semantic.get("structured")
+        if left_structured is not None and right_structured is not None:
+            if left_structured == right_structured:
+                return True, "structured-equivalent"
+            return False, "structured content mismatch"
+
+        left_tokens = left_semantic.get("text", {}).get("tokens", [])
+        right_tokens = right_semantic.get("text", {}).get("tokens", [])
+        if isinstance(left_tokens, list) and isinstance(right_tokens, list):
+            score = _jaccard_similarity(left_tokens, right_tokens)
+            if score >= self.text_similarity_threshold:
+                return True, f"text-equivalent score={score:.2f}"
+            return False, f"text drift score={score:.2f}"
+        return False, "semantic payload mismatch"
+
+    def check_parity(self, outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+        host_results: dict[str, dict[str, Any]] = {}
+        drift_details: list[str] = []
+
+        baseline_host = ""
+        baseline_output: dict[str, Any] | None = None
+
+        for host in self.canonical_hosts:
+            if host not in outputs_by_host:
+                host_results[host] = {
+                    "present": False,
+                    "passed": False,
+                    "reason": "missing host output",
+                    "normalized": {},
+                }
+                drift_details.append(f"missing host output: {host}")
+                continue
+
+            normalized = self.normalize_output(host, outputs_by_host[host], context)
+            host_results[host] = {
+                "present": True,
+                "passed": True,
+                "reason": "baseline",
+                "normalized": normalized,
+            }
+            if baseline_output is None:
+                baseline_host = host
+                baseline_output = normalized
+
+        if baseline_output is not None:
+            for host in self.canonical_hosts:
+                current = host_results.get(host, {})
+                if not current.get("present"):
+                    continue
+                normalized = current.get("normalized")
+                if not isinstance(normalized, dict):
+                    continue
+                equivalent, reason = self._is_semantically_equivalent(baseline_output, normalized)
+                current["passed"] = equivalent
+                current["reason"] = reason if host != baseline_host else "baseline"
+                if not equivalent:
+                    drift_details.append(f"semantic drift: {host} vs {baseline_host} ({reason})")
+
+        drift_detected = bool(drift_details)
+        passed = not drift_detected and all(
+            isinstance(host_results.get(host), dict)
+            and bool(host_results[host].get("present"))
+            and bool(host_results[host].get("passed"))
+            for host in self.canonical_hosts
+        )
+        return ParityCheckResult(
+            passed=passed,
+            host_results=host_results,
+            drift_detected=drift_detected,
+            drift_details=drift_details,
+        )
+
+
+def normalize_output(host: str, raw_output: Any, context: dict[str, Any] | None = None) -> dict[str, Any]:
+    return HostParityNormalizer().normalize_output(host, raw_output, context)
+
+
+def check_parity(outputs_by_host: dict[str, Any], context: dict[str, Any] | None = None) -> ParityCheckResult:
+    return HostParityNormalizer().check_parity(outputs_by_host, context)
+
+
+def emit_parity_report(
+    run_id: str,
+    results: ParityCheckResult,
+    *,
+    project_dir: str | None = None,
+) -> str:
+    base_dir = Path(project_dir or os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd()))
+    out_path = base_dir / ".omg" / "evidence" / f"host-parity-{run_id}.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    report = HostParityReport(
+        run_id=str(run_id),
+        timestamp=_now_iso(),
+        canonical_hosts=get_canonical_hosts(),
+        parity_results=asdict(results),
+        overall_status="ok" if results.passed else "drift",
+    )
+    out_path.write_text(json.dumps(asdict(report), indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+    return str(out_path)
+
+
+__all__ = [
+    "HostParityNormalizer",
+    "ParityCheckResult",
+    "HostParityReport",
+    "normalize_output",
+    "check_parity",
+    "emit_parity_report",
+]