claude-turing 4.2.0 → 4.4.0

package/templates/scripts/generate_model_card.py

@@ -18,6 +18,8 @@ import sys
 from datetime import datetime, timezone
 from pathlib import Path
 
+import yaml
+
 from scripts.turing_io import load_config, load_experiments
 
 
@@ -93,22 +95,113 @@ def load_model_contract(contract_path: str) -> dict:
     return {"version": version, "bundle_format": bundle_format, "raw": text}
 
 
+def load_registry_status(registry_path: str = "experiments/registry.yaml") -> dict | None:
+    """Load registry status for the best model."""
+    path = Path(registry_path)
+    if not path.exists():
+        return None
+    try:
+        with open(path) as f:
+            data = yaml.safe_load(f)
+        if isinstance(data, dict) and data.get("models"):
+            return data
+    except (Exception,):
+        pass
+    return None
+
+
+def compute_fairness_metrics(
+    predictions: list | None = None,
+    labels: list | None = None,
+    protected_attribute: list | None = None,
+    group_names: list[str] | None = None,
+) -> dict | None:
+    """Compute demographic parity and equal opportunity metrics.
+
+    Args:
+        predictions: Model predictions.
+        labels: True labels.
+        protected_attribute: Group membership for each sample.
+        group_names: Names of groups.
+
+    Returns:
+        Fairness metrics dict or None if insufficient data.
+    """
+    if predictions is None or protected_attribute is None:
+        return None
+    if len(predictions) != len(protected_attribute):
+        return None
+    if len(predictions) == 0:
+        return None
+
+    import numpy as np
+
+    preds = np.array(predictions)
+    groups = np.array(protected_attribute)
+    unique_groups = sorted(set(groups))
+
+    if group_names is None:
+        group_names = [str(g) for g in unique_groups]
+
+    # Demographic parity: P(Y_hat=1 | G=g) for each group
+    group_positive_rates = {}
+    for g, name in zip(unique_groups, group_names):
+        mask = groups == g
+        if mask.sum() == 0:
+            continue
+        rate = float(preds[mask].mean()) if preds[mask].size > 0 else 0
+        group_positive_rates[name] = round(rate, 4)
+
+    # Demographic parity difference
+    rates = list(group_positive_rates.values())
+    dp_diff = round(max(rates) - min(rates), 4) if len(rates) >= 2 else 0
+
+    result = {
+        "group_positive_rates": group_positive_rates,
+        "demographic_parity_difference": dp_diff,
+        "n_groups": len(unique_groups),
+    }
+
+    # Equal opportunity (if labels available): P(Y_hat=1 | Y=1, G=g)
+    if labels is not None and len(labels) == len(predictions):
+        labs = np.array(labels)
+        group_tpr = {}
+        for g, name in zip(unique_groups, group_names):
+            mask = (groups == g) & (labs == 1)
+            if mask.sum() == 0:
+                continue
+            tpr = float(preds[mask].mean()) if preds[mask].size > 0 else 0
+            group_tpr[name] = round(tpr, 4)
+
+        result["group_true_positive_rates"] = group_tpr
+        tpr_vals = list(group_tpr.values())
+        result["equal_opportunity_difference"] = round(max(tpr_vals) - min(tpr_vals), 4) if len(tpr_vals) >= 2 else 0
+
+    return result
+
+
 def generate_card(
     config_path: str = "config.yaml",
     log_path: str = "experiments/log.jsonl",
     contract_path: str = "model_contract.md",
     output_path: str | None = None,
+    include_fairness: bool = False,
+    fairness_data: dict | None = None,
+    registry_path: str = "experiments/registry.yaml",
 ) -> str:
     """Produce a model card markdown document.
 
     Combines information from the project config, experiment log,
-    and model contract into a standardized model card.
+    model contract, registry, and optional fairness data.
 
     Args:
         config_path: Path to config.yaml.
         log_path: Path to experiments/log.jsonl.
         contract_path: Path to model_contract.md.
         output_path: If given, write the card to this file.
+        include_fairness: If True, add fairness section.
+        fairness_data: Pre-computed fairness data {predictions, labels, protected_attribute}.
+        registry_path: Path to registry YAML.
 
     Returns:
         The model card as a markdown string.
@@ -247,7 +340,6 @@ def generate_card(
     if best:
         seed_study_path = Path("experiments/seed_studies") / f"{best.get('experiment_id', 'unknown')}-seeds.yaml"
         if seed_study_path.exists():
-            import yaml
             with open(seed_study_path) as f:
                 seed_study = yaml.safe_load(f) or {}
             if seed_study and "mean" in seed_study:
@@ -306,6 +398,57 @@ def generate_card(
         "- Not intended for: <placeholder for user to fill>",
     ])
 
+    # --- Registry Status ---
+    registry_data = load_registry_status(registry_path)
+    if registry_data and best:
+        exp_id = best.get("experiment_id", "")
+        for model in registry_data.get("models", []):
+            if model.get("exp_id") == exp_id:
+                lines.extend([
+                    "",
+                    "## Registry Status",
+                    "",
+                    f"- **Stage:** {model.get('stage', 'unregistered')}",
+                    f"- **Version:** {model.get('version', 'N/A')}",
+                    f"- **Registered:** {model.get('registered_at', 'N/A')[:10]}",
+                    f"- **Gates passed:** {', '.join(model.get('gates_passed', [])) or 'none'}",
+                ])
+                break
+
+    # --- Fairness ---
+    if include_fairness:
+        lines.extend([
+            "",
+            "## Fairness Analysis",
+            "",
+        ])
+        if fairness_data:
+            fairness = compute_fairness_metrics(
+                predictions=fairness_data.get("predictions"),
+                labels=fairness_data.get("labels"),
+                protected_attribute=fairness_data.get("protected_attribute"),
+                group_names=fairness_data.get("group_names"),
+            )
+            if fairness:
+                lines.append("### Demographic Parity")
+                lines.append("")
+                for group, rate in fairness.get("group_positive_rates", {}).items():
+                    lines.append(f"- **{group}:** {rate:.4f}")
+                lines.append(f"- **Parity difference:** {fairness['demographic_parity_difference']:.4f}")
+
+                if "group_true_positive_rates" in fairness:
+                    lines.append("")
+                    lines.append("### Equal Opportunity")
+                    lines.append("")
+                    for group, tpr in fairness["group_true_positive_rates"].items():
+                        lines.append(f"- **{group}:** {tpr:.4f}")
+                    lines.append(f"- **Opportunity difference:** {fairness['equal_opportunity_difference']:.4f}")
+            else:
+                lines.append("- Fairness analysis requested but insufficient data provided")
+        else:
+            lines.append("- Fairness analysis requested but no protected attribute data available")
+            lines.append("- Provide `--fairness-data` with predictions, labels, and protected attributes")
+
     # --- Ethical Considerations ---
     lines.extend([
         "",
@@ -354,9 +497,17 @@ def main() -> None:
     parser.add_argument("--log", default="experiments/log.jsonl", help="Path to experiment log")
     parser.add_argument("--contract", default="model_contract.md", help="Path to model contract")
     parser.add_argument("--output", default=None, help="Output path (default: print to stdout)")
+    parser.add_argument("--include", default=None, help="Include extra sections (e.g., 'fairness')")
+    parser.add_argument("--registry", default="experiments/registry.yaml", help="Path to model registry")
     args = parser.parse_args()
 
-    card = generate_card(args.config, args.log, args.contract, args.output)
+    include_fairness = args.include and "fairness" in args.include
+
+    card = generate_card(
+        args.config, args.log, args.contract, args.output,
+        include_fairness=include_fairness,
+        registry_path=args.registry,
+    )
     if args.output:
         print(f"Model card written to {args.output}")
     else:

package/templates/scripts/harness_doctor.py

@@ -0,0 +1,466 @@
+#!/usr/bin/env python3
+"""Harness self-diagnosis for the autoresearch pipeline.
+
+Checks environment health, project integrity, resource availability,
+and git state. Identifies common issues and auto-fixes where safe.
+
+Usage:
+    python scripts/harness_doctor.py
+    python scripts/harness_doctor.py --fix
+    python scripts/harness_doctor.py --verbose
+    python scripts/harness_doctor.py --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import json
+import shutil
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+from scripts.turing_io import load_config, load_experiments
+
+DEFAULT_LOG_PATH = "experiments/log.jsonl"
+MIN_DISK_MB = 1024  # 1 GB
+
+REQUIRED_SCRIPTS = ["train.py", "prepare.py", "evaluate.py"]
+REQUIRED_CONFIG_FIELDS = ["evaluation"]
+
+CHECK_CATEGORIES = ["environment", "dependencies", "config", "experiment_log",
+                     "scripts", "disk_space", "git_state"]
+
+
+# --- Individual Checks ---
+
+
+def check_environment() -> dict:
+    """Check Python environment health."""
+    issues = []
+    version = sys.version_info
+
+    if version < (3, 10):
+        issues.append(f"Python {version.major}.{version.minor} — recommend 3.10+")
+
+    # Check if running in a venv
+    in_venv = hasattr(sys, "real_prefix") or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix)
+
+    return {
+        "name": "Python environment",
+        "status": "PASS" if not issues else "WARN",
+        "detail": f"Python {version.major}.{version.minor}.{version.micro}, venv={'active' if in_venv else 'not active'}",
+        "issues": issues,
+    }
+
+
+def check_dependencies(required: list[str] | None = None) -> dict:
+    """Check that required packages are importable."""
+    if required is None:
+        required = ["yaml", "numpy", "sklearn", "pandas", "scipy"]
+
+    missing = []
+    for pkg in required:
+        try:
+            __import__(pkg)
+        except ImportError:
+            missing.append(pkg)
+
+    if missing:
+        return {
+            "name": "Dependencies",
+            "status": "FAIL",
+            "detail": f"{len(missing)} packages missing: {', '.join(missing)}",
+            "issues": [f"Cannot import: {pkg}" for pkg in missing],
+            "fix": f"pip install {' '.join(missing)}",
+        }
+
+    return {
+        "name": "Dependencies",
+        "status": "PASS",
+        "detail": f"All {len(required)} packages importable",
+        "issues": [],
+    }
+
+
+def check_config(config_path: str = "config.yaml") -> dict:
+    """Check config.yaml validity and required fields."""
+    path = Path(config_path)
+    issues = []
+
+    if not path.exists():
+        return {
+            "name": "Config",
+            "status": "FAIL",
+            "detail": f"{config_path} not found",
+            "issues": [f"{config_path} missing"],
+            "fix": "Run /turing:init to scaffold the project",
+        }
+
+    try:
+        with open(path) as f:
+            config = yaml.safe_load(f)
+    except yaml.YAMLError as e:
+        return {
+            "name": "Config",
+            "status": "FAIL",
+            "detail": f"{config_path} has YAML parse error",
+            "issues": [str(e)],
+        }
+
+    if not isinstance(config, dict):
+        return {
+            "name": "Config",
+            "status": "FAIL",
+            "detail": f"{config_path} is not a YAML mapping",
+            "issues": ["Config must be a YAML dict"],
+        }
+
+    for field in REQUIRED_CONFIG_FIELDS:
+        if field not in config:
+            issues.append(f"Missing required field: {field}")
+
+    status = "PASS" if not issues else "WARN"
+    return {
+        "name": "Config",
+        "status": status,
+        "detail": f"{config_path} valid, {len(config)} top-level keys",
+        "issues": issues,
+    }
+
+
+def check_experiment_log(log_path: str = DEFAULT_LOG_PATH) -> dict:
+    """Check experiment log integrity."""
+    path = Path(log_path)
+
+    if not path.exists():
+        return {
+            "name": "Experiment log",
+            "status": "WARN",
+            "detail": "No experiment log yet — run /turing:train first",
+            "issues": [],
+        }
+
+    issues = []
+    total_lines = 0
+    valid_lines = 0
+    corrupt_lines = []
+    missing_fields = []
+
+    with open(path) as f:
+        for i, line in enumerate(f, 1):
+            total_lines += 1
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+                valid_lines += 1
+                # Check for expected fields
+                if "metrics" not in entry:
+                    missing_fields.append(i)
+            except json.JSONDecodeError:
+                corrupt_lines.append(i)
+
+    if corrupt_lines:
+        issues.append(f"{len(corrupt_lines)} corrupt lines: {corrupt_lines[:5]}")
+    if missing_fields:
+        issues.append(f"{len(missing_fields)} entries missing 'metrics' field")
+
+    status = "FAIL" if corrupt_lines else ("WARN" if missing_fields else "PASS")
+    return {
+        "name": "Experiment log",
+        "status": status,
+        "detail": f"{valid_lines}/{total_lines} valid entries",
+        "issues": issues,
+        "corrupt_lines": corrupt_lines,
+        "fixable": len(corrupt_lines) > 0,
+    }
+
+
+def check_scripts(script_dir: str = ".") -> dict:
+    """Check that required scripts exist and are syntactically valid."""
+    issues = []
+    checked = 0
+
+    for script in REQUIRED_SCRIPTS:
+        path = Path(script_dir) / script
+        if not path.exists():
+            issues.append(f"{script} not found")
+            continue
+
+        try:
+            source = path.read_text(encoding="utf-8")
+            ast.parse(source, filename=script)
+            checked += 1
+        except SyntaxError as e:
+            issues.append(f"{script} has syntax error: {e.msg} (line {e.lineno})")
+
+    status = "PASS" if not issues else ("WARN" if checked > 0 else "FAIL")
+    return {
+        "name": "Scripts",
+        "status": status,
+        "detail": f"{checked}/{len(REQUIRED_SCRIPTS)} scripts valid",
+        "issues": issues,
+    }
+
+
+def check_disk_space(project_dir: str = ".", min_mb: int = MIN_DISK_MB) -> dict:
+    """Check available disk space."""
+    try:
+        usage = shutil.disk_usage(project_dir)
+        free_mb = usage.free / (1024 * 1024)
+        total_mb = usage.total / (1024 * 1024)
+
+        if free_mb < min_mb:
+            return {
+                "name": "Disk space",
+                "status": "FAIL",
+                "detail": f"{free_mb:.0f} MB remaining — below {min_mb} MB threshold",
+                "issues": [f"Low disk space: {free_mb:.0f} MB free of {total_mb:.0f} MB"],
+                "fix": "Run /turing:archive to reclaim space",
+                "free_mb": round(free_mb),
+            }
+
+        return {
+            "name": "Disk space",
+            "status": "PASS",
+            "detail": f"{free_mb:.0f} MB free",
+            "issues": [],
+            "free_mb": round(free_mb),
+        }
+    except OSError as e:
+        return {
+            "name": "Disk space",
+            "status": "WARN",
+            "detail": f"Could not check disk: {e}",
+            "issues": [str(e)],
+        }
+
+
+def check_git_state(project_dir: str = ".") -> dict:
+    """Check git working tree state."""
+    import subprocess
+
+    try:
+        result = subprocess.run(
+            ["git", "status", "--porcelain"],
+            capture_output=True, text=True, timeout=10,
+            cwd=project_dir,
+        )
+        if result.returncode != 0:
+            return {
+                "name": "Git state",
+                "status": "WARN",
+                "detail": "Not a git repository or git not available",
+                "issues": [],
+            }
+
+        modified = result.stdout.strip().split("\n") if result.stdout.strip() else []
+        issues = []
+
+        # Check if critical files are modified
+        critical = {"evaluate.py", "prepare.py"}
+        for line in modified:
+            if len(line) >= 3:
+                filepath = line[3:].strip()
+                if any(c in filepath for c in critical):
+                    issues.append(f"Uncommitted changes to {filepath} — evaluation integrity at risk")
+
+        status = "WARN" if issues else "PASS"
+        detail = "Working tree clean" if not modified else f"{len(modified)} modified files"
+
+        return {
+            "name": "Git state",
+            "status": status,
+            "detail": detail,
+            "issues": issues,
+        }
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        return {
+            "name": "Git state",
+            "status": "WARN",
+            "detail": "Git check skipped (timeout or not available)",
+            "issues": [],
+        }
+
+
+# --- Fix Operations ---
+
+
+def fix_corrupt_log(log_path: str = DEFAULT_LOG_PATH) -> dict:
+    """Remove corrupt lines from experiment log."""
+    path = Path(log_path)
+    if not path.exists():
+        return {"fixed": False, "reason": "Log not found"}
+
+    valid_lines = []
+    removed = 0
+
+    with open(path) as f:
+        for line in f:
+            line_stripped = line.strip()
+            if not line_stripped:
+                continue
+            try:
+                json.loads(line_stripped)
+                valid_lines.append(line)
+            except json.JSONDecodeError:
+                removed += 1
+
+    if removed > 0:
+        # Backup first
+        backup = path.with_suffix(".jsonl.bak")
+        shutil.copy2(path, backup)
+        with open(path, "w") as f:
+            f.writelines(valid_lines)
+        return {"fixed": True, "removed": removed, "backup": str(backup)}
+
+    return {"fixed": False, "reason": "No corrupt lines found"}
+
+
+# --- Full Doctor ---
+
+
+def run_doctor(
+    config_path: str = "config.yaml",
+    log_path: str = DEFAULT_LOG_PATH,
+    fix: bool = False,
+    verbose: bool = False,
+) -> dict:
+    """Run all diagnostic checks.
+
+    Args:
+        config_path: Path to config.yaml.
+        log_path: Path to experiment log.
+        fix: If True, auto-fix safe issues.
+        verbose: Include detailed info.
+
+    Returns:
+        Doctor report with all check results and score.
+    """
+    checks = [
+        check_environment(),
+        check_dependencies(),
+        check_config(config_path),
+        check_experiment_log(log_path),
+        check_scripts(),
+        check_disk_space(),
+        check_git_state(),
+    ]
+
+    # Apply fixes if requested
+    fixes_applied = []
+    if fix:
+        log_check = next((c for c in checks if c["name"] == "Experiment log"), None)
+        if log_check and log_check.get("fixable"):
+            fix_result = fix_corrupt_log(log_path)
+            if fix_result.get("fixed"):
+                fixes_applied.append(f"Removed {fix_result['removed']} corrupt log entries (backup: {fix_result['backup']})")
+                # Re-run log check
+                for i, c in enumerate(checks):
+                    if c["name"] == "Experiment log":
+                        checks[i] = check_experiment_log(log_path)
+                        break
+
+    # Compute score
+    passed = sum(1 for c in checks if c["status"] == "PASS")
+    warned = sum(1 for c in checks if c["status"] == "WARN")
+    failed = sum(1 for c in checks if c["status"] == "FAIL")
+    total = len(checks)
+
+    return {
+        "checks": checks,
+        "score": {"passed": passed, "warned": warned, "failed": failed, "total": total},
+        "fixes_applied": fixes_applied,
+        "overall": "HEALTHY" if failed == 0 and warned == 0 else ("DEGRADED" if failed == 0 else "UNHEALTHY"),
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+# --- Report Formatting ---
+
+
+def save_doctor_report(report: dict, output_dir: str = "experiments/doctor") -> Path:
+    """Save doctor report to YAML."""
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    filepath = out_path / f"doctor-{ts}.yaml"
+    with open(filepath, "w") as f:
+        yaml.dump(report, f, default_flow_style=False, sort_keys=False)
+    return filepath
+
+
+def format_doctor_report(report: dict) -> str:
+    """Format doctor report as readable text."""
+    lines = ["# Turing Doctor Report", ""]
+
+    status_icons = {"PASS": "✓ PASS ", "WARN": "⚠ WARN ", "FAIL": "✗ FAIL "}
+
+    for check in report.get("checks", []):
+        icon = status_icons.get(check["status"], "? ")
+        lines.append(f"{icon} {check['name']} ({check.get('detail', '')})")
+        for issue in check.get("issues", []):
+            lines.append(f"         {issue}")
+        fix = check.get("fix")
+        if fix:
+            lines.append(f"         Fix: {fix}")
+
+    score = report.get("score", {})
+    lines.extend([
+        "",
+        f"Score: {score.get('passed', 0)}/{score.get('total', 0)} pass, "
+        f"{score.get('warned', 0)} warning{'s' if score.get('warned', 0) != 1 else ''}, "
+        f"{score.get('failed', 0)} failure{'s' if score.get('failed', 0) != 1 else ''}",
+        f"Overall: {report.get('overall', 'UNKNOWN')}",
+    ])
+
+    fixes = report.get("fixes_applied", [])
+    if fixes:
+        lines.extend(["", "Fixes applied:"])
+        for f in fixes:
+            lines.append(f"  - {f}")
+
+    lines.append("")
+    lines.append(f"*Generated: {report.get('generated_at', 'N/A')}*")
+    return "\n".join(lines)
+
+
+# --- CLI ---
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Harness self-diagnosis — check environment, project, and resource health"
+    )
+    parser.add_argument("--fix", action="store_true", help="Auto-fix safe issues")
+    parser.add_argument("--verbose", action="store_true", help="Show detailed info")
+    parser.add_argument("--config", default="config.yaml", help="Path to config.yaml")
+    parser.add_argument("--log", default=DEFAULT_LOG_PATH, help="Path to experiment log")
+    parser.add_argument("--json", action="store_true", help="Output raw JSON")
+
+    args = parser.parse_args()
+
+    report = run_doctor(
+        config_path=args.config,
+        log_path=args.log,
+        fix=args.fix,
+        verbose=args.verbose,
+    )
+
+    if args.json:
+        print(json.dumps(report, indent=2))
+    else:
+        print(format_doctor_report(report))
+
+    saved = save_doctor_report(report)
+    if not args.json:
+        print(f"\nSaved: {saved}")
+
+
+if __name__ == "__main__":
+    main()