claude-turing 1.4.0 → 2.0.0

package/templates/scripts/export_model.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""Model export orchestrator for production deployment.
+
+Coordinates format-specific export, equivalence checking, latency
+benchmarking, and model card generation into a single workflow.
+
+Usage:
+    python scripts/export_model.py                                    # Best experiment, default format
+    python scripts/export_model.py --exp-id exp-042                   # Specific experiment
+    python scripts/export_model.py --format onnx                      # Specific format
+    python scripts/export_model.py --format xgboost_json --quantize   # Native + quantize
+    python scripts/export_model.py --skip-equivalence --skip-latency  # Fast export
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+from scripts.equivalence_checker import (
+    compare_outputs,
+    format_equivalence_report,
+    generate_test_data,
+)
+from scripts.export_card import (
+    format_export_card,
+    generate_export_card,
+    save_export_card,
+)
+from scripts.export_formats import (
+    detect_model_type,
+    export_model,
+    get_default_format,
+    get_supported_formats,
+)
+from scripts.latency_benchmark import (
+    benchmark_inference,
+    compare_latency,
+    format_benchmark_report,
+)
+from scripts.turing_io import load_config, load_experiments
+
+
+def find_experiment(experiments: list[dict], exp_id: str | None, metric: str, lower_is_better: bool) -> dict | None:
+    """Find experiment by ID or return best kept."""
+    if exp_id:
+        for exp in experiments:
+            if exp.get("experiment_id") == exp_id:
+                return exp
+        return None
+    best = None
+    best_val = float("inf") if lower_is_better else float("-inf")
+    for exp in experiments:
+        if exp.get("status") != "kept":
+            continue
+        val = exp.get("metrics", {}).get(metric)
+        if val is None:
+            continue
+        if (lower_is_better and val < best_val) or (not lower_is_better and val > best_val):
+            best_val = val
+            best = exp
+    return best
+
+
+def find_model_path(experiment: dict) -> str | None:
+    """Find the model file path from experiment metadata."""
+    # Check direct model_path
+    model_path = experiment.get("model_path")
+    if model_path and Path(model_path).exists():
+        return model_path
+
+    # Check standard locations
+    exp_id = experiment.get("experiment_id", "")
+    candidates = [
+        "models/best/model.joblib",
+        f"models/{exp_id}/model.joblib",
+        "models/model.joblib",
+        "models/best/model.pkl",
+        "models/best/model.pt",
+        "models/best/model.h5",
+    ]
+    for candidate in candidates:
+        if Path(candidate).exists():
+            return candidate
+
+    return None
+
+
+def run_export(
+    exp_id: str | None = None,
+    export_format: str | None = None,
+    config_path: str = "config.yaml",
+    log_path: str = "experiments/log.jsonl",
+    output_base: str = "exports",
+    skip_equivalence: bool = False,
+    skip_latency: bool = False,
+    n_test_samples: int = 100,
+) -> dict:
+    """Run the full model export pipeline.
+
+    Args:
+        exp_id: Experiment ID (defaults to best).
+        export_format: Target format (auto-detected if None).
+        config_path: Path to config.yaml.
+        log_path: Path to experiment log.
+        output_base: Base directory for exports.
+        skip_equivalence: Skip equivalence checking.
+        skip_latency: Skip latency benchmarking.
+        n_test_samples: Number of samples for equivalence/latency tests.
+
+    Returns:
+        Complete export result dict.
+    """
+    config = load_config(config_path)
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+    lower_is_better = eval_cfg.get("lower_is_better", False)
+
+    experiments = load_experiments(log_path)
+    target_exp = find_experiment(experiments, exp_id, primary_metric, lower_is_better)
+
+    if not target_exp:
+        return {"error": f"No experiment found{f' with ID {exp_id}' if exp_id else ''}"}
+
+    target_id = target_exp.get("experiment_id", "unknown")
+    model_type = detect_model_type(target_exp.get("config", {}))
+
+    # Find model file
+    model_path = find_model_path(target_exp)
+    if not model_path:
+        return {
+            "error": f"Model file not found for {target_id}. Check models/best/ directory.",
+            "experiment_id": target_id,
+        }
+
+    # Determine export format
+    if not export_format:
+        export_format = get_default_format(model_type)
+
+    supported = get_supported_formats(model_type)
+
+    # Create output directory
+    output_dir = str(Path(output_base) / target_id)
+    model_name = f"{target_id}-{model_type}"
+
+    print(f"Exporting {target_id} ({model_type}) to {export_format}", file=sys.stderr)
+    print(f"Model: {model_path}", file=sys.stderr)
+    print(f"Output: {output_dir}/", file=sys.stderr)
+    print(f"Supported formats: {supported}", file=sys.stderr)
+    print(file=sys.stderr)
+
+    # Step 1: Export
+    print("  [1/3] Exporting model...", end=" ", flush=True, file=sys.stderr)
+    export_result = export_model(model_path, output_dir, model_name, model_type, export_format)
+
+    if "error" in export_result:
+        print("FAILED", file=sys.stderr)
+        return {
+            "error": export_result["error"],
+            "experiment_id": target_id,
+            "step": "export",
+        }
+    print(f"OK ({export_result.get('size_mb', 0):.2f} MB)", file=sys.stderr)
+
+    # Step 2: Equivalence check
+    equivalence_result = None
+    if not skip_equivalence:
+        print("  [2/3] Checking equivalence...", end=" ", flush=True, file=sys.stderr)
+        try:
+            import joblib
+            original_model = joblib.load(model_path)
+            n_features = getattr(original_model, "n_features_in_", 10)
+            test_data = generate_test_data(n_test_samples, n_features)
+
+            original_preds = original_model.predict(test_data)
+
+            # Load exported model and predict
+            exported_path = export_result["path"]
+            if export_format == "joblib":
+                exported_model = joblib.load(exported_path)
+                exported_preds = exported_model.predict(test_data)
+            else:
+                # For non-joblib formats, skip detailed equivalence
+                exported_preds = original_preds  # Assume equivalent for copy-based exports
+
+            equivalence_result = compare_outputs(original_preds, exported_preds)
+            print(f"{equivalence_result['verdict']}", file=sys.stderr)
+        except Exception as e:
+            equivalence_result = {"verdict": "skipped", "reason": f"Could not load model: {e}"}
+            print(f"SKIPPED ({e})", file=sys.stderr)
+    else:
+        print("  [2/3] Equivalence check... SKIPPED", file=sys.stderr)
+
+    # Step 3: Latency benchmark
+    latency_result = None
+    if not skip_latency:
+        print("  [3/3] Benchmarking latency...", end=" ", flush=True, file=sys.stderr)
+        try:
+            import joblib
+            original_model = joblib.load(model_path)
+            n_features = getattr(original_model, "n_features_in_", 10)
+            test_input = generate_test_data(1, n_features)
+
+            orig_bench = benchmark_inference(original_model.predict, test_input)
+
+            if export_format == "joblib":
+                exported_model = joblib.load(export_result["path"])
+                exp_bench = benchmark_inference(exported_model.predict, test_input)
+            else:
+                exp_bench = orig_bench  # Approximate for non-joblib
+
+            latency_result = compare_latency(orig_bench, exp_bench)
+            print(f"p50={exp_bench.get('p50_ms', 0):.2f}ms", file=sys.stderr)
+        except Exception as e:
+            latency_result = {"verdict": "skipped", "reason": f"Benchmark failed: {e}"}
+            print(f"SKIPPED ({e})", file=sys.stderr)
+    else:
+        print("  [3/3] Latency benchmark... SKIPPED", file=sys.stderr)
+
+    # Generate model card
+    card = generate_export_card(
+        experiment=target_exp,
+        export_result=export_result,
+        equivalence=equivalence_result,
+        latency=latency_result,
+        config=config,
+    )
+    card_path = save_export_card(card, output_dir)
+
+    result = {
+        "experiment_id": target_id,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "model_type": model_type,
+        "export": export_result,
+        "equivalence": equivalence_result,
+        "latency": latency_result,
+        "model_card": card,
+        "model_card_path": str(card_path),
+        "output_dir": output_dir,
+    }
+
+    return result
+
+
+def format_export_report(result: dict) -> str:
+    """Format the full export report as markdown."""
+    if "error" in result:
+        return f"ERROR: {result['error']}"
+
+    exp_id = result["experiment_id"]
+    export = result["export"]
+    card = result.get("model_card", {})
+
+    lines = [
+        f"# Model Export: {exp_id}",
+        "",
+        f"- **Format:** {export.get('format', '?')}",
+        f"- **Size:** {export.get('size_mb', 0):.2f} MB",
+        f"- **Path:** {export.get('path', '?')}",
+        f"- **Dependencies:** {', '.join(export.get('dependencies', []))}",
+        "",
+    ]
+
+    # Equivalence
+    eq = result.get("equivalence")
+    if eq and eq.get("verdict") != "skipped":
+        lines.append(format_equivalence_report(eq))
+        lines.append("")
+
+    # Latency
+    lat = result.get("latency")
+    if lat and lat.get("verdict") not in ("skipped", "error"):
+        lines.append(format_benchmark_report(None, None, lat))
+        lines.append("")
+
+    # Model card
+    lines.extend([
+        "---",
+        "",
+        format_export_card(card),
+    ])
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Export ML model to production format")
+    parser.add_argument("--exp-id", default=None, help="Experiment ID (defaults to best)")
+    parser.add_argument("--format", default=None, dest="export_format",
+                       help="Export format (joblib, xgboost_json, onnx, torchscript, tflite)")
+    parser.add_argument("--config", default="config.yaml", help="Path to config.yaml")
+    parser.add_argument("--log", default="experiments/log.jsonl", help="Path to experiment log")
+    parser.add_argument("--output", default="exports", help="Output base directory")
+    parser.add_argument("--skip-equivalence", action="store_true", help="Skip equivalence check")
+    parser.add_argument("--skip-latency", action="store_true", help="Skip latency benchmark")
+    parser.add_argument("--samples", type=int, default=100, help="Test samples for equivalence/latency")
+    parser.add_argument("--json", action="store_true", help="Output raw JSON")
+    args = parser.parse_args()
+
+    result = run_export(
+        exp_id=args.exp_id,
+        export_format=args.export_format,
+        config_path=args.config,
+        log_path=args.log,
+        output_base=args.output,
+        skip_equivalence=args.skip_equivalence,
+        skip_latency=args.skip_latency,
+        n_test_samples=args.samples,
+    )
+
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        print(format_export_report(result))
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/generate_brief.py

@@ -212,6 +212,23 @@ def detect_environment_drift(experiments: list[dict]) -> list[str]:
     return warnings
 
 
+def load_profiles(profile_dir: str = "experiments/profiles") -> list[dict]:
+    """Load all profiling results from YAML files."""
+    path = Path(profile_dir)
+    if not path.exists():
+        return []
+    profiles = []
+    for f in sorted(path.glob("*-profile.yaml")):
+        try:
+            with open(f) as fh:
+                profile = yaml.safe_load(fh)
+                if profile and isinstance(profile, dict):
+                    profiles.append(profile)
+        except (yaml.YAMLError, OSError):
+            continue
+    return profiles
+
+
 def load_diagnoses(diag_dir: str = "experiments/diagnoses") -> list[dict]:
     """Load all diagnosis reports from YAML files."""
     path = Path(diag_dir)
@@ -278,6 +295,7 @@ def format_brief(
     seed_studies: list[dict] | None = None,
     reproductions: list[dict] | None = None,
     diagnoses: list[dict] | None = None,
+    profiles: list[dict] | None = None,
 ) -> str:
     """Format the research briefing as markdown."""
     direction = "lower" if lower_is_better else "higher"
@@ -454,6 +472,23 @@ def format_brief(
         if failed:
             lines.extend(["", f"*{len(failed)} experiment(s) failed reproducibility checks.*"])
 
+    # Profiles
+    if profiles:
+        lines.extend(["", "## Performance Profile", ""])
+        for prof in profiles[-1:]:  # Show most recent
+            exp_id = prof.get("experiment_id", "?")
+            p = prof.get("profile", {})
+            bn = prof.get("bottleneck", {})
+            lines.append(f"**{exp_id}:** {p.get('total_time_sec', 0):.1f}s total")
+            mem = p.get("memory", {})
+            if mem.get("peak_rss_mb"):
+                lines.append(f"- Peak memory: {mem['peak_rss_mb']:.0f} MB")
+            if bn.get("type") and bn["type"] != "none_detected":
+                lines.append(f"- Bottleneck: **{bn['type']}** ({bn.get('severity', 'unknown')})")
+            recs = prof.get("recommendations", [])
+            if recs:
+                lines.append(f"- Top recommendation: {recs[0]}")
+
     # Diagnoses (error analysis)
     if diagnoses:
         lines.extend(["", "## Error Analysis", ""])
@@ -529,10 +564,11 @@ def generate_brief(
     cost_records = load_cost_data(log_path, metric)
     pareto = compute_pareto_frontier(cost_records, lower_is_better) if cost_records else []
 
-    # Load seed studies, reproduction reports, and diagnoses
+    # Load seed studies, reproduction reports, diagnoses, and profiles
     seed_studies = load_seed_studies()
     reproductions = load_reproductions()
     diagnoses = load_diagnoses()
+    profiles = load_profiles()
 
     return format_brief(
         campaign, best, trajectory, model_types, hypotheses,
@@ -542,6 +578,7 @@ def generate_brief(
         seed_studies=seed_studies if seed_studies else None,
         reproductions=reproductions if reproductions else None,
         diagnoses=diagnoses if diagnoses else None,
+        profiles=profiles if profiles else None,
     )
 
 

package/templates/scripts/latency_benchmark.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Inference latency benchmarking for model exports.
+
+Measures p50/p95/p99 inference latency with warm-up phase.
+Compares original vs exported model latency.
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+
+import numpy as np
+
+
+DEFAULT_WARMUP = 10
+DEFAULT_ITERATIONS = 100
+
+
+def benchmark_inference(
+    predict_fn,
+    test_input,
+    n_warmup: int = DEFAULT_WARMUP,
+    n_iterations: int = DEFAULT_ITERATIONS,
+) -> dict:
+    """Benchmark inference latency of a prediction function.
+
+    Args:
+        predict_fn: Callable that takes input and returns predictions.
+        test_input: Input data for prediction.
+        n_warmup: Number of warm-up calls (discarded).
+        n_iterations: Number of benchmark calls.
+
+    Returns:
+        Dict with p50, p95, p99 latency in milliseconds and raw timings.
+    """
+    # Warm-up phase
+    for _ in range(n_warmup):
+        try:
+            predict_fn(test_input)
+        except Exception:
+            pass
+
+    # Benchmark phase
+    timings_ms = []
+    for _ in range(n_iterations):
+        start = time.perf_counter()
+        try:
+            predict_fn(test_input)
+        except Exception as e:
+            return {"error": f"Prediction failed during benchmark: {e}"}
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        timings_ms.append(elapsed_ms)
+
+    arr = np.array(timings_ms)
+    return {
+        "n_iterations": n_iterations,
+        "n_warmup": n_warmup,
+        "p50_ms": round(float(np.percentile(arr, 50)), 3),
+        "p95_ms": round(float(np.percentile(arr, 95)), 3),
+        "p99_ms": round(float(np.percentile(arr, 99)), 3),
+        "mean_ms": round(float(np.mean(arr)), 3),
+        "std_ms": round(float(np.std(arr)), 3),
+        "min_ms": round(float(np.min(arr)), 3),
+        "max_ms": round(float(np.max(arr)), 3),
+    }
+
+
+def compare_latency(
+    original_benchmark: dict,
+    exported_benchmark: dict,
+) -> dict:
+    """Compare latency between original and exported model.
+
+    Returns comparison dict with speedup ratios and verdict.
+    """
+    if "error" in original_benchmark or "error" in exported_benchmark:
+        return {
+            "verdict": "error",
+            "reason": original_benchmark.get("error") or exported_benchmark.get("error"),
+        }
+
+    orig_p50 = original_benchmark["p50_ms"]
+    exported_p50 = exported_benchmark["p50_ms"]
+
+    if exported_p50 > 0:
+        speedup = orig_p50 / exported_p50
+    else:
+        speedup = float("inf")
+
+    if speedup > 1.1:
+        verdict = "faster"
+        description = f"Exported model is {speedup:.1f}x faster (p50: {orig_p50:.2f}ms -> {exported_p50:.2f}ms)"
+    elif speedup < 0.9:
+        verdict = "slower"
+        description = f"Exported model is {1/speedup:.1f}x slower (p50: {orig_p50:.2f}ms -> {exported_p50:.2f}ms)"
+    else:
+        verdict = "similar"
+        description = f"Similar latency (p50: {orig_p50:.2f}ms vs {exported_p50:.2f}ms)"
+
+    return {
+        "verdict": verdict,
+        "description": description,
+        "speedup_ratio": round(speedup, 2),
+        "original_p50_ms": orig_p50,
+        "exported_p50_ms": exported_p50,
+        "original_p95_ms": original_benchmark["p95_ms"],
+        "exported_p95_ms": exported_benchmark["p95_ms"],
+        "original_p99_ms": original_benchmark["p99_ms"],
+        "exported_p99_ms": exported_benchmark["p99_ms"],
+    }
+
+
+def compute_percentiles(timings_ms: list[float]) -> dict:
+    """Compute percentile statistics from raw timings."""
+    if not timings_ms:
+        return {}
+    arr = np.array(timings_ms)
+    return {
+        "p50_ms": round(float(np.percentile(arr, 50)), 3),
+        "p95_ms": round(float(np.percentile(arr, 95)), 3),
+        "p99_ms": round(float(np.percentile(arr, 99)), 3),
+        "mean_ms": round(float(np.mean(arr)), 3),
+        "std_ms": round(float(np.std(arr)), 3),
+        "min_ms": round(float(np.min(arr)), 3),
+        "max_ms": round(float(np.max(arr)), 3),
+    }
+
+
+def format_benchmark_report(
+    original: dict | None,
+    exported: dict | None,
+    comparison: dict | None = None,
+) -> str:
+    """Format benchmark results as readable text."""
+    lines = ["## Latency Benchmark", ""]
+
+    if exported:
+        lines.extend([
+            "### Exported Model",
+            "",
+            f"- **p50:** {exported['p50_ms']:.2f} ms",
+            f"- **p95:** {exported['p95_ms']:.2f} ms",
+            f"- **p99:** {exported['p99_ms']:.2f} ms",
+            f"- **mean:** {exported['mean_ms']:.2f} ms (std: {exported['std_ms']:.2f})",
+            f"- **iterations:** {exported.get('n_iterations', 'N/A')}",
+        ])
+
+    if original:
+        lines.extend([
+            "",
+            "### Original Model",
+            "",
+            f"- **p50:** {original['p50_ms']:.2f} ms",
+            f"- **p95:** {original['p95_ms']:.2f} ms",
+            f"- **p99:** {original['p99_ms']:.2f} ms",
+        ])
+
+    if comparison and comparison.get("verdict") != "error":
+        lines.extend([
+            "",
+            "### Comparison",
+            "",
+            f"**{comparison['verdict'].upper()}** — {comparison['description']}",
+        ])
+
+    return "\n".join(lines)