claude-turing 1.4.0 → 2.0.0

package/templates/scripts/export_card.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Deployment model card generation for exported models.
+
+Produces a structured model card with metrics, seed study results,
+export format, equivalence check, latency benchmarks, and dependencies.
+"""
+
+from __future__ import annotations
+
+import yaml
+from datetime import datetime, timezone
+from pathlib import Path
+
+from scripts.turing_io import load_config, load_seed_study
+
+
+def generate_export_card(
+    experiment: dict,
+    export_result: dict,
+    equivalence: dict | None = None,
+    latency: dict | None = None,
+    config: dict | None = None,
+) -> dict:
+    """Generate a deployment model card for an exported model.
+
+    Args:
+        experiment: Original experiment dict from log.jsonl.
+        export_result: Result from export_formats.export_model().
+        equivalence: Result from equivalence_checker.compare_outputs().
+        latency: Latency comparison from latency_benchmark.compare_latency().
+        config: Project config dict.
+
+    Returns:
+        Model card dict.
+    """
+    exp_id = experiment.get("experiment_id", "unknown")
+    metrics = experiment.get("metrics", {})
+    exp_config = experiment.get("config", {})
+    model_type = exp_config.get("model_type", config.get("model", {}).get("type", "unknown") if config else "unknown")
+
+    eval_cfg = config.get("evaluation", {}) if config else {}
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+    task_desc = config.get("task_description", eval_cfg.get("primary_metric", "N/A")) if config else "N/A"
+
+    card = {
+        "name": f"{exp_id}-{model_type}",
+        "experiment_id": exp_id,
+        "task": task_desc,
+        "model_type": model_type,
+        "primary_metric": primary_metric,
+        "metrics": {k: round(v, 4) if isinstance(v, float) else v for k, v in metrics.items()},
+        "export_format": export_result.get("format", "unknown"),
+        "export_path": export_result.get("path"),
+        "size_mb": export_result.get("size_mb", 0),
+        "dependencies": export_result.get("dependencies", []),
+        "training_date": experiment.get("timestamp", "unknown"),
+        "export_date": datetime.now(timezone.utc).isoformat(),
+    }
+
+    # Seed study (if available)
+    seed_study = load_seed_study(exp_id)
+    if seed_study and "mean" in seed_study:
+        card["seed_study"] = {
+            "mean": seed_study["mean"],
+            "std": seed_study.get("std", 0),
+            "cv_percent": seed_study.get("cv_percent", 0),
+            "seed_sensitive": seed_study.get("seed_sensitive", False),
+            "seeds_tested": len(seed_study.get("seeds_run", [])),
+        }
+
+    # Equivalence check
+    if equivalence:
+        card["equivalence"] = {
+            "verdict": equivalence.get("verdict", "unknown"),
+            "max_delta": equivalence.get("max_delta", 0),
+            "n_samples_tested": equivalence.get("n_samples", 0),
+        }
+
+    # Latency benchmark
+    if latency and latency.get("verdict") != "error":
+        card["inference_latency"] = {
+            "exported_p50_ms": latency.get("exported_p50_ms"),
+            "exported_p95_ms": latency.get("exported_p95_ms"),
+            "original_p50_ms": latency.get("original_p50_ms"),
+            "speedup": latency.get("speedup_ratio"),
+        }
+
+    # Environment
+    env = experiment.get("environment")
+    if env:
+        card["training_environment"] = {
+            "python_version": env.get("python_version"),
+            "gpu": env.get("gpu_name") or env.get("gpu"),
+        }
+
+    return card
+
+
+def save_export_card(card: dict, output_dir: str) -> Path:
+    """Save export model card to YAML file."""
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    filepath = out_path / "model_card.yaml"
+    with open(filepath, "w") as f:
+        yaml.dump(card, f, default_flow_style=False, sort_keys=False)
+    return filepath
+
+
+def format_export_card(card: dict) -> str:
+    """Format export model card as readable markdown."""
+    lines = [
+        f"# Export Model Card: {card.get('name', 'unknown')}",
+        "",
+        f"- **Experiment:** {card.get('experiment_id', '?')}",
+        f"- **Task:** {card.get('task', 'N/A')}",
+        f"- **Model type:** {card.get('model_type', '?')}",
+        f"- **Export format:** {card.get('export_format', '?')}",
+        f"- **Size:** {card.get('size_mb', 0):.2f} MB",
+        f"- **Dependencies:** {', '.join(card.get('dependencies', []))}",
+        "",
+        "## Metrics",
+        "",
+    ]
+
+    for metric, value in card.get("metrics", {}).items():
+        if isinstance(value, float):
+            lines.append(f"- **{metric}:** {value:.4f}")
+        else:
+            lines.append(f"- **{metric}:** {value}")
+
+    # Seed study
+    seed = card.get("seed_study")
+    if seed:
+        status = "SEED-SENSITIVE" if seed.get("seed_sensitive") else "STABLE"
+        lines.extend([
+            "",
+            "## Seed Study",
+            "",
+            f"- **Status:** {status}",
+            f"- **Mean ± Std:** {seed['mean']:.4f} ± {seed.get('std', 0):.4f}",
+            f"- **CV:** {seed.get('cv_percent', 0):.2f}%",
+            f"- **Seeds tested:** {seed.get('seeds_tested', 0)}",
+        ])
+
+    # Equivalence
+    eq = card.get("equivalence")
+    if eq:
+        verdict_markers = {
+            "equivalent": "PASS (exact)",
+            "approximately_equivalent": "PASS (approx)",
+            "divergent": "FAIL",
+        }
+        marker = verdict_markers.get(eq["verdict"], eq["verdict"])
+        lines.extend([
+            "",
+            "## Equivalence",
+            "",
+            f"- **Verdict:** {marker}",
+            f"- **Max delta:** {eq.get('max_delta', 0):.2e}",
+            f"- **Samples tested:** {eq.get('n_samples_tested', 0)}",
+        ])
+
+    # Latency
+    lat = card.get("inference_latency")
+    if lat:
+        lines.extend([
+            "",
+            "## Inference Latency",
+            "",
+            f"- **Exported p50:** {lat.get('exported_p50_ms', 0):.2f} ms",
+            f"- **Exported p95:** {lat.get('exported_p95_ms', 0):.2f} ms",
+        ])
+        if lat.get("original_p50_ms"):
+            lines.append(f"- **Original p50:** {lat['original_p50_ms']:.2f} ms")
+        if lat.get("speedup"):
+            lines.append(f"- **Speedup:** {lat['speedup']:.1f}x")
+
+    lines.extend([
+        "",
+        f"*Exported: {card.get('export_date', 'unknown')}*",
+    ])
+
+    return "\n".join(lines)

package/templates/scripts/export_formats.py

@@ -0,0 +1,385 @@
+#!/usr/bin/env python3
+"""Format-specific model export handlers.
+
+Each handler knows how to export a specific model type to a production
+format. Returns the export path and metadata.
+
+Supported formats:
+  - joblib: scikit-learn, XGBoost, LightGBM (default)
+  - xgboost_json: XGBoost native JSON format
+  - lightgbm_text: LightGBM native text format
+  - onnx: ONNX via framework-specific converters
+  - torchscript: PyTorch JIT trace
+  - tflite: TensorFlow Lite
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+
+# Registry of model types -> supported export formats
+FORMAT_REGISTRY = {
+    "xgboost": ["joblib", "xgboost_json", "onnx"],
+    "lightgbm": ["joblib", "lightgbm_text", "onnx"],
+    "random_forest": ["joblib", "onnx"],
+    "gradient_boosting": ["joblib", "onnx"],
+    "logistic_regression": ["joblib", "onnx"],
+    "svm": ["joblib", "onnx"],
+    "mlp": ["joblib", "onnx"],
+    "pytorch": ["torchscript", "onnx"],
+    "tensorflow": ["tflite", "onnx"],
+    "keras": ["tflite", "onnx"],
+    "catboost": ["joblib", "onnx"],
+}
+
+# Default format for each model type
+DEFAULT_FORMAT = {
+    "xgboost": "joblib",
+    "lightgbm": "joblib",
+    "random_forest": "joblib",
+    "gradient_boosting": "joblib",
+    "logistic_regression": "joblib",
+    "svm": "joblib",
+    "mlp": "joblib",
+    "pytorch": "torchscript",
+    "tensorflow": "tflite",
+    "keras": "tflite",
+    "catboost": "joblib",
+}
+
+# File extensions for each format
+FORMAT_EXTENSIONS = {
+    "joblib": ".joblib",
+    "xgboost_json": ".json",
+    "lightgbm_text": ".txt",
+    "onnx": ".onnx",
+    "torchscript": ".pt",
+    "tflite": ".tflite",
+}
+
+# Dependencies required for each format
+FORMAT_DEPENDENCIES = {
+    "joblib": ["joblib"],
+    "xgboost_json": ["xgboost>=1.7"],
+    "lightgbm_text": ["lightgbm>=3.0"],
+    "onnx": ["onnx", "onnxruntime"],
+    "torchscript": ["torch>=1.9"],
+    "tflite": ["tensorflow>=2.0"],
+}
+
+
+def get_supported_formats(model_type: str) -> list[str]:
+    """Get supported export formats for a model type."""
+    return FORMAT_REGISTRY.get(model_type, ["joblib"])
+
+
+def get_default_format(model_type: str) -> str:
+    """Get the default export format for a model type."""
+    return DEFAULT_FORMAT.get(model_type, "joblib")
+
+
+def detect_model_type(config: dict) -> str:
+    """Detect model type from experiment config."""
+    model_type = config.get("model", {}).get("type", "")
+    if not model_type:
+        model_type = config.get("model_type", "unknown")
+    return model_type.lower().replace("-", "_").replace(" ", "_")
+
+
+def export_joblib(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export model as joblib bundle (copy if already joblib, else convert).
+
+    Returns dict with path, size_bytes, format, and dependencies.
+    """
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    dst = out_path / f"{model_name}.joblib"
+    shutil.copy2(str(src), str(dst))
+
+    return {
+        "path": str(dst),
+        "format": "joblib",
+        "size_bytes": dst.stat().st_size,
+        "size_mb": round(dst.stat().st_size / 1024**2, 2),
+        "dependencies": FORMAT_DEPENDENCIES["joblib"],
+    }
+
+
+def export_xgboost_json(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export XGBoost model to native JSON format."""
+    try:
+        import joblib
+        import xgboost as xgb
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    try:
+        model = joblib.load(str(src))
+        # Handle wrapped models (e.g., in a pipeline)
+        if hasattr(model, "get_booster"):
+            booster = model.get_booster()
+        elif isinstance(model, xgb.Booster):
+            booster = model
+        else:
+            return {"error": "Model is not an XGBoost model or doesn't have get_booster()"}
+
+        dst = out_path / f"{model_name}.json"
+        booster.save_model(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "xgboost_json",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["xgboost_json"],
+        }
+    except Exception as e:
+        return {"error": f"XGBoost JSON export failed: {e}"}
+
+
+def export_lightgbm_text(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export LightGBM model to native text format."""
+    try:
+        import joblib
+        import lightgbm as lgb
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    try:
+        model = joblib.load(str(src))
+        if hasattr(model, "booster_"):
+            booster = model.booster_
+        elif isinstance(model, lgb.Booster):
+            booster = model
+        else:
+            return {"error": "Model is not a LightGBM model"}
+
+        dst = out_path / f"{model_name}.txt"
+        booster.save_model(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "lightgbm_text",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["lightgbm_text"],
+        }
+    except Exception as e:
+        return {"error": f"LightGBM text export failed: {e}"}
+
+
+def export_onnx(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+    model_type: str,
+) -> dict:
+    """Export model to ONNX format."""
+    try:
+        import joblib
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.onnx"
+
+    try:
+        model = joblib.load(str(src))
+
+        # Try sklearn-onnx for scikit-learn compatible models
+        try:
+            from skl2onnx import convert_sklearn
+            from skl2onnx.common.data_types import FloatTensorType
+            import numpy as np
+
+            # Infer input shape from model if possible
+            n_features = getattr(model, "n_features_in_", 10)
+            initial_type = [("float_input", FloatTensorType([None, n_features]))]
+            onx = convert_sklearn(model, initial_types=initial_type)
+
+            with open(dst, "wb") as f:
+                f.write(onx.SerializeToString())
+
+            return {
+                "path": str(dst),
+                "format": "onnx",
+                "size_bytes": dst.stat().st_size,
+                "size_mb": round(dst.stat().st_size / 1024**2, 2),
+                "dependencies": FORMAT_DEPENDENCIES["onnx"] + ["skl2onnx"],
+            }
+        except ImportError:
+            return {"error": "ONNX export requires skl2onnx: pip install skl2onnx"}
+        except Exception as e:
+            return {"error": f"ONNX conversion failed: {e}"}
+
+    except Exception as e:
+        return {"error": f"ONNX export failed: {e}"}
+
+
+def export_torchscript(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export PyTorch model to TorchScript."""
+    try:
+        import torch
+    except ImportError:
+        return {"error": "TorchScript export requires PyTorch: pip install torch"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.pt"
+
+    try:
+        model = torch.load(str(src), map_location="cpu")
+        if hasattr(model, "eval"):
+            model.eval()
+
+        # Try tracing with dummy input
+        if hasattr(model, "example_input"):
+            dummy = model.example_input
+        else:
+            # Create a dummy input — user may need to customize
+            dummy = torch.randn(1, 10)
+
+        scripted = torch.jit.trace(model, dummy)
+        scripted.save(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "torchscript",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["torchscript"],
+        }
+    except Exception as e:
+        return {"error": f"TorchScript export failed: {e}"}
+
+
+def export_tflite(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export TensorFlow/Keras model to TFLite."""
+    try:
+        import tensorflow as tf
+    except ImportError:
+        return {"error": "TFLite export requires TensorFlow: pip install tensorflow"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.tflite"
+
+    try:
+        model = tf.keras.models.load_model(str(src))
+        converter = tf.lite.TFLiteConverter.from_keras_model(model)
+        tflite_model = converter.convert()
+
+        with open(dst, "wb") as f:
+            f.write(tflite_model)
+
+        return {
+            "path": str(dst),
+            "format": "tflite",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["tflite"],
+        }
+    except Exception as e:
+        return {"error": f"TFLite export failed: {e}"}
+
+
+def export_model(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+    model_type: str,
+    export_format: str | None = None,
+) -> dict:
+    """Export a model to the specified format.
+
+    Auto-selects format if not specified. Dispatches to format-specific handler.
+
+    Args:
+        model_path: Path to the original model file.
+        output_dir: Directory to write exported model.
+        model_name: Base name for the exported file.
+        model_type: Model type (e.g., "xgboost", "pytorch").
+        export_format: Target format (e.g., "joblib", "onnx"). Auto-detected if None.
+
+    Returns:
+        Export result dict with path, format, size, dependencies.
+    """
+    if not export_format:
+        export_format = get_default_format(model_type)
+
+    supported = get_supported_formats(model_type)
+    if export_format not in supported:
+        return {
+            "error": f"Format '{export_format}' not supported for model type '{model_type}'. "
+                     f"Supported: {supported}",
+        }
+
+    handlers = {
+        "joblib": lambda: export_joblib(model_path, output_dir, model_name),
+        "xgboost_json": lambda: export_xgboost_json(model_path, output_dir, model_name),
+        "lightgbm_text": lambda: export_lightgbm_text(model_path, output_dir, model_name),
+        "onnx": lambda: export_onnx(model_path, output_dir, model_name, model_type),
+        "torchscript": lambda: export_torchscript(model_path, output_dir, model_name),
+        "tflite": lambda: export_tflite(model_path, output_dir, model_name),
+    }
+
+    handler = handlers.get(export_format)
+    if not handler:
+        return {"error": f"No handler for format '{export_format}'"}
+
+    return handler()