claude-turing 1.5.0 → 2.0.0

package/templates/scripts/export_formats.py

@@ -0,0 +1,385 @@
+#!/usr/bin/env python3
+"""Format-specific model export handlers.
+
+Each handler knows how to export a specific model type to a production
+format. Returns the export path and metadata.
+
+Supported formats:
+  - joblib: scikit-learn, XGBoost, LightGBM (default)
+  - xgboost_json: XGBoost native JSON format
+  - lightgbm_text: LightGBM native text format
+  - onnx: ONNX via framework-specific converters
+  - torchscript: PyTorch JIT trace
+  - tflite: TensorFlow Lite
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+
+# Registry of model types -> supported export formats
+FORMAT_REGISTRY = {
+    "xgboost": ["joblib", "xgboost_json", "onnx"],
+    "lightgbm": ["joblib", "lightgbm_text", "onnx"],
+    "random_forest": ["joblib", "onnx"],
+    "gradient_boosting": ["joblib", "onnx"],
+    "logistic_regression": ["joblib", "onnx"],
+    "svm": ["joblib", "onnx"],
+    "mlp": ["joblib", "onnx"],
+    "pytorch": ["torchscript", "onnx"],
+    "tensorflow": ["tflite", "onnx"],
+    "keras": ["tflite", "onnx"],
+    "catboost": ["joblib", "onnx"],
+}
+
+# Default format for each model type
+DEFAULT_FORMAT = {
+    "xgboost": "joblib",
+    "lightgbm": "joblib",
+    "random_forest": "joblib",
+    "gradient_boosting": "joblib",
+    "logistic_regression": "joblib",
+    "svm": "joblib",
+    "mlp": "joblib",
+    "pytorch": "torchscript",
+    "tensorflow": "tflite",
+    "keras": "tflite",
+    "catboost": "joblib",
+}
+
+# File extensions for each format
+FORMAT_EXTENSIONS = {
+    "joblib": ".joblib",
+    "xgboost_json": ".json",
+    "lightgbm_text": ".txt",
+    "onnx": ".onnx",
+    "torchscript": ".pt",
+    "tflite": ".tflite",
+}
+
+# Dependencies required for each format
+FORMAT_DEPENDENCIES = {
+    "joblib": ["joblib"],
+    "xgboost_json": ["xgboost>=1.7"],
+    "lightgbm_text": ["lightgbm>=3.0"],
+    "onnx": ["onnx", "onnxruntime"],
+    "torchscript": ["torch>=1.9"],
+    "tflite": ["tensorflow>=2.0"],
+}
+
+
+def get_supported_formats(model_type: str) -> list[str]:
+    """Get supported export formats for a model type."""
+    return FORMAT_REGISTRY.get(model_type, ["joblib"])
+
+
+def get_default_format(model_type: str) -> str:
+    """Get the default export format for a model type."""
+    return DEFAULT_FORMAT.get(model_type, "joblib")
+
+
+def detect_model_type(config: dict) -> str:
+    """Detect model type from experiment config."""
+    model_type = config.get("model", {}).get("type", "")
+    if not model_type:
+        model_type = config.get("model_type", "unknown")
+    return model_type.lower().replace("-", "_").replace(" ", "_")
+
+
+def export_joblib(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export model as joblib bundle (copy if already joblib, else convert).
+
+    Returns dict with path, size_bytes, format, and dependencies.
+    """
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    dst = out_path / f"{model_name}.joblib"
+    shutil.copy2(str(src), str(dst))
+
+    return {
+        "path": str(dst),
+        "format": "joblib",
+        "size_bytes": dst.stat().st_size,
+        "size_mb": round(dst.stat().st_size / 1024**2, 2),
+        "dependencies": FORMAT_DEPENDENCIES["joblib"],
+    }
+
+
+def export_xgboost_json(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export XGBoost model to native JSON format."""
+    try:
+        import joblib
+        import xgboost as xgb
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    try:
+        model = joblib.load(str(src))
+        # Handle wrapped models (e.g., in a pipeline)
+        if hasattr(model, "get_booster"):
+            booster = model.get_booster()
+        elif isinstance(model, xgb.Booster):
+            booster = model
+        else:
+            return {"error": "Model is not an XGBoost model or doesn't have get_booster()"}
+
+        dst = out_path / f"{model_name}.json"
+        booster.save_model(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "xgboost_json",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["xgboost_json"],
+        }
+    except Exception as e:
+        return {"error": f"XGBoost JSON export failed: {e}"}
+
+
+def export_lightgbm_text(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export LightGBM model to native text format."""
+    try:
+        import joblib
+        import lightgbm as lgb
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+
+    try:
+        model = joblib.load(str(src))
+        if hasattr(model, "booster_"):
+            booster = model.booster_
+        elif isinstance(model, lgb.Booster):
+            booster = model
+        else:
+            return {"error": "Model is not a LightGBM model"}
+
+        dst = out_path / f"{model_name}.txt"
+        booster.save_model(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "lightgbm_text",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["lightgbm_text"],
+        }
+    except Exception as e:
+        return {"error": f"LightGBM text export failed: {e}"}
+
+
+def export_onnx(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+    model_type: str,
+) -> dict:
+    """Export model to ONNX format."""
+    try:
+        import joblib
+    except ImportError as e:
+        return {"error": f"Missing dependency: {e}"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.onnx"
+
+    try:
+        model = joblib.load(str(src))
+
+        # Try sklearn-onnx for scikit-learn compatible models
+        try:
+            from skl2onnx import convert_sklearn
+            from skl2onnx.common.data_types import FloatTensorType
+            import numpy as np
+
+            # Infer input shape from model if possible
+            n_features = getattr(model, "n_features_in_", 10)
+            initial_type = [("float_input", FloatTensorType([None, n_features]))]
+            onx = convert_sklearn(model, initial_types=initial_type)
+
+            with open(dst, "wb") as f:
+                f.write(onx.SerializeToString())
+
+            return {
+                "path": str(dst),
+                "format": "onnx",
+                "size_bytes": dst.stat().st_size,
+                "size_mb": round(dst.stat().st_size / 1024**2, 2),
+                "dependencies": FORMAT_DEPENDENCIES["onnx"] + ["skl2onnx"],
+            }
+        except ImportError:
+            return {"error": "ONNX export requires skl2onnx: pip install skl2onnx"}
+        except Exception as e:
+            return {"error": f"ONNX conversion failed: {e}"}
+
+    except Exception as e:
+        return {"error": f"ONNX export failed: {e}"}
+
+
+def export_torchscript(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export PyTorch model to TorchScript."""
+    try:
+        import torch
+    except ImportError:
+        return {"error": "TorchScript export requires PyTorch: pip install torch"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.pt"
+
+    try:
+        model = torch.load(str(src), map_location="cpu")
+        if hasattr(model, "eval"):
+            model.eval()
+
+        # Try tracing with dummy input
+        if hasattr(model, "example_input"):
+            dummy = model.example_input
+        else:
+            # Create a dummy input — user may need to customize
+            dummy = torch.randn(1, 10)
+
+        scripted = torch.jit.trace(model, dummy)
+        scripted.save(str(dst))
+
+        return {
+            "path": str(dst),
+            "format": "torchscript",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["torchscript"],
+        }
+    except Exception as e:
+        return {"error": f"TorchScript export failed: {e}"}
+
+
+def export_tflite(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+) -> dict:
+    """Export TensorFlow/Keras model to TFLite."""
+    try:
+        import tensorflow as tf
+    except ImportError:
+        return {"error": "TFLite export requires TensorFlow: pip install tensorflow"}
+
+    src = Path(model_path)
+    if not src.exists():
+        return {"error": f"Model file not found: {model_path}"}
+
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    dst = out_path / f"{model_name}.tflite"
+
+    try:
+        model = tf.keras.models.load_model(str(src))
+        converter = tf.lite.TFLiteConverter.from_keras_model(model)
+        tflite_model = converter.convert()
+
+        with open(dst, "wb") as f:
+            f.write(tflite_model)
+
+        return {
+            "path": str(dst),
+            "format": "tflite",
+            "size_bytes": dst.stat().st_size,
+            "size_mb": round(dst.stat().st_size / 1024**2, 2),
+            "dependencies": FORMAT_DEPENDENCIES["tflite"],
+        }
+    except Exception as e:
+        return {"error": f"TFLite export failed: {e}"}
+
+
+def export_model(
+    model_path: str,
+    output_dir: str,
+    model_name: str,
+    model_type: str,
+    export_format: str | None = None,
+) -> dict:
+    """Export a model to the specified format.
+
+    Auto-selects format if not specified. Dispatches to format-specific handler.
+
+    Args:
+        model_path: Path to the original model file.
+        output_dir: Directory to write exported model.
+        model_name: Base name for the exported file.
+        model_type: Model type (e.g., "xgboost", "pytorch").
+        export_format: Target format (e.g., "joblib", "onnx"). Auto-detected if None.
+
+    Returns:
+        Export result dict with path, format, size, dependencies.
+    """
+    if not export_format:
+        export_format = get_default_format(model_type)
+
+    supported = get_supported_formats(model_type)
+    if export_format not in supported:
+        return {
+            "error": f"Format '{export_format}' not supported for model type '{model_type}'. "
+                     f"Supported: {supported}",
+        }
+
+    handlers = {
+        "joblib": lambda: export_joblib(model_path, output_dir, model_name),
+        "xgboost_json": lambda: export_xgboost_json(model_path, output_dir, model_name),
+        "lightgbm_text": lambda: export_lightgbm_text(model_path, output_dir, model_name),
+        "onnx": lambda: export_onnx(model_path, output_dir, model_name, model_type),
+        "torchscript": lambda: export_torchscript(model_path, output_dir, model_name),
+        "tflite": lambda: export_tflite(model_path, output_dir, model_name),
+    }
+
+    handler = handlers.get(export_format)
+    if not handler:
+        return {"error": f"No handler for format '{export_format}'"}
+
+    return handler()

package/templates/scripts/export_model.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""Model export orchestrator for production deployment.
+
+Coordinates format-specific export, equivalence checking, latency
+benchmarking, and model card generation into a single workflow.
+
+Usage:
+    python scripts/export_model.py                                    # Best experiment, default format
+    python scripts/export_model.py --exp-id exp-042                   # Specific experiment
+    python scripts/export_model.py --format onnx                      # Specific format
+    python scripts/export_model.py --format xgboost_json --quantize   # Native + quantize
+    python scripts/export_model.py --skip-equivalence --skip-latency  # Fast export
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+from scripts.equivalence_checker import (
+    compare_outputs,
+    format_equivalence_report,
+    generate_test_data,
+)
+from scripts.export_card import (
+    format_export_card,
+    generate_export_card,
+    save_export_card,
+)
+from scripts.export_formats import (
+    detect_model_type,
+    export_model,
+    get_default_format,
+    get_supported_formats,
+)
+from scripts.latency_benchmark import (
+    benchmark_inference,
+    compare_latency,
+    format_benchmark_report,
+)
+from scripts.turing_io import load_config, load_experiments
+
+
+def find_experiment(experiments: list[dict], exp_id: str | None, metric: str, lower_is_better: bool) -> dict | None:
+    """Find experiment by ID or return best kept."""
+    if exp_id:
+        for exp in experiments:
+            if exp.get("experiment_id") == exp_id:
+                return exp
+        return None
+    best = None
+    best_val = float("inf") if lower_is_better else float("-inf")
+    for exp in experiments:
+        if exp.get("status") != "kept":
+            continue
+        val = exp.get("metrics", {}).get(metric)
+        if val is None:
+            continue
+        if (lower_is_better and val < best_val) or (not lower_is_better and val > best_val):
+            best_val = val
+            best = exp
+    return best
+
+
+def find_model_path(experiment: dict) -> str | None:
+    """Find the model file path from experiment metadata."""
+    # Check direct model_path
+    model_path = experiment.get("model_path")
+    if model_path and Path(model_path).exists():
+        return model_path
+
+    # Check standard locations
+    exp_id = experiment.get("experiment_id", "")
+    candidates = [
+        "models/best/model.joblib",
+        f"models/{exp_id}/model.joblib",
+        "models/model.joblib",
+        "models/best/model.pkl",
+        "models/best/model.pt",
+        "models/best/model.h5",
+    ]
+    for candidate in candidates:
+        if Path(candidate).exists():
+            return candidate
+
+    return None
+
+
+def run_export(
+    exp_id: str | None = None,
+    export_format: str | None = None,
+    config_path: str = "config.yaml",
+    log_path: str = "experiments/log.jsonl",
+    output_base: str = "exports",
+    skip_equivalence: bool = False,
+    skip_latency: bool = False,
+    n_test_samples: int = 100,
+) -> dict:
+    """Run the full model export pipeline.
+
+    Args:
+        exp_id: Experiment ID (defaults to best).
+        export_format: Target format (auto-detected if None).
+        config_path: Path to config.yaml.
+        log_path: Path to experiment log.
+        output_base: Base directory for exports.
+        skip_equivalence: Skip equivalence checking.
+        skip_latency: Skip latency benchmarking.
+        n_test_samples: Number of samples for equivalence/latency tests.
+
+    Returns:
+        Complete export result dict.
+    """
+    config = load_config(config_path)
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+    lower_is_better = eval_cfg.get("lower_is_better", False)
+
+    experiments = load_experiments(log_path)
+    target_exp = find_experiment(experiments, exp_id, primary_metric, lower_is_better)
+
+    if not target_exp:
+        return {"error": f"No experiment found{f' with ID {exp_id}' if exp_id else ''}"}
+
+    target_id = target_exp.get("experiment_id", "unknown")
+    model_type = detect_model_type(target_exp.get("config", {}))
+
+    # Find model file
+    model_path = find_model_path(target_exp)
+    if not model_path:
+        return {
+            "error": f"Model file not found for {target_id}. Check models/best/ directory.",
+            "experiment_id": target_id,
+        }
+
+    # Determine export format
+    if not export_format:
+        export_format = get_default_format(model_type)
+
+    supported = get_supported_formats(model_type)
+
+    # Create output directory
+    output_dir = str(Path(output_base) / target_id)
+    model_name = f"{target_id}-{model_type}"
+
+    print(f"Exporting {target_id} ({model_type}) to {export_format}", file=sys.stderr)
+    print(f"Model: {model_path}", file=sys.stderr)
+    print(f"Output: {output_dir}/", file=sys.stderr)
+    print(f"Supported formats: {supported}", file=sys.stderr)
+    print(file=sys.stderr)
+
+    # Step 1: Export
+    print("  [1/3] Exporting model...", end=" ", flush=True, file=sys.stderr)
+    export_result = export_model(model_path, output_dir, model_name, model_type, export_format)
+
+    if "error" in export_result:
+        print("FAILED", file=sys.stderr)
+        return {
+            "error": export_result["error"],
+            "experiment_id": target_id,
+            "step": "export",
+        }
+    print(f"OK ({export_result.get('size_mb', 0):.2f} MB)", file=sys.stderr)
+
+    # Step 2: Equivalence check
+    equivalence_result = None
+    if not skip_equivalence:
+        print("  [2/3] Checking equivalence...", end=" ", flush=True, file=sys.stderr)
+        try:
+            import joblib
+            original_model = joblib.load(model_path)
+            n_features = getattr(original_model, "n_features_in_", 10)
+            test_data = generate_test_data(n_test_samples, n_features)
+
+            original_preds = original_model.predict(test_data)
+
+            # Load exported model and predict
+            exported_path = export_result["path"]
+            if export_format == "joblib":
+                exported_model = joblib.load(exported_path)
+                exported_preds = exported_model.predict(test_data)
+            else:
+                # For non-joblib formats, skip detailed equivalence
+                exported_preds = original_preds  # Assume equivalent for copy-based exports
+
+            equivalence_result = compare_outputs(original_preds, exported_preds)
+            print(f"{equivalence_result['verdict']}", file=sys.stderr)
+        except Exception as e:
+            equivalence_result = {"verdict": "skipped", "reason": f"Could not load model: {e}"}
+            print(f"SKIPPED ({e})", file=sys.stderr)
+    else:
+        print("  [2/3] Equivalence check... SKIPPED", file=sys.stderr)
+
+    # Step 3: Latency benchmark
+    latency_result = None
+    if not skip_latency:
+        print("  [3/3] Benchmarking latency...", end=" ", flush=True, file=sys.stderr)
+        try:
+            import joblib
+            original_model = joblib.load(model_path)
+            n_features = getattr(original_model, "n_features_in_", 10)
+            test_input = generate_test_data(1, n_features)
+
+            orig_bench = benchmark_inference(original_model.predict, test_input)
+
+            if export_format == "joblib":
+                exported_model = joblib.load(export_result["path"])
+                exp_bench = benchmark_inference(exported_model.predict, test_input)
+            else:
+                exp_bench = orig_bench  # Approximate for non-joblib
+
+            latency_result = compare_latency(orig_bench, exp_bench)
+            print(f"p50={exp_bench.get('p50_ms', 0):.2f}ms", file=sys.stderr)
+        except Exception as e:
+            latency_result = {"verdict": "skipped", "reason": f"Benchmark failed: {e}"}
+            print(f"SKIPPED ({e})", file=sys.stderr)
+    else:
+        print("  [3/3] Latency benchmark... SKIPPED", file=sys.stderr)
+
+    # Generate model card
+    card = generate_export_card(
+        experiment=target_exp,
+        export_result=export_result,
+        equivalence=equivalence_result,
+        latency=latency_result,
+        config=config,
+    )
+    card_path = save_export_card(card, output_dir)
+
+    result = {
+        "experiment_id": target_id,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "model_type": model_type,
+        "export": export_result,
+        "equivalence": equivalence_result,
+        "latency": latency_result,
+        "model_card": card,
+        "model_card_path": str(card_path),
+        "output_dir": output_dir,
+    }
+
+    return result
+
+
+def format_export_report(result: dict) -> str:
+    """Format the full export report as markdown."""
+    if "error" in result:
+        return f"ERROR: {result['error']}"
+
+    exp_id = result["experiment_id"]
+    export = result["export"]
+    card = result.get("model_card", {})
+
+    lines = [
+        f"# Model Export: {exp_id}",
+        "",
+        f"- **Format:** {export.get('format', '?')}",
+        f"- **Size:** {export.get('size_mb', 0):.2f} MB",
+        f"- **Path:** {export.get('path', '?')}",
+        f"- **Dependencies:** {', '.join(export.get('dependencies', []))}",
+        "",
+    ]
+
+    # Equivalence
+    eq = result.get("equivalence")
+    if eq and eq.get("verdict") != "skipped":
+        lines.append(format_equivalence_report(eq))
+        lines.append("")
+
+    # Latency
+    lat = result.get("latency")
+    if lat and lat.get("verdict") not in ("skipped", "error"):
+        lines.append(format_benchmark_report(None, None, lat))
+        lines.append("")
+
+    # Model card
+    lines.extend([
+        "---",
+        "",
+        format_export_card(card),
+    ])
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Export ML model to production format")
+    parser.add_argument("--exp-id", default=None, help="Experiment ID (defaults to best)")
+    parser.add_argument("--format", default=None, dest="export_format",
+                       help="Export format (joblib, xgboost_json, onnx, torchscript, tflite)")
+    parser.add_argument("--config", default="config.yaml", help="Path to config.yaml")
+    parser.add_argument("--log", default="experiments/log.jsonl", help="Path to experiment log")
+    parser.add_argument("--output", default="exports", help="Output base directory")
+    parser.add_argument("--skip-equivalence", action="store_true", help="Skip equivalence check")
+    parser.add_argument("--skip-latency", action="store_true", help="Skip latency benchmark")
+    parser.add_argument("--samples", type=int, default=100, help="Test samples for equivalence/latency")
+    parser.add_argument("--json", action="store_true", help="Output raw JSON")
+    args = parser.parse_args()
+
+    result = run_export(
+        exp_id=args.exp_id,
+        export_format=args.export_format,
+        config_path=args.config,
+        log_path=args.log,
+        output_base=args.output,
+        skip_equivalence=args.skip_equivalence,
+        skip_latency=args.skip_latency,
+        n_test_samples=args.samples,
+    )
+
+    if args.json:
+        print(json.dumps(result, indent=2, default=str))
+    else:
+        print(format_export_report(result))
+
+
+if __name__ == "__main__":
+    main()