claude-turing 3.1.0 → 3.3.0

package/templates/scripts/calibration.py

@@ -0,0 +1,364 @@
+#!/usr/bin/env python3
+"""Probability calibration for the autoresearch pipeline.
+
+Measures whether model probabilities are well-calibrated, computes ECE/MCE,
+generates reliability diagrams, and applies post-hoc calibration (Platt
+scaling, isotonic regression, temperature scaling).
+
+Usage:
+    python scripts/calibration.py exp-042
+    python scripts/calibration.py exp-042 --method platt
+    python scripts/calibration.py exp-042 --method auto
+    python scripts/calibration.py --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+import yaml
+
+from scripts.turing_io import load_config, load_experiments
+
+DEFAULT_LOG_PATH = "experiments/log.jsonl"
+DEFAULT_N_BINS = 10
+CALIBRATION_METHODS = ["platt", "isotonic", "temperature"]
+
+
+# --- Calibration Metrics ---
+
+
+def compute_ece(
+    probabilities: np.ndarray,
+    labels: np.ndarray,
+    n_bins: int = DEFAULT_N_BINS,
+) -> float:
+    """Compute Expected Calibration Error.
+
+    ECE = sum(|bin_accuracy - bin_confidence| * bin_size / total)
+    """
+    if len(probabilities) == 0:
+        return 0.0
+
+    bin_boundaries = np.linspace(0, 1, n_bins + 1)
+    ece = 0.0
+
+    for i in range(n_bins):
+        mask = (probabilities >= bin_boundaries[i]) & (probabilities < bin_boundaries[i + 1])
+        if i == n_bins - 1:
+            mask = (probabilities >= bin_boundaries[i]) & (probabilities <= bin_boundaries[i + 1])
+
+        bin_size = np.sum(mask)
+        if bin_size == 0:
+            continue
+
+        bin_accuracy = np.mean(labels[mask])
+        bin_confidence = np.mean(probabilities[mask])
+        ece += abs(bin_accuracy - bin_confidence) * bin_size / len(probabilities)
+
+    return round(float(ece), 6)
+
+
+def compute_mce(
+    probabilities: np.ndarray,
+    labels: np.ndarray,
+    n_bins: int = DEFAULT_N_BINS,
+) -> float:
+    """Compute Maximum Calibration Error."""
+    if len(probabilities) == 0:
+        return 0.0
+
+    bin_boundaries = np.linspace(0, 1, n_bins + 1)
+    max_gap = 0.0
+
+    for i in range(n_bins):
+        mask = (probabilities >= bin_boundaries[i]) & (probabilities < bin_boundaries[i + 1])
+        if i == n_bins - 1:
+            mask = (probabilities >= bin_boundaries[i]) & (probabilities <= bin_boundaries[i + 1])
+
+        if np.sum(mask) == 0:
+            continue
+
+        bin_accuracy = np.mean(labels[mask])
+        bin_confidence = np.mean(probabilities[mask])
+        max_gap = max(max_gap, abs(bin_accuracy - bin_confidence))
+
+    return round(float(max_gap), 6)
+
+
+def compute_reliability_diagram(
+    probabilities: np.ndarray,
+    labels: np.ndarray,
+    n_bins: int = DEFAULT_N_BINS,
+) -> list[dict]:
+    """Compute reliability diagram data."""
+    if len(probabilities) == 0:
+        return []
+
+    bin_boundaries = np.linspace(0, 1, n_bins + 1)
+    bins = []
+
+    for i in range(n_bins):
+        lo = bin_boundaries[i]
+        hi = bin_boundaries[i + 1]
+        mask = (probabilities >= lo) & (probabilities < hi)
+        if i == n_bins - 1:
+            mask = (probabilities >= lo) & (probabilities <= hi)
+
+        bin_size = int(np.sum(mask))
+        if bin_size == 0:
+            bins.append({"bin": f"[{lo:.1f}-{hi:.1f}]", "predicted": None,
+                        "actual": None, "gap": None, "n": 0})
+            continue
+
+        predicted = float(np.mean(probabilities[mask]))
+        actual = float(np.mean(labels[mask]))
+        gap = actual - predicted
+
+        bins.append({
+            "bin": f"[{lo:.1f}-{hi:.1f}]",
+            "predicted": round(predicted, 4),
+            "actual": round(actual, 4),
+            "gap": round(gap, 4),
+            "n": bin_size,
+        })
+
+    return bins
+
+
+# --- Calibration Methods ---
+
+
+def platt_scaling(
+    logits: np.ndarray,
+    labels: np.ndarray,
+) -> dict:
+    """Apply Platt scaling (logistic regression on logits)."""
+    from scipy.special import expit
+
+    # Fit logistic regression: P(y=1|f) = sigmoid(a*f + b)
+    # Simple gradient descent for a, b
+    a, b = 1.0, 0.0
+    lr = 0.01
+    for _ in range(1000):
+        pred = expit(a * logits + b)
+        pred = np.clip(pred, 1e-7, 1 - 1e-7)
+        grad_a = np.mean((pred - labels) * logits)
+        grad_b = np.mean(pred - labels)
+        a -= lr * grad_a
+        b -= lr * grad_b
+
+    calibrated = expit(a * logits + b)
+    return {"method": "platt", "params": {"a": round(float(a), 6), "b": round(float(b), 6)},
+            "calibrated_probabilities": calibrated}
+
+
+def isotonic_calibration(
+    probabilities: np.ndarray,
+    labels: np.ndarray,
+) -> dict:
+    """Apply isotonic regression calibration."""
+    from sklearn.isotonic import IsotonicRegression
+
+    iso = IsotonicRegression(out_of_bounds="clip")
+    calibrated = iso.fit_transform(probabilities, labels)
+    return {"method": "isotonic", "params": {},
+            "calibrated_probabilities": np.clip(calibrated, 0, 1)}
+
+
+def temperature_scaling(
+    logits: np.ndarray,
+    labels: np.ndarray,
+) -> dict:
+    """Apply temperature scaling (single parameter T)."""
+    from scipy.special import expit
+
+    best_t = 1.0
+    best_ece = float("inf")
+
+    for t in np.arange(0.5, 5.0, 0.1):
+        scaled = expit(logits / t)
+        ece = compute_ece(scaled, labels)
+        if ece < best_ece:
+            best_ece = ece
+            best_t = t
+
+    calibrated = expit(logits / best_t)
+    return {"method": "temperature", "params": {"T": round(float(best_t), 2)},
+            "calibrated_probabilities": calibrated}
+
+
+# --- Full Pipeline ---
+
+
+def calibrate_model(
+    probabilities: np.ndarray | None = None,
+    logits: np.ndarray | None = None,
+    labels: np.ndarray | None = None,
+    method: str = "auto",
+    exp_id: str | None = None,
+    config_path: str = "config.yaml",
+) -> dict:
+    """Run calibration analysis and optionally apply post-hoc calibration."""
+    if (probabilities is None and logits is None) or labels is None:
+        return {"error": "Provide probabilities (or logits) and labels for calibration"}
+
+    if probabilities is None and logits is not None:
+        from scipy.special import expit
+        probabilities = expit(logits)
+
+    # Before calibration
+    ece_before = compute_ece(probabilities, labels)
+    mce_before = compute_mce(probabilities, labels)
+    reliability = compute_reliability_diagram(probabilities, labels)
+
+    # Determine overconfidence
+    overconfident_bins = [b for b in reliability if b.get("gap") is not None and b["gap"] < -0.05 and b["n"] > 0]
+
+    report = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "experiment_id": exp_id,
+        "before": {"ece": ece_before, "mce": mce_before},
+        "reliability_diagram": reliability,
+        "overconfident_bins": len(overconfident_bins),
+    }
+
+    # Apply calibration
+    methods_to_try = CALIBRATION_METHODS if method == "auto" else [method]
+    results = []
+
+    for m in methods_to_try:
+        try:
+            if m == "platt" and logits is not None:
+                cal = platt_scaling(logits, labels)
+            elif m == "isotonic":
+                cal = isotonic_calibration(probabilities, labels)
+            elif m == "temperature" and logits is not None:
+                cal = temperature_scaling(logits, labels)
+            else:
+                continue
+
+            ece_after = compute_ece(cal["calibrated_probabilities"], labels)
+            results.append({
+                "method": m,
+                "ece_after": ece_after,
+                "improvement": round(ece_before - ece_after, 6),
+                "params": cal.get("params", {}),
+            })
+        except Exception:
+            continue
+
+    # Find best method
+    best = None
+    if results:
+        best = min(results, key=lambda r: r["ece_after"])
+
+    report["calibration_results"] = results
+    report["best_method"] = best
+
+    # Verdict
+    if ece_before < 0.02:
+        report["verdict"] = "already_calibrated"
+        report["reason"] = f"ECE {ece_before:.4f} is already low — calibration not needed"
+    elif best and best["improvement"] > 0.01:
+        report["verdict"] = "improved"
+        report["reason"] = f"{best['method']} reduces ECE from {ece_before:.4f} to {best['ece_after']:.4f}"
+    elif best:
+        report["verdict"] = "marginal_improvement"
+        report["reason"] = f"Best method ({best['method']}) improves ECE by only {best['improvement']:.4f}"
+    else:
+        report["verdict"] = "no_improvement"
+        report["reason"] = "No calibration method improved ECE"
+
+    return report
+
+
+# --- Report Formatting ---
+
+
+def save_calibration_report(report: dict, output_dir: str = "experiments/calibration") -> Path:
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    exp_id = report.get("experiment_id", "unknown")
+    filepath = out_path / f"{exp_id}-calibration.yaml"
+    clean = json.loads(json.dumps(report, default=str))
+    with open(filepath, "w") as f:
+        yaml.dump(clean, f, default_flow_style=False, sort_keys=False)
+    return filepath
+
+
+def format_calibration_report(report: dict) -> str:
+    if "error" in report:
+        return f"ERROR: {report['error']}"
+
+    exp_id = report.get("experiment_id", "?")
+    before = report.get("before", {})
+
+    lines = [f"# Calibration: {exp_id}", "",
+             f"*Generated {report.get('generated_at', 'N/A')[:19]}*", "",
+             f"**ECE before:** {before.get('ece', '?')}",
+             f"**MCE before:** {before.get('mce', '?')}", ""]
+
+    # Reliability diagram
+    diagram = report.get("reliability_diagram", [])
+    if diagram:
+        lines.extend(["## Reliability Diagram", "",
+                      "| Bin | Predicted | Actual | Gap |",
+                      "|-----|-----------|--------|-----|"])
+        for b in diagram:
+            if b["predicted"] is not None:
+                gap_marker = " overconfident" if b["gap"] is not None and b["gap"] < -0.05 else ""
+                lines.append(f"| {b['bin']} | {b['predicted']:.4f} | {b['actual']:.4f} | {b['gap']:+.4f}{gap_marker} |")
+        lines.append("")
+
+    # Calibration results
+    results = report.get("calibration_results", [])
+    if results:
+        lines.extend(["## Calibration Methods", "",
+                      "| Method | ECE After | Improvement |",
+                      "|--------|-----------|-------------|"])
+        best = report.get("best_method", {})
+        for r in results:
+            marker = " BEST" if r["method"] == best.get("method") else ""
+            lines.append(f"| {r['method']} | {r['ece_after']:.4f} | {r['improvement']:+.4f} |{marker}")
+        lines.append("")
+
+    # Verdict
+    verdict = report.get("verdict", "?")
+    labels = {"already_calibrated": "ALREADY CALIBRATED", "improved": "IMPROVED",
+              "marginal_improvement": "MARGINAL IMPROVEMENT", "no_improvement": "NO IMPROVEMENT"}
+    lines.extend(["## Verdict", "", f"**{labels.get(verdict, verdict.upper())}**", "",
+                  report.get("reason", "")])
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Probability calibration")
+    parser.add_argument("exp_id", nargs="?", help="Experiment ID")
+    parser.add_argument("--method", choices=CALIBRATION_METHODS + ["auto"], default="auto")
+    parser.add_argument("--config", default="config.yaml")
+    parser.add_argument("--log", default=DEFAULT_LOG_PATH)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    # Without data, show usage
+    report = calibrate_model(exp_id=args.exp_id, method=args.method, config_path=args.config)
+
+    if "error" not in report:
+        filepath = save_calibration_report(report)
+        print(f"Saved to {filepath}", file=sys.stderr)
+
+    if args.json:
+        print(json.dumps(report, indent=2, default=str))
+    else:
+        print(format_calibration_report(report))
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/curriculum_optimizer.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""Training curriculum optimization for the autoresearch pipeline.
+
+Orders training data by difficulty and measures whether curriculum
+learning improves convergence speed or final performance. Tests
+easy-to-hard, hard-to-easy, self-paced, and random strategies.
+
+Usage:
+    python scripts/curriculum_optimizer.py exp-042
+    python scripts/curriculum_optimizer.py --strategies easy-to-hard,random
+    python scripts/curriculum_optimizer.py --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+import yaml
+
+from scripts.turing_io import load_config
+
+DEFAULT_STRATEGIES = ["random", "easy_to_hard", "hard_to_easy", "self_paced"]
+IMPOSSIBLE_THRESHOLD = 0.9  # Samples with difficulty > this across all strategies
+
+
+# --- Difficulty Scoring ---
+
+
+def score_difficulty_by_loss(
+    losses: np.ndarray,
+) -> np.ndarray:
+    """Score sample difficulty by loss value (higher loss = harder).
+
+    Normalizes to [0, 1].
+    """
+    if len(losses) == 0:
+        return np.array([])
+
+    min_loss = np.min(losses)
+    max_loss = np.max(losses)
+    if max_loss == min_loss:
+        return np.zeros(len(losses))
+
+    return (losses - min_loss) / (max_loss - min_loss)
+
+
+def score_difficulty_by_margin(
+    margins: np.ndarray,
+) -> np.ndarray:
+    """Score sample difficulty by margin (smaller margin = harder).
+
+    Margins = distance from decision boundary. Normalizes to [0, 1].
+    """
+    if len(margins) == 0:
+        return np.array([])
+
+    min_m = np.min(margins)
+    max_m = np.max(margins)
+    if max_m == min_m:
+        return np.full(len(margins), 0.5)
+
+    # Invert: small margin = high difficulty
+    return 1.0 - (margins - min_m) / (max_m - min_m)
+
+
+def score_difficulty_by_disagreement(
+    multi_seed_predictions: list[np.ndarray],
+    labels: np.ndarray,
+) -> np.ndarray:
+    """Score difficulty by prediction disagreement across seeds.
+
+    Samples where different seeds disagree are "hard" (and possibly mislabeled).
+    """
+    if not multi_seed_predictions or len(labels) == 0:
+        return np.array([])
+
+    n_samples = len(labels)
+    n_seeds = len(multi_seed_predictions)
+
+    agreement = np.zeros(n_samples)
+    for preds in multi_seed_predictions:
+        if len(preds) == n_samples:
+            agreement += (preds == labels).astype(float)
+
+    agreement /= n_seeds  # Fraction of seeds that got it right
+
+    # Disagreement = difficulty
+    return 1.0 - agreement
+
+
+# --- Curriculum Strategies ---
+
+
+def apply_curriculum(
+    indices: np.ndarray,
+    difficulties: np.ndarray,
+    strategy: str,
+) -> np.ndarray:
+    """Reorder sample indices according to curriculum strategy.
+
+    Args:
+        indices: Original sample indices.
+        difficulties: Difficulty scores [0, 1] per sample.
+        strategy: Curriculum strategy name.
+
+    Returns:
+        Reordered indices.
+    """
+    if len(indices) == 0:
+        return indices
+
+    if strategy == "random":
+        np.random.shuffle(indices)
+        return indices
+
+    elif strategy == "easy_to_hard":
+        order = np.argsort(difficulties)
+        return indices[order]
+
+    elif strategy == "hard_to_easy":
+        order = np.argsort(difficulties)[::-1]
+        return indices[order]
+
+    elif strategy == "self_paced":
+        # Start with easiest 20%, then gradually include harder
+        order = np.argsort(difficulties)
+        n = len(order)
+        # Shuffle within difficulty bands
+        bands = [order[:n // 5], order[n // 5:2 * n // 5],
+                 order[2 * n // 5:3 * n // 5], order[3 * n // 5:4 * n // 5],
+                 order[4 * n // 5:]]
+        result = []
+        for band in bands:
+            np.random.shuffle(band)
+            result.extend(band)
+        return np.array(result)
+
+    return indices
+
+
+def detect_impossible_samples(
+    difficulties: np.ndarray,
+    threshold: float = IMPOSSIBLE_THRESHOLD,
+) -> list[int]:
+    """Find samples that are consistently difficult (likely mislabeled).
+
+    Returns list of sample indices.
+    """
+    return [int(i) for i in range(len(difficulties)) if difficulties[i] > threshold]
+
+
+# --- Strategy Comparison ---
+
+
+def compare_strategies(
+    strategy_results: dict[str, dict],
+    primary_metric: str = "accuracy",
+) -> dict:
+    """Compare curriculum strategy results.
+
+    Args:
+        strategy_results: {strategy_name: {metric_value, convergence_epoch, ...}}
+
+    Returns:
+        Comparison report with best strategy and verdict.
+    """
+    if not strategy_results:
+        return {"best_strategy": None, "verdict": "no_data"}
+
+    # Find baseline (random)
+    baseline = strategy_results.get("random", {})
+    baseline_metric = baseline.get("metric_value", 0)
+    baseline_epochs = baseline.get("convergence_epoch")
+
+    results = []
+    for name, data in strategy_results.items():
+        metric = data.get("metric_value", 0)
+        epochs = data.get("convergence_epoch")
+        speedup = None
+        if epochs and baseline_epochs and baseline_epochs > 0:
+            speedup = round(1 - epochs / baseline_epochs, 4)
+
+        results.append({
+            "strategy": name,
+            "metric_value": round(metric, 6) if metric else None,
+            "convergence_epoch": epochs,
+            "delta_vs_random": round(metric - baseline_metric, 6) if metric and baseline_metric else None,
+            "speedup": speedup,
+        })
+
+    # Find best by metric
+    with_metric = [r for r in results if r["metric_value"] is not None]
+    best = max(with_metric, key=lambda r: r["metric_value"]) if with_metric else None
+
+    verdict = "no_improvement"
+    if best and best.get("delta_vs_random") and best["delta_vs_random"] > 0.005:
+        verdict = "curriculum_helps"
+    elif best and best.get("speedup") and best["speedup"] > 0.1:
+        verdict = "faster_convergence"
+
+    return {
+        "results": results,
+        "best_strategy": best.get("strategy") if best else None,
+        "verdict": verdict,
+    }
+
+
+# --- Full Pipeline ---
+
+
+def curriculum_analysis(
+    difficulties: np.ndarray | None = None,
+    strategy_results: dict[str, dict] | None = None,
+    exp_id: str | None = None,
+    config_path: str = "config.yaml",
+) -> dict:
+    """Run curriculum analysis."""
+    config = load_config(config_path)
+    primary_metric = config.get("evaluation", {}).get("primary_metric", "accuracy")
+
+    report = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "experiment_id": exp_id,
+        "primary_metric": primary_metric,
+    }
+
+    if difficulties is not None:
+        impossible = detect_impossible_samples(difficulties)
+        report["difficulty_stats"] = {
+            "n_samples": len(difficulties),
+            "mean_difficulty": round(float(np.mean(difficulties)), 4),
+            "n_impossible": len(impossible),
+            "impossible_indices": impossible[:20],
+        }
+
+    if strategy_results:
+        comparison = compare_strategies(strategy_results, primary_metric)
+        report["comparison"] = comparison
+    else:
+        report["note"] = "Provide strategy_results for comparison. Use /turing:curriculum to run strategies."
+        report["available_strategies"] = DEFAULT_STRATEGIES
+
+    return report
+
+
+# --- Report Formatting ---
+
+
+def save_curriculum_report(report: dict, output_dir: str = "experiments/curriculum") -> Path:
+    out_path = Path(output_dir)
+    out_path.mkdir(parents=True, exist_ok=True)
+    exp_id = report.get("experiment_id", "unknown")
+    filepath = out_path / f"{exp_id}-curriculum.yaml"
+    clean = json.loads(json.dumps(report, default=str))
+    with open(filepath, "w") as f:
+        yaml.dump(clean, f, default_flow_style=False, sort_keys=False)
+    return filepath
+
+
+def format_curriculum_report(report: dict) -> str:
+    if "error" in report:
+        return f"ERROR: {report['error']}"
+
+    exp_id = report.get("experiment_id", "?")
+    metric = report.get("primary_metric", "metric")
+
+    lines = [f"# Curriculum Analysis: {exp_id}", "",
+             f"*Generated {report.get('generated_at', 'N/A')[:19]}*", ""]
+
+    # Difficulty stats
+    diff_stats = report.get("difficulty_stats")
+    if diff_stats:
+        lines.extend([
+            "## Difficulty Distribution",
+            f"- **Samples:** {diff_stats['n_samples']}",
+            f"- **Mean difficulty:** {diff_stats['mean_difficulty']:.4f}",
+            f"- **Impossible samples:** {diff_stats['n_impossible']} (likely mislabeled)",
+            "",
+        ])
+
+    # Strategy comparison
+    comparison = report.get("comparison")
+    if comparison:
+        results = comparison.get("results", [])
+        if results:
+            lines.extend(["## Strategy Comparison", "",
+                         f"| Strategy | {metric} | Δ vs Random | Speedup |",
+                         "|----------|--------|-------------|---------|"])
+            best_name = comparison.get("best_strategy")
+            for r in results:
+                val = f"{r['metric_value']:.4f}" if r.get("metric_value") is not None else "N/A"
+                delta = f"{r['delta_vs_random']:+.4f}" if r.get("delta_vs_random") is not None else "—"
+                speedup = f"{r['speedup']:+.0%}" if r.get("speedup") is not None else "—"
+                marker = " ← BEST" if r["strategy"] == best_name else ""
+                lines.append(f"| {r['strategy']} | {val} | {delta} | {speedup} |{marker}")
+            lines.append("")
+
+        verdict_labels = {
+            "curriculum_helps": "Curriculum learning improves final performance",
+            "faster_convergence": "Curriculum learning converges faster (similar final performance)",
+            "no_improvement": "No significant improvement from curriculum ordering",
+        }
+        verdict = comparison.get("verdict", "?")
+        lines.extend(["## Verdict", "", f"**{verdict_labels.get(verdict, verdict.upper())}**"])
+    elif report.get("note"):
+        lines.append(f"*{report['note']}*")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Training curriculum optimization")
+    parser.add_argument("exp_id", nargs="?", help="Experiment ID")
+    parser.add_argument("--strategies", help="Comma-separated strategies")
+    parser.add_argument("--config", default="config.yaml")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    report = curriculum_analysis(exp_id=args.exp_id, config_path=args.config)
+
+    if "error" not in report:
+        filepath = save_curriculum_report(report)
+        print(f"Saved to {filepath}", file=sys.stderr)
+
+    if args.json:
+        print(json.dumps(report, indent=2, default=str))
+    else:
+        print(format_curriculum_report(report))
+
+
+if __name__ == "__main__":
+    main()