claude-turing 1.0.0

package/templates/scripts/show_families.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Experiment family viewer for the autoresearch pipeline.
+
+Groups experiments by strategic theme (family tag) and shows
+per-family performance summaries. Tells the human when an entire
+research direction is exhausted.
+
+Usage:
+    python scripts/show_families.py [--log experiments/log.jsonl] [--config config.yaml]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+from scripts.turing_io import load_experiments
+
+
+def group_by_family(experiments: list[dict]) -> dict[str, list[dict]]:
+    """Group experiments by family tag."""
+    families: dict[str, list[dict]] = {}
+    for exp in experiments:
+        family = exp.get("family") or "untagged"
+        families.setdefault(family, []).append(exp)
+    return families
+
+
+def family_stats(
+    experiments: list[dict],
+    metric: str,
+    lower_is_better: bool,
+) -> dict:
+    """Compute stats for a family of experiments."""
+    total = len(experiments)
+    kept = [e for e in experiments if e.get("status") == "kept"]
+    discarded = [e for e in experiments if e.get("status") == "discarded"]
+
+    metric_vals = [
+        e.get("metrics", {}).get(metric)
+        for e in kept
+        if e.get("metrics", {}).get(metric) is not None
+    ]
+
+    best_val = None
+    best_id = None
+    if metric_vals:
+        if lower_is_better:
+            best_val = min(metric_vals)
+        else:
+            best_val = max(metric_vals)
+        for e in kept:
+            if e.get("metrics", {}).get(metric) == best_val:
+                best_id = e.get("experiment_id")
+                break
+
+    # Compute recent trend (last 3 experiments)
+    recent = experiments[-3:] if len(experiments) >= 3 else experiments
+    recent_kept = sum(1 for e in recent if e.get("status") == "kept")
+    recent_rate = recent_kept / len(recent) if recent else 0
+
+    return {
+        "total": total,
+        "kept": len(kept),
+        "discarded": len(discarded),
+        "keep_rate": round(len(kept) / total, 2) if total > 0 else 0,
+        "best_metric": best_val,
+        "best_experiment": best_id,
+        "recent_keep_rate": round(recent_rate, 2),
+        "exhausted": total >= 3 and recent_rate == 0,
+    }
+
+
+def format_families(
+    families: dict[str, dict],
+    metric_name: str,
+) -> str:
+    """Format family summaries as a table."""
+    if not families:
+        return "No experiments logged yet."
+
+    lines = [
+        f"{'Family':<25} {'Total':>6} {'Kept':>6} {'Rate':>6} {'Best ' + metric_name:>15} {'Status':<12}",
+        "-" * 80,
+    ]
+
+    for name, stats in sorted(families.items(), key=lambda x: -(x[1].get("best_metric") or 0)):
+        best_str = f"{stats['best_metric']:.4f}" if stats["best_metric"] is not None else "N/A"
+        status = "EXHAUSTED" if stats["exhausted"] else "active"
+        line = f"{name:<25} {stats['total']:>6} {stats['kept']:>6} {stats['keep_rate']:>5.0%} {best_str:>15} {status:<12}"
+        lines.append(line)
+
+    # Summary
+    total_exps = sum(s["total"] for s in families.values())
+    exhausted = sum(1 for s in families.values() if s["exhausted"])
+    lines.extend([
+        "",
+        f"Total: {total_exps} experiments across {len(families)} families ({exhausted} exhausted)",
+    ])
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Show experiment families")
+    parser.add_argument("--log", default="experiments/log.jsonl")
+    parser.add_argument("--config", default="config.yaml")
+    args = parser.parse_args()
+
+    config = {}
+    if Path(args.config).exists():
+        with open(args.config) as f:
+            config = yaml.safe_load(f) or {}
+
+    eval_cfg = config.get("evaluation", {})
+    metric = eval_cfg.get("primary_metric", "accuracy")
+    lower_is_better = eval_cfg.get("lower_is_better", False)
+
+    experiments = load_experiments(args.log)
+    families = group_by_family(experiments)
+
+    family_data = {name: family_stats(exps, metric, lower_is_better) for name, exps in families.items()}
+
+    print(format_families(family_data, metric))
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/show_metrics.py

@@ -0,0 +1,157 @@
+"""Display experiment metrics from experiments/log.jsonl.
+
+Reads the JSONL experiment log and prints a formatted table,
+highlighting the current best by primary metric. This is the
+agent's primary observation tool at the start of each loop iteration.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+from scripts.turing_io import load_config, load_experiments
+
+DEFAULT_LOG_PATH = "experiments/log.jsonl"
+
+
+def find_best(experiments: list[dict], primary_metric: str, lower_is_better: bool) -> str | None:
+    """Find experiment_id with best primary metric among 'keep' entries."""
+    best_id = None
+    best_value = float("inf") if lower_is_better else float("-inf")
+
+    for exp in experiments:
+        if exp.get("status") != "kept":
+            continue
+        value = exp.get("metrics", {}).get(primary_metric)
+        if value is None:
+            continue
+        if lower_is_better and value < best_value:
+            best_value = value
+            best_id = exp["experiment_id"]
+        elif not lower_is_better and value > best_value:
+            best_value = value
+            best_id = exp["experiment_id"]
+    return best_id
+
+
+def format_table(experiments: list[dict], best_id: str | None, metric_names: list[str]) -> str:
+    """Format experiments as a text table."""
+    if not experiments:
+        return "No experiments logged yet."
+
+    # Build dynamic header based on configured metrics
+    metric_headers = "".join(f"{m:>12}" for m in metric_names)
+    header = f"{'ID':<10} {'Status':<10} {'Model':<15}{metric_headers} {'Timestamp':<22}"
+    sep = "-" * len(header)
+    lines = [header, sep]
+
+    for exp in experiments:
+        metrics = exp.get("metrics", {})
+        model_type = exp.get("config", {}).get("model_type", "unknown")
+
+        metric_values = ""
+        for m in metric_names:
+            val = metrics.get(m)
+            if isinstance(val, (int, float)):
+                metric_values += f"{val:>12.4f}"
+            else:
+                metric_values += f"{'N/A':>12}"
+
+        ts = exp.get("timestamp", "")[:19]
+        marker = " *BEST*" if exp.get("experiment_id") == best_id else ""
+        line = f"{exp.get('experiment_id', '?'):<10} {exp.get('status', '?'):<10} {model_type:<15}{metric_values} {ts}{marker}"
+        lines.append(line)
+
+    return "\n".join(lines)
+
+
+def get_experiment_diffs(experiments: list[dict], max_diffs: int = 3) -> str:
+    """Get git diffs for recent discarded experiments."""
+    import subprocess
+
+    discarded = [e for e in experiments if e.get("status") == "discarded"]
+    if not discarded:
+        return ""
+
+    recent = discarded[-max_diffs:]
+    lines = ["\n--- Recent Failed Experiment Diffs ---\n"]
+
+    for exp in recent:
+        exp_id = exp.get("experiment_id", "unknown")
+        description = exp.get("description", "no description")
+        lines.append(f"=== {exp_id}: {description} ===")
+
+        # Try to find the experiment branch
+        branch = f"exp/{exp_id.replace('exp-', '')}"
+        try:
+            result = subprocess.run(
+                ["git", "diff", f"main...{branch}", "--", "train.py", "config.yaml"],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            if result.returncode == 0 and result.stdout.strip():
+                # Truncate long diffs
+                diff_lines = result.stdout.strip().splitlines()
+                if len(diff_lines) > 30:
+                    diff_lines = diff_lines[:30] + [f"... ({len(diff_lines) - 30} more lines)"]
+                lines.append("\n".join(diff_lines))
+            else:
+                lines.append("  (branch not found or no diff)")
+        except (subprocess.TimeoutExpired, FileNotFoundError):
+            lines.append("  (git not available)")
+
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Show experiment metrics")
+    parser.add_argument(
+        "--log",
+        default=DEFAULT_LOG_PATH,
+        help=f"Path to experiment log (default: {DEFAULT_LOG_PATH})",
+    )
+    parser.add_argument(
+        "--last",
+        type=int,
+        default=None,
+        help="Show only last N experiments",
+    )
+    parser.add_argument(
+        "--with-diffs",
+        action="store_true",
+        help="Include git diffs for discarded experiments",
+    )
+    args = parser.parse_args()
+
+    config = load_config()
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+    metric_names = eval_cfg.get("metrics", ["accuracy", "f1_weighted"])
+    lower_is_better = eval_cfg.get("lower_is_better", False)
+
+    experiments = load_experiments(args.log)
+
+    if args.last and args.last > 0:
+        experiments = experiments[-args.last:]
+
+    best_id = find_best(experiments, primary_metric, lower_is_better)
+    print(format_table(experiments, best_id, metric_names))
+
+    if args.with_diffs:
+        all_experiments = load_experiments(args.log)
+        diffs = get_experiment_diffs(all_experiments)
+        if diffs:
+            print(diffs)
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/statistical_compare.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+"""Multi-run statistical comparison for the autoresearch pipeline.
+
+Runs the same configuration N times with different random seeds and
+compares metric distributions rather than point estimates. This prevents
+keep/discard decisions based on noise.
+
+A model that scored 0.87 vs 0.86 on a single run might just be seed
+variance. This script runs both configurations multiple times and
+uses the Mann-Whitney U test to determine if the difference is real.
+
+Usage:
+    python scripts/statistical_compare.py --n-runs 3 [--config config.yaml]
+    python scripts/statistical_compare.py --compare <log1.jsonl> <log2.jsonl>
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import yaml
+
+
+def run_multi_seed(
+    n_runs: int,
+    config_path: str = "config.yaml",
+    base_seed: int = 42,
+) -> list[dict]:
+    """Run training N times with different seeds, collect metrics.
+
+    Args:
+        n_runs: Number of runs to execute.
+        config_path: Path to config.yaml.
+        base_seed: Starting seed (incremented for each run).
+
+    Returns:
+        List of metric dicts from each run.
+    """
+    results = []
+    for i in range(n_runs):
+        seed = base_seed + i
+        print(f"  Run {i+1}/{n_runs} (seed={seed})...", file=sys.stderr, end=" ")
+
+        cmd = f"python train.py --config {config_path} --seed {seed}"
+        proc = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, timeout=600,
+        )
+
+        if proc.returncode != 0:
+            print(f"FAILED", file=sys.stderr)
+            continue
+
+        # Parse metrics from output (between --- delimiters)
+        metrics = parse_metrics_from_output(proc.stdout)
+        if metrics:
+            metrics["seed"] = seed
+            results.append(metrics)
+            primary = next((v for k, v in metrics.items() if k not in ("seed", "model_type", "train_seconds")), None)
+            print(f"done ({primary})", file=sys.stderr)
+        else:
+            print(f"no metrics parsed", file=sys.stderr)
+
+    return results
+
+
+def parse_metrics_from_output(output: str) -> dict:
+    """Parse metrics from the --- delimited format in train.py output."""
+    lines = output.strip().split("\n")
+    in_block = False
+    metrics = {}
+    metadata_keys = {"model_type", "train_seconds"}
+
+    for line in lines:
+        line = line.strip()
+        if line == "---":
+            if in_block:
+                break  # end of block
+            in_block = True
+            continue
+        if in_block and ":" in line:
+            key, value = line.split(":", 1)
+            key = key.strip()
+            value = value.strip()
+            if key in metadata_keys:
+                metrics[key] = value
+            else:
+                try:
+                    metrics[key] = float(value)
+                except ValueError:
+                    metrics[key] = value
+
+    return metrics
+
+
+def compute_statistics(results: list[dict], metric_name: str) -> dict:
+    """Compute summary statistics for a metric across runs.
+
+    Returns dict with mean, std, min, max, ci_lower, ci_upper (95%).
+    """
+    values = [r[metric_name] for r in results if metric_name in r and isinstance(r[metric_name], (int, float))]
+
+    if not values:
+        return {"n": 0, "mean": None, "std": None}
+
+    arr = np.array(values)
+    n = len(arr)
+    mean = float(np.mean(arr))
+    std = float(np.std(arr, ddof=1)) if n > 1 else 0.0
+
+    # 95% confidence interval (t-distribution approximation for small n)
+    if n > 1:
+        from scipy import stats as scipy_stats
+        t_crit = scipy_stats.t.ppf(0.975, df=n - 1)
+        margin = t_crit * std / np.sqrt(n)
+    else:
+        margin = 0.0
+
+    return {
+        "n": n,
+        "mean": round(mean, 6),
+        "std": round(std, 6),
+        "min": round(float(np.min(arr)), 6),
+        "max": round(float(np.max(arr)), 6),
+        "ci_lower": round(mean - margin, 6),
+        "ci_upper": round(mean + margin, 6),
+        "values": [round(float(v), 6) for v in arr],
+    }
+
+
+def mann_whitney_test(values_a: list[float], values_b: list[float]) -> dict:
+    """Run Mann-Whitney U test comparing two sets of metric values.
+
+    Returns dict with statistic, p_value, and verdict.
+    """
+    if len(values_a) < 2 or len(values_b) < 2:
+        return {
+            "statistic": None,
+            "p_value": None,
+            "verdict": "insufficient_data",
+            "detail": f"Need at least 2 values per group (got {len(values_a)} and {len(values_b)})",
+        }
+
+    from scipy import stats as scipy_stats
+    stat, p_value = scipy_stats.mannwhitneyu(
+        values_a, values_b, alternative="two-sided",
+    )
+
+    if p_value < 0.05:
+        mean_a = np.mean(values_a)
+        mean_b = np.mean(values_b)
+        verdict = "significantly_different"
+        detail = f"p={p_value:.4f} < 0.05 — the difference is statistically significant"
+    else:
+        verdict = "not_significant"
+        detail = f"p={p_value:.4f} >= 0.05 — the difference could be random noise"
+
+    return {
+        "statistic": round(float(stat), 4),
+        "p_value": round(float(p_value), 6),
+        "verdict": verdict,
+        "detail": detail,
+    }
+
+
+def format_comparison_report(
+    stats_a: dict,
+    stats_b: dict,
+    test_result: dict,
+    label_a: str,
+    label_b: str,
+    metric_name: str,
+    lower_is_better: bool,
+) -> str:
+    """Format a statistical comparison report."""
+    direction = "lower" if lower_is_better else "higher"
+    lines = [
+        f"# Statistical Comparison: {metric_name} ({direction} is better)",
+        "",
+        f"| Statistic | {label_a} | {label_b} |",
+        f"|-----------|{'---' * len(label_a)}--|{'---' * len(label_b)}--|",
+        f"| N runs | {stats_a['n']} | {stats_b['n']} |",
+        f"| Mean | {stats_a['mean']:.4f} | {stats_b['mean']:.4f} |",
+        f"| Std | {stats_a['std']:.4f} | {stats_b['std']:.4f} |",
+        f"| Min | {stats_a['min']:.4f} | {stats_b['min']:.4f} |",
+        f"| Max | {stats_a['max']:.4f} | {stats_b['max']:.4f} |",
+        f"| 95% CI | [{stats_a['ci_lower']:.4f}, {stats_a['ci_upper']:.4f}] | [{stats_b['ci_lower']:.4f}, {stats_b['ci_upper']:.4f}] |",
+        "",
+        "## Mann-Whitney U Test",
+        "",
+        f"- **Verdict:** {test_result['verdict']}",
+        f"- **Detail:** {test_result['detail']}",
+    ]
+
+    if test_result["statistic"] is not None:
+        lines.append(f"- **U statistic:** {test_result['statistic']}")
+        lines.append(f"- **p-value:** {test_result['p_value']}")
+
+    lines.extend(["", "## Recommendation", ""])
+
+    if test_result["verdict"] == "significantly_different":
+        mean_a = stats_a["mean"]
+        mean_b = stats_b["mean"]
+        if lower_is_better:
+            better = label_a if mean_a < mean_b else label_b
+        else:
+            better = label_a if mean_a > mean_b else label_b
+        lines.append(f"**{better}** is statistically better. Safe to keep.")
+    elif test_result["verdict"] == "not_significant":
+        lines.append("The difference is not statistically significant. Consider:")
+        lines.append("- Running more trials (increase n_runs)")
+        lines.append("- Keeping the simpler/faster configuration")
+        lines.append("- Treating this as a tie and moving to a different hypothesis")
+    else:
+        lines.append("Insufficient data for statistical comparison. Run more trials.")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Multi-run statistical comparison for ML experiments"
+    )
+    parser.add_argument("--n-runs", type=int, default=3, help="Number of runs per configuration")
+    parser.add_argument("--config", default="config.yaml", help="Config file path")
+    parser.add_argument("--seed", type=int, default=42, help="Base random seed")
+    args = parser.parse_args()
+
+    print(f"Running {args.n_runs} trials with seeds {args.seed} to {args.seed + args.n_runs - 1}...", file=sys.stderr)
+    results = run_multi_seed(args.n_runs, args.config, args.seed)
+
+    if not results:
+        print("No successful runs.", file=sys.stderr)
+        sys.exit(1)
+
+    # Load config for metric info
+    config = {}
+    if Path(args.config).exists():
+        with open(args.config) as f:
+            config = yaml.safe_load(f) or {}
+
+    eval_cfg = config.get("evaluation", {})
+    metric = eval_cfg.get("primary_metric", "accuracy")
+
+    stats = compute_statistics(results, metric)
+    print(f"\n{metric} over {stats['n']} runs:")
+    print(f"  Mean:   {stats['mean']:.4f}")
+    print(f"  Std:    {stats['std']:.4f}")
+    print(f"  95% CI: [{stats['ci_lower']:.4f}, {stats['ci_upper']:.4f}]")
+    print(f"  Values: {stats['values']}")
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/stop-hook.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Convergence detection hook for the autoresearch pipeline.
+#
+# Fired by Claude Code Stop hook after each training iteration.
+# Thin wrapper around scripts/check_convergence.py — the actual
+# algorithm lives in testable Python, not inline bash.
+#
+# This implements a discrete analogue of early stopping from gradient
+# descent, adapted for the experiment loop context.
+#
+# Exit codes:
+#   0 = not converged, agent should continue
+#   2 = converged, agent should stop (signals /loop to halt)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ML_DIR="$(dirname "$SCRIPT_DIR")"
+EXPERIMENT_LOG="${ML_DIR}/experiments/log.jsonl"
+CONFIG_FILE="${ML_DIR}/config.yaml"
+
+# Check if log.jsonl exists
+if [[ ! -f "$EXPERIMENT_LOG" ]]; then
+    echo "stop-hook: No log.jsonl found, not enough data to judge." >&2
+    exit 0
+fi
+
+# Activate venv and delegate to Python module
+cd "$ML_DIR"
+source .venv/bin/activate 2>/dev/null || true
+
+python3 scripts/check_convergence.py \
+    --config "$CONFIG_FILE" \
+    --log "$EXPERIMENT_LOG"