claude-turing 1.0.0

package/templates/scripts/validate_stability.py

@@ -0,0 +1,167 @@
+"""Stability validation for experiment metrics.
+
+Runs the training pipeline N times and computes coefficient of variation.
+If CV > 5%, the metric is too noisy for reliable single-run comparison.
+
+Usage:
+    python scripts/validate_stability.py              # Check stability (5 runs)
+    python scripts/validate_stability.py --runs 10    # Custom run count
+    python scripts/validate_stability.py --auto       # Auto-fix if unstable
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import statistics
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+from scripts.turing_io import load_config
+
+DEFAULT_RUNS = 5
+CV_THRESHOLD = 5.0  # Percent
+
+
+def run_experiment(seed: int) -> float | None:
+    """Run a single training pass and extract the primary metric."""
+    config = load_config()
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+
+    result = subprocess.run(
+        ["python", "train.py", "--seed", str(seed)],
+        capture_output=True,
+        text=True,
+        timeout=600,
+    )
+
+    # Parse metric from stdout (between --- delimiters)
+    output = result.stdout
+    in_block = False
+    for line in output.splitlines():
+        if line.strip() == "---":
+            in_block = not in_block
+            continue
+        if in_block:
+            match = re.match(rf"^{re.escape(primary_metric)}:\s*(.+)$", line.strip())
+            if match:
+                try:
+                    return float(match.group(1))
+                except ValueError:
+                    pass
+    return None
+
+
+def check_stability(n_runs: int = DEFAULT_RUNS) -> dict:
+    """Run N experiments and compute stability metrics."""
+    config = load_config()
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "accuracy")
+
+    print(f"Running {n_runs} stability validation passes...")
+    print(f"Primary metric: {primary_metric}")
+    print()
+
+    results = []
+    for i in range(n_runs):
+        seed = 42 + i  # Deterministic but different seeds
+        print(f"  Run {i + 1}/{n_runs} (seed={seed})...", end=" ", flush=True)
+        value = run_experiment(seed)
+        if value is not None:
+            results.append(value)
+            print(f"{primary_metric}={value:.4f}")
+        else:
+            print("FAILED (no metric)")
+
+    if len(results) < 2:
+        return {
+            "stable": False,
+            "reason": f"Only {len(results)} successful runs out of {n_runs}",
+            "results": results,
+            "recommendation": "Fix pipeline errors before validating stability",
+        }
+
+    mean = statistics.mean(results)
+    stdev = statistics.stdev(results)
+    cv = (stdev / abs(mean) * 100) if mean != 0 else float("inf")
+
+    stable = cv < CV_THRESHOLD
+
+    return {
+        "stable": stable,
+        "n_runs": len(results),
+        "mean": round(mean, 4),
+        "stdev": round(stdev, 4),
+        "cv_percent": round(cv, 2),
+        "min": round(min(results), 4),
+        "max": round(max(results), 4),
+        "results": results,
+        "recommendation": (
+            "Metric is stable — single-run evaluation is reliable."
+            if stable
+            else f"CV={cv:.1f}% exceeds {CV_THRESHOLD}% threshold. "
+            f"Recommend setting evaluation.n_runs: 3 in config.yaml "
+            f"to use median of 3 runs per experiment."
+        ),
+    }
+
+
+def auto_fix_config() -> bool:
+    """Set evaluation.n_runs: 3 in config.yaml if not already set."""
+    config_path = Path("config.yaml")
+    if not config_path.exists():
+        return False
+
+    with open(config_path) as f:
+        config = yaml.safe_load(f) or {}
+
+    eval_cfg = config.setdefault("evaluation", {})
+    if eval_cfg.get("n_runs", 1) >= 3:
+        return False  # Already configured
+
+    eval_cfg["n_runs"] = 3
+    with open(config_path, "w") as f:
+        yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+
+    return True
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Validate experiment metric stability")
+    parser.add_argument("--runs", type=int, default=DEFAULT_RUNS, help=f"Number of runs (default: {DEFAULT_RUNS})")
+    parser.add_argument("--auto", action="store_true", help="Auto-configure multi-run if unstable")
+    args = parser.parse_args()
+
+    result = check_stability(args.runs)
+
+    print()
+    print("=" * 50)
+    print(f"  Stability: {'STABLE' if result['stable'] else 'UNSTABLE'}")
+    if "mean" in result:
+        print(f"  Mean:      {result['mean']}")
+        print(f"  Stdev:     {result['stdev']}")
+        print(f"  CV:        {result['cv_percent']}%")
+        print(f"  Range:     [{result['min']}, {result['max']}]")
+    print(f"  Runs:      {result.get('n_runs', 0)}")
+    print()
+    print(f"  {result['recommendation']}")
+    print("=" * 50)
+
+    if args.auto and not result["stable"]:
+        if auto_fix_config():
+            print()
+            print("  AUTO-FIX: Set evaluation.n_runs: 3 in config.yaml")
+            print("  Each experiment will now run 3 times and report the median.")
+        else:
+            print()
+            print("  evaluation.n_runs already >= 3, no changes made.")
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/verify_placeholders.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Post-scaffolding placeholder verification.
+
+Scans all scaffolded files for unreplaced {{PLACEHOLDER}} markers and
+reports any that remain. Returns exit code 1 if any are found.
+
+This converts silent-wrong-behavior failures into loud-and-immediate errors.
+A missed placeholder like {{TARGET_METRIC}} produces valid Python that silently
+does the wrong thing — this script catches it before the first experiment runs.
+
+Usage:
+    python scripts/verify_placeholders.py [directory]
+
+    directory: path to scan (default: current directory)
+
+Exit codes:
+    0 = all placeholders replaced
+    1 = unreplaced placeholders found
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+# Files to scan for placeholders (relative to project root)
+SCANNABLE_EXTENSIONS = {".py", ".yaml", ".yml", ".md", ".toml", ".sh", ".txt"}
+
+# Known placeholder pattern
+PLACEHOLDER_RE = re.compile(r"\{\{([A-Z_]+)\}\}")
+
+# Known valid placeholders (from config/defaults.yaml)
+KNOWN_PLACEHOLDERS = {
+    "PROJECT_NAME",
+    "TARGET_METRIC",
+    "TASK_DESCRIPTION",
+    "ML_DIR",
+    "DATA_SOURCE",
+    "METRIC_DIRECTION",
+}
+
+
+def scan_file(path: Path) -> list[tuple[int, str, str]]:
+    """Scan a single file for unreplaced placeholders.
+
+    Returns list of (line_number, placeholder_name, line_content) tuples.
+    """
+    findings: list[tuple[int, str, str]] = []
+    try:
+        text = path.read_text(encoding="utf-8")
+    except (UnicodeDecodeError, PermissionError):
+        return findings
+
+    for i, line in enumerate(text.splitlines(), start=1):
+        for match in PLACEHOLDER_RE.finditer(line):
+            placeholder = match.group(1)
+            if placeholder in KNOWN_PLACEHOLDERS:
+                findings.append((i, placeholder, line.strip()))
+
+    return findings
+
+
+def scan_directory(root: Path) -> dict[Path, list[tuple[int, str, str]]]:
+    """Scan all files in directory for unreplaced placeholders.
+
+    Returns dict mapping file paths to their findings.
+    """
+    results: dict[Path, list[tuple[int, str, str]]] = {}
+
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        if path.suffix not in SCANNABLE_EXTENSIONS:
+            continue
+        # Skip venv, node_modules, git
+        parts = path.parts
+        if any(p in (".venv", "node_modules", ".git", "__pycache__") for p in parts):
+            continue
+
+        findings = scan_file(path)
+        if findings:
+            results[path] = findings
+
+    return results
+
+
+def main() -> None:
+    """CLI entry point."""
+    root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(".")
+
+    if not root.is_dir():
+        print(f"Error: {root} is not a directory", file=sys.stderr)
+        sys.exit(1)
+
+    results = scan_directory(root)
+
+    if not results:
+        print("All placeholders replaced successfully.")
+        sys.exit(0)
+
+    # Report findings
+    total = sum(len(findings) for findings in results.values())
+    print(f"Found {total} unreplaced placeholder(s) in {len(results)} file(s):\n")
+
+    for path, findings in results.items():
+        rel_path = path.relative_to(root) if path.is_relative_to(root) else path
+        print(f"  {rel_path}:")
+        for line_num, placeholder, line_content in findings:
+            print(f"    line {line_num}: {{{{{placeholder}}}}} — {line_content[:80]}")
+        print()
+
+    print("Fix: replace all {{PLACEHOLDER}} markers with project-specific values.")
+    print("See config.yaml and program.md for placeholder descriptions.")
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/templates/sweep_config.yaml

@@ -0,0 +1,14 @@
+# Hyperparameter sweep configuration for {{PROJECT_NAME}}.
+# Each key under 'sweep' maps to a config.yaml dotted path.
+# Values are lists -- sweep.py generates the cartesian product.
+
+sweep:
+  model.hyperparams.n_estimators: [50, 100, 200]
+  model.hyperparams.max_depth: [3, 4, 6, 8]
+  model.hyperparams.learning_rate: [0.01, 0.05, 0.1]
+
+# Base config to overlay sweep values onto
+base_config: "config.yaml"
+
+# Output queue file
+output: "experiments/queue.yaml"

package/templates/tests/conftest.py

@@ -0,0 +1,91 @@
+"""Shared test fixtures for the {{PROJECT_NAME}} ML pipeline tests.
+
+These fixtures provide deterministic sample data for testing the
+pipeline components. Customize with records matching your actual
+data schema — the autoresearch agent uses these when running tests.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def sample_training_data() -> list[dict]:
+    """Sample training records for testing.
+
+    Replace these with records matching your data schema.
+    Include a representative mix of all label/target values.
+    """
+    # TODO: Replace with your actual data schema
+    return [
+        {"feature_1": 1.0, "feature_2": "category_a", "label": "positive"},
+        {"feature_1": 2.0, "feature_2": "category_b", "label": "positive"},
+        {"feature_1": 3.0, "feature_2": "category_a", "label": "negative"},
+        {"feature_1": 4.0, "feature_2": "category_b", "label": "negative"},
+        {"feature_1": 5.0, "feature_2": "category_a", "label": "positive"},
+        {"feature_1": 6.0, "feature_2": "category_b", "label": "negative"},
+        {"feature_1": 7.0, "feature_2": "category_a", "label": "positive"},
+        {"feature_1": 8.0, "feature_2": "category_b", "label": "negative"},
+        {"feature_1": 9.0, "feature_2": "category_a", "label": "positive"},
+        {"feature_1": 10.0, "feature_2": "category_b", "label": "negative"},
+    ]
+
+
+@pytest.fixture
+def sample_jsonl_file(tmp_path: Path, sample_training_data: list[dict]) -> Path:
+    """Write sample training data to a temporary JSONL file."""
+    path = tmp_path / "test_data.jsonl"
+    with open(path, "w") as f:
+        for record in sample_training_data:
+            f.write(json.dumps(record) + "\n")
+    return path
+
+
+@pytest.fixture
+def sample_config(tmp_path: Path) -> Path:
+    """Write a minimal config.yaml for testing."""
+    import yaml
+
+    config = {
+        "data": {
+            "source": str(tmp_path / "test_data.jsonl"),
+            "splits_dir": str(tmp_path / "splits"),
+            "target_column": "label",
+            "split_ratios": {"train": 0.70, "val": 0.15, "test": 0.15},
+            "random_state": 42,
+        },
+        "evaluation": {
+            "primary_metric": "accuracy",
+            "metrics": ["accuracy", "f1_weighted"],
+            "lower_is_better": False,
+        },
+        "convergence": {
+            "patience": 3,
+            "improvement_threshold": 0.005,
+        },
+        "model": {
+            "type": "xgboost",
+            "hyperparams": {
+                "n_estimators": 10,
+                "max_depth": 3,
+                "verbosity": 0,
+            },
+        },
+        "output": {
+            "models_dir": str(tmp_path / "models"),
+            "best_model_dir": str(tmp_path / "models" / "best"),
+            "archive_dir": str(tmp_path / "models" / "archive"),
+            "experiment_log": str(tmp_path / "experiments" / "log.jsonl"),
+            "results_tsv": str(tmp_path / "experiments" / "results.tsv"),
+        },
+    }
+
+    config_path = tmp_path / "config.yaml"
+    with open(config_path, "w") as f:
+        yaml.dump(config, f)
+
+    return config_path

package/templates/train.py

@@ -0,0 +1,240 @@
+# AGENT-EDITABLE — HYPOTHESIS SPACE.
+#
+# This is the ONLY code file the autoresearch agent should modify.
+# Training code for {{PROJECT_NAME}}.
+# Default implementation: XGBoost classifier.
+#
+# The agent iteratively modifies this file to test hypotheses about
+# model architecture, hyperparameters, and feature usage. The evaluation
+# harness (evaluate.py) remains immutable — ensuring all experiments
+# are measured by the same yardstick.
+#
+# See program.md for the full experiment loop protocol.
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import random
+import sys
+import time
+from pathlib import Path
+
+import joblib
+import numpy as np
+import pandas as pd
+from xgboost import XGBClassifier
+
+from evaluate import evaluate_model, format_metrics
+from features.featurizers import get_default_featurizer
+from prepare import load_config, load_splits
+
+
+def capture_environment(seed: int) -> dict:
+    """Capture the full runtime environment for reproducibility.
+
+    Records everything needed to reproduce this exact experiment:
+    python version, package versions, hardware, OS, and all random seeds.
+    """
+    env = {
+        "python_version": platform.python_version(),
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+        "os": platform.system(),
+        "seeds": {
+            "random_state": seed,
+            "PYTHONHASHSEED": os.environ.get("PYTHONHASHSEED", "not set"),
+        },
+        "packages": {},
+    }
+
+    # Key package versions
+    for pkg_name in ["xgboost", "lightgbm", "sklearn", "numpy", "pandas", "joblib", "scipy"]:
+        try:
+            mod = __import__(pkg_name)
+            env["packages"][pkg_name] = getattr(mod, "__version__", "unknown")
+        except ImportError:
+            pass
+
+    # GPU info (if torch is available)
+    try:
+        import torch
+        env["packages"]["torch"] = torch.__version__
+        if torch.cuda.is_available():
+            env["gpu"] = {
+                "name": torch.cuda.get_device_name(0),
+                "cuda_version": torch.version.cuda,
+                "device_count": torch.cuda.device_count(),
+            }
+            env["seeds"]["cuda_deterministic"] = True
+    except ImportError:
+        pass
+
+    # Config file hash for drift detection
+    config_path = Path("config.yaml")
+    if config_path.exists():
+        env["config_hash"] = hashlib.sha256(config_path.read_bytes()).hexdigest()[:16]
+
+    return env
+
+
+def pin_all_seeds(seed: int) -> None:
+    """Pin all random seeds for full reproducibility.
+
+    Covers: stdlib random, numpy, PYTHONHASHSEED.
+    Torch/CUDA seeds are pinned only if torch is available.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+
+    try:
+        import torch
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False
+    except ImportError:
+        pass
+
+
+def train_model(
+    config_path: str = "config.yaml",
+    output_dir: str | None = None,
+    seed_override: int | None = None,
+) -> None:
+    """Train a model and print metrics.
+
+    Flow:
+      1. Load config and pin seeds
+      2. Load train/val splits
+      3. Apply featurizer to get feature matrices
+      4. Train model with hyperparams from config
+      5. Predict on val set
+      6. Evaluate using evaluate_model()
+      7. Print formatted metrics
+      8. Save model to models/ directory
+
+    Customize this function for your specific ML task.
+    """
+    start_time = time.time()
+
+    # 1. Load config and pin all seeds
+    config = load_config(config_path)
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "{{TARGET_METRIC}}")
+    random_state = seed_override if seed_override is not None else config["data"].get("random_state", 42)
+    pin_all_seeds(random_state)
+
+    # 2. Load splits
+    splits_dir = config["data"]["splits_dir"]
+    train_df, val_df, _test_df = load_splits(splits_dir)
+
+    # 3. Apply featurizer
+    featurizer = get_default_featurizer()
+    target_column = config["data"].get("target_column", "label")
+
+    X_train = featurizer.fit_transform(train_df)
+    X_val = featurizer.transform(val_df)
+
+    # Fill NaN values that may arise from feature engineering
+    X_train = X_train.fillna(0)
+    X_val = X_val.fillna(0)
+
+    y_train = train_df[target_column].values
+    y_val = val_df[target_column].values
+
+    # 4. Train model (hyperparams from config.yaml)
+    model_config = config.get("model", {})
+    hyperparams = model_config.get("hyperparams", {})
+    model = XGBClassifier(
+        **hyperparams,
+        random_state=random_state,
+    )
+    model.fit(X_train, y_train)
+
+    # 5. Predict on train and val sets
+    y_pred_val = model.predict(X_val)
+    y_pred_train = model.predict(X_train)
+
+    # 6. Evaluate on val (primary) and train (gap monitoring)
+    metrics = evaluate_model(y_pred_val, y_val, config)
+    train_metrics = evaluate_model(y_pred_train, y_train, config)
+
+    # Compute train/val gap for overfitting detection
+    eval_cfg = config.get("evaluation", {})
+    primary_metric = eval_cfg.get("primary_metric", "{{TARGET_METRIC}}")
+    val_score = metrics.get(primary_metric)
+    train_score = train_metrics.get(primary_metric)
+    if val_score is not None and train_score is not None:
+        metrics["train_" + primary_metric] = train_score
+        metrics["overfit_gap"] = round(train_score - val_score, 4)
+
+    train_seconds = time.time() - start_time
+
+    # Add metadata for parseable output
+    metrics["model_type"] = model_config.get("type", "xgboost")
+    metrics["train_seconds"] = round(train_seconds, 1)
+
+    # 7. Print formatted metrics
+    print(format_metrics(metrics))
+
+    # 8. Save model
+    models_dir = output_dir or config["output"]["models_dir"]
+    models_path = Path(models_dir)
+    models_path.mkdir(parents=True, exist_ok=True)
+
+    model_file = models_path / "model.joblib"
+    joblib.dump(
+        {
+            "model": model,
+            "featurizer": featurizer,
+            "config": config,
+        },
+        model_file,
+    )
+    print(f"\nModel saved to {model_file}")
+
+    # 9. Write training metadata for behavioral validation + reproducibility
+    metadata = {
+        "train_time_sec": round(train_seconds, 1),
+        "model_size_bytes": model_file.stat().st_size if model_file.exists() else 0,
+        "n_train_samples": len(X_train),
+        "n_features": X_train.shape[1] if hasattr(X_train, 'shape') else len(X_train.columns),
+        "predictions_unique": int(len(set(y_pred_val.tolist()))),
+        "environment": capture_environment(random_state),
+    }
+    metadata_path = Path("train_metadata.json")
+    with open(metadata_path, "w") as f:
+        json.dump(metadata, f, indent=2)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Train {{PROJECT_NAME}} model")
+    parser.add_argument(
+        "--config",
+        default="config.yaml",
+        help="Path to config override (default: config.yaml)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default=None,
+        help="Model output directory (default: from config)",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="Override random_state for multi-run statistical comparison",
+    )
+    args = parser.parse_args()
+    train_model(config_path=args.config, output_dir=args.output_dir, seed_override=args.seed)
+
+
+if __name__ == "__main__":
+    main()