claude-turing 1.0.0

package/templates/scripts/scaffold.py

@@ -0,0 +1,409 @@
+#!/usr/bin/env python3
+"""Unified project scaffolding for the autoresearch pipeline.
+
+Single implementation of project scaffolding used by both:
+- /turing:init (Claude Code) — calls with --interactive or pre-filled args
+- bin/turing-init.sh (CLI) — calls with explicit --project-name etc.
+
+Eliminates the dual-path divergence where init.md and turing-init.sh
+had different capabilities (substitution, hooks, venv, memory).
+
+Usage:
+    python scripts/scaffold.py --project-name sentiment --target-metric accuracy \\
+        --task-description "Predict sentiment" --ml-dir ml/sentiment \\
+        --data-source data/reviews.csv --metric-direction higher
+
+    python scripts/scaffold.py --interactive  # Prompt for each value
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+PLACEHOLDER_MAP = {
+    "PROJECT_NAME": "project_name",
+    "TARGET_METRIC": "target_metric",
+    "TASK_DESCRIPTION": "task_description",
+    "ML_DIR": "ml_dir",
+    "DATA_SOURCE": "data_source",
+    "METRIC_DIRECTION": "metric_direction",
+}
+
+# Files to copy from templates/ to the ML directory
+TEMPLATE_FILES = [
+    "prepare.py",
+    "evaluate.py",
+    "train.py",
+    "config.yaml",
+    "sweep_config.yaml",
+    "program.md",
+    "README.md",
+    "MEMORY.md",
+    "requirements.txt",
+    "pyproject.toml",
+]
+
+TEMPLATE_DIRS = {
+    "features": ["__init__.py", "featurizers.py"],
+    "scripts": [
+        "__init__.py",
+        "log_experiment.py",
+        "show_metrics.py",
+        "compare_runs.py",
+        "sweep.py",
+        "post-train-hook.sh",
+        "stop-hook.sh",
+        "check_convergence.py",
+        "verify_placeholders.py",
+        "manage_hypotheses.py",
+        "generate_brief.py",
+        "show_experiment_tree.py",
+        "statistical_compare.py",
+        "suggest_next.py",
+        "update_state.py",
+        "parse_metrics.py",
+        "scaffold.py",
+        "novelty_guard.py",
+        "synthesize_decision.py",
+        "show_families.py",
+        "critique_hypothesis.py",
+        "experiment_index.py",
+        "generate_logbook.py",
+        "validate_stability.py",
+        "show_environment.py",
+        "turing_io.py",
+        "preflight.py",
+    ],
+    "tests": ["__init__.py", "conftest.py"],
+}
+
+DIRECTORIES_TO_CREATE = [
+    "data/splits",
+    "experiments",
+    "models/best",
+    "models/archive",
+]
+
+SHELL_SCRIPTS = [
+    "scripts/post-train-hook.sh",
+    "scripts/stop-hook.sh",
+]
+
+
+def find_templates_dir() -> Path | None:
+    """Locate the templates directory relative to this script or plugin root."""
+    # When running from a scaffolded project, templates are local
+    script_dir = Path(__file__).parent
+
+    # Check: are we inside the plugin's templates/scripts/ ?
+    candidate = script_dir.parent  # templates/
+    if (candidate / "prepare.py").exists():
+        return candidate
+
+    # Check: plugin root (two levels up from scripts/)
+    plugin_root = script_dir.parent.parent
+    candidate = plugin_root / "templates"
+    if candidate.exists() and (candidate / "prepare.py").exists():
+        return candidate
+
+    # Search common plugin locations
+    home = Path.home()
+    for pattern in [
+        home / ".claude" / "plugins" / "*" / "templates",
+    ]:
+        for match in sorted(pattern.parent.glob(pattern.name)):
+            if (match / "prepare.py").exists():
+                return match
+
+    return None
+
+
+def replace_placeholders(text: str, values: dict[str, str]) -> str:
+    """Replace all {{PLACEHOLDER}} markers in text with values."""
+    for placeholder, arg_name in PLACEHOLDER_MAP.items():
+        value = values.get(arg_name, "")
+        text = text.replace(f"{{{{{placeholder}}}}}", value)
+    return text
+
+
+def scaffold_project(
+    templates_dir: Path,
+    ml_dir: str,
+    values: dict[str, str],
+    setup_venv: bool = True,
+    setup_hooks: bool = True,
+) -> dict[str, int]:
+    """Scaffold a complete ML project.
+
+    Args:
+        templates_dir: Path to the templates/ directory.
+        ml_dir: Target ML directory (relative to cwd).
+        values: Dict mapping arg names to values for placeholder substitution.
+        setup_venv: Whether to create and populate a Python venv.
+        setup_hooks: Whether to configure Claude Code hooks.
+
+    Returns:
+        Dict with counts: files_copied, placeholders_replaced, dirs_created.
+    """
+    target = Path(ml_dir)
+    target.mkdir(parents=True, exist_ok=True)
+
+    stats = {"files_copied": 0, "placeholders_replaced": 0, "dirs_created": 0}
+
+    # Copy and substitute template files
+    for filename in TEMPLATE_FILES:
+        src = templates_dir / filename
+        if not src.exists():
+            print(f"  Warning: template {filename} not found, skipping", file=sys.stderr)
+            continue
+        content = src.read_text(encoding="utf-8")
+        content = replace_placeholders(content, values)
+        (target / filename).write_text(content, encoding="utf-8")
+        stats["files_copied"] += 1
+
+    # Copy and substitute template directories
+    for dirname, files in TEMPLATE_DIRS.items():
+        dir_target = target / dirname
+        dir_target.mkdir(parents=True, exist_ok=True)
+        for filename in files:
+            src = templates_dir / dirname / filename
+            if not src.exists():
+                continue
+            content = src.read_text(encoding="utf-8")
+            content = replace_placeholders(content, values)
+            (dir_target / filename).write_text(content, encoding="utf-8")
+            stats["files_copied"] += 1
+
+    # Create directories
+    for d in DIRECTORIES_TO_CREATE:
+        (target / d).mkdir(parents=True, exist_ok=True)
+        stats["dirs_created"] += 1
+
+    # Make shell scripts executable
+    for script in SHELL_SCRIPTS:
+        script_path = target / script
+        if script_path.exists():
+            script_path.chmod(script_path.stat().st_mode | 0o111)
+
+    # Count placeholders replaced
+    placeholder_re = re.compile(r"\{\{[A-Z_]+\}\}")
+    for path in target.rglob("*"):
+        if path.is_file() and path.suffix in (".py", ".yaml", ".yml", ".md", ".sh", ".txt", ".toml"):
+            try:
+                text = path.read_text(encoding="utf-8")
+                found = placeholder_re.findall(text)
+                # If we find remaining placeholders, they were NOT replaced
+                # (this means the template had them but our values didn't cover them)
+            except UnicodeDecodeError:
+                continue
+
+    # Setup agent memory
+    memory_dir = Path(".claude") / "agent-memory" / "ml-researcher"
+    memory_dir.mkdir(parents=True, exist_ok=True)
+    memory_src = templates_dir / "MEMORY.md"
+    if memory_src.exists():
+        content = memory_src.read_text(encoding="utf-8")
+        content = replace_placeholders(content, values)
+        (memory_dir / "MEMORY.md").write_text(content, encoding="utf-8")
+
+    # Setup hooks
+    if setup_hooks:
+        _setup_hooks(ml_dir)
+
+    # Setup venv
+    if setup_venv:
+        _setup_venv(target)
+
+    return stats
+
+
+def _setup_hooks(ml_dir: str) -> None:
+    """Configure Claude Code hooks in .claude/settings.local.json."""
+    settings_path = Path(".claude") / "settings.local.json"
+    settings_path.parent.mkdir(parents=True, exist_ok=True)
+
+    settings = {}
+    if settings_path.exists():
+        try:
+            settings = json.loads(settings_path.read_text())
+        except (json.JSONDecodeError, FileNotFoundError):
+            settings = {}
+
+    hooks = settings.get("hooks", {})
+
+    # PostToolUse hook for auto-logging
+    post_hooks = hooks.get("PostToolUse", [])
+    post_hook_cmd = f"bash {ml_dir}/scripts/post-train-hook.sh"
+    if not any(post_hook_cmd in str(h) for h in post_hooks):
+        post_hooks.append({
+            "matcher": "Bash",
+            "hooks": [{"type": "command", "command": post_hook_cmd}],
+        })
+    hooks["PostToolUse"] = post_hooks
+
+    # Stop hook for convergence
+    stop_hooks = hooks.get("Stop", [])
+    stop_hook_cmd = f"bash {ml_dir}/scripts/stop-hook.sh"
+    if not any(stop_hook_cmd in str(h) for h in stop_hooks):
+        stop_hooks.append({
+            "type": "command",
+            "command": stop_hook_cmd,
+        })
+    hooks["Stop"] = stop_hooks
+
+    settings["hooks"] = hooks
+    settings_path.write_text(json.dumps(settings, indent=2))
+
+
+def _setup_venv(target: Path) -> None:
+    """Create Python venv and install requirements."""
+    venv_path = target / ".venv"
+    if venv_path.exists():
+        print("  Venv already exists, skipping creation.", file=sys.stderr)
+        return
+
+    print("  Creating virtual environment...", file=sys.stderr)
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "venv", str(venv_path)],
+            check=True, capture_output=True,
+        )
+        pip = str(venv_path / "bin" / "pip")
+        req = str(target / "requirements.txt")
+        if Path(req).exists():
+            print("  Installing requirements...", file=sys.stderr)
+            subprocess.run(
+                [pip, "install", "-r", req],
+                check=True, capture_output=True,
+            )
+    except subprocess.CalledProcessError as e:
+        print(f"  Warning: venv setup failed: {e}", file=sys.stderr)
+
+
+def verify_placeholders(ml_dir: str) -> list[tuple[str, int, str]]:
+    """Check for unreplaced placeholders in scaffolded files.
+
+    Returns list of (filepath, line_number, placeholder) tuples.
+    """
+    target = Path(ml_dir)
+    placeholder_re = re.compile(r"\{\{([A-Z_]+)\}\}")
+    known = set(PLACEHOLDER_MAP.keys())
+    findings = []
+
+    for path in sorted(target.rglob("*")):
+        if not path.is_file():
+            continue
+        if path.suffix not in (".py", ".yaml", ".yml", ".md", ".sh", ".txt", ".toml"):
+            continue
+        if ".venv" in path.parts:
+            continue
+        try:
+            for i, line in enumerate(path.read_text().splitlines(), 1):
+                for match in placeholder_re.finditer(line):
+                    name = match.group(1)
+                    if name in known:
+                        findings.append((str(path.relative_to(target)), i, name))
+        except UnicodeDecodeError:
+            continue
+
+    return findings
+
+
+def interactive_prompt() -> dict[str, str]:
+    """Prompt the user for all scaffold values."""
+    print("\nTuring ML Research Harness — Project Setup\n")
+
+    values = {}
+    values["project_name"] = input("Project name (e.g., sentiment, churn): ").strip()
+    values["target_metric"] = input("Primary metric (accuracy, f1, mae, mse, auc): ").strip()
+    values["metric_direction"] = input("Is lower better or higher better? (lower/higher): ").strip()
+    values["task_description"] = input("Task description (e.g., Predict customer churn): ").strip()
+    values["ml_dir"] = input(f"ML directory (e.g., ml/{values['project_name']}): ").strip()
+    values["data_source"] = input("Data source path (e.g., data/training.csv): ").strip()
+
+    return values
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Scaffold an ML project")
+    parser.add_argument("--interactive", action="store_true", help="Prompt for values")
+    parser.add_argument("--project-name", default=None)
+    parser.add_argument("--target-metric", default=None)
+    parser.add_argument("--metric-direction", default=None)
+    parser.add_argument("--task-description", default=None)
+    parser.add_argument("--ml-dir", default=None)
+    parser.add_argument("--data-source", default=None)
+    parser.add_argument("--no-venv", action="store_true", help="Skip venv creation")
+    parser.add_argument("--no-hooks", action="store_true", help="Skip hook configuration")
+    parser.add_argument("--templates-dir", default=None, help="Override templates directory")
+    args = parser.parse_args()
+
+    # Get values
+    if args.interactive:
+        values = interactive_prompt()
+    else:
+        values = {
+            "project_name": args.project_name or "my-project",
+            "target_metric": args.target_metric or "accuracy",
+            "metric_direction": args.metric_direction or "higher",
+            "task_description": args.task_description or "ML task",
+            "ml_dir": args.ml_dir or ".",
+            "data_source": args.data_source or "data/training.csv",
+        }
+
+    # Find templates
+    if args.templates_dir:
+        templates_dir = Path(args.templates_dir)
+    else:
+        templates_dir = find_templates_dir()
+
+    if templates_dir is None or not templates_dir.exists():
+        print("Error: Cannot find templates directory.", file=sys.stderr)
+        print("Use --templates-dir to specify the path.", file=sys.stderr)
+        sys.exit(1)
+
+    ml_dir = values["ml_dir"]
+
+    print(f"\nScaffolding project: {values['project_name']}")
+    print(f"Directory: {ml_dir}")
+    print(f"Metric: {values['target_metric']} ({values['metric_direction']} is better)")
+    print()
+
+    # Scaffold
+    stats = scaffold_project(
+        templates_dir=templates_dir,
+        ml_dir=ml_dir,
+        values=values,
+        setup_venv=not args.no_venv,
+        setup_hooks=not args.no_hooks,
+    )
+
+    print(f"\nScaffolded {stats['files_copied']} files, {stats['dirs_created']} directories.")
+
+    # Verify
+    findings = verify_placeholders(ml_dir)
+    if findings:
+        print(f"\nWarning: {len(findings)} unreplaced placeholder(s):", file=sys.stderr)
+        for filepath, line_num, placeholder in findings:
+            print(f"  {filepath}:{line_num} — {{{{{placeholder}}}}}", file=sys.stderr)
+        sys.exit(1)
+    else:
+        print("All placeholders replaced successfully.")
+
+    print(f"\nNext steps:")
+    print(f"  1. Add training data to {values['data_source']}")
+    print(f"  2. cd {ml_dir} && source .venv/bin/activate")
+    print(f"  3. python prepare.py")
+    print(f"  4. /turing:train  (or: python train.py > run.log 2>&1)")
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/show_environment.py

@@ -0,0 +1,92 @@
+"""Display the runtime environment from train_metadata.json.
+
+Shows python version, package versions, seeds, hardware, and config hash.
+Useful for debugging reproducibility issues or comparing environments
+across experiments.
+
+Usage:
+    python scripts/show_environment.py                    # Current experiment
+    python scripts/show_environment.py --file path/to/metadata.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+DEFAULT_METADATA_PATH = "train_metadata.json"
+
+
+def format_environment(metadata: dict) -> str:
+    """Format environment info for display."""
+    env = metadata.get("environment", {})
+    if not env:
+        return "No environment data recorded."
+
+    lines = ["Environment"]
+    lines.append("=" * 50)
+
+    # Platform
+    lines.append(f"  Python:    {env.get('python_version', '?')}")
+    lines.append(f"  Platform:  {env.get('platform', '?')}")
+    lines.append(f"  Machine:   {env.get('machine', '?')}")
+    lines.append(f"  OS:        {env.get('os', '?')}")
+
+    # GPU
+    gpu = env.get("gpu")
+    if gpu:
+        lines.append(f"  GPU:       {gpu.get('name', '?')} (CUDA {gpu.get('cuda_version', '?')})")
+        lines.append(f"  GPUs:      {gpu.get('device_count', '?')}")
+    else:
+        lines.append("  GPU:       none")
+
+    # Seeds
+    seeds = env.get("seeds", {})
+    lines.append("")
+    lines.append("Seeds")
+    lines.append("-" * 50)
+    for key, value in sorted(seeds.items()):
+        lines.append(f"  {key}: {value}")
+
+    # Packages
+    packages = env.get("packages", {})
+    if packages:
+        lines.append("")
+        lines.append("Packages")
+        lines.append("-" * 50)
+        for pkg, version in sorted(packages.items()):
+            lines.append(f"  {pkg}: {version}")
+
+    # Config hash
+    config_hash = env.get("config_hash")
+    if config_hash:
+        lines.append("")
+        lines.append(f"Config hash: {config_hash}")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Show experiment environment")
+    parser.add_argument(
+        "--file",
+        default=DEFAULT_METADATA_PATH,
+        help=f"Path to metadata file (default: {DEFAULT_METADATA_PATH})",
+    )
+    args = parser.parse_args()
+
+    path = Path(args.file)
+    if not path.exists():
+        print(f"No metadata file at {path}. Run training first.", file=sys.stderr)
+        sys.exit(1)
+
+    with open(path) as f:
+        metadata = json.load(f)
+
+    print(format_environment(metadata))
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/show_experiment_tree.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Experiment dependency tree visualizer.
+
+Displays the experiment lineage — which experiments inspired which —
+as a text tree. Makes the agent's reasoning chain visible to the human.
+
+Usage:
+    python scripts/show_experiment_tree.py [--log experiments/log.jsonl]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from scripts.turing_io import load_experiments
+
+
+def build_tree(experiments: list[dict]) -> dict[str, list[str]]:
+    """Build parent -> children mapping from experiments."""
+    children: dict[str, list[str]] = {}
+    for exp in experiments:
+        eid = exp.get("experiment_id", "")
+        parent = exp.get("parent_experiment")
+        if parent:
+            children.setdefault(parent, []).append(eid)
+        # Ensure every experiment appears as a key
+        children.setdefault(eid, [])
+    return children
+
+
+def find_roots(experiments: list[dict], children: dict[str, list[str]]) -> list[str]:
+    """Find experiments with no parent (tree roots)."""
+    all_ids = {e.get("experiment_id", "") for e in experiments}
+    child_ids = set()
+    for kids in children.values():
+        child_ids.update(kids)
+
+    # Roots are experiments that are not children of anything
+    # But also need to handle experiments whose parent doesn't exist in the log
+    roots = []
+    for exp in experiments:
+        eid = exp.get("experiment_id", "")
+        parent = exp.get("parent_experiment")
+        if parent is None or parent not in all_ids:
+            roots.append(eid)
+    return roots
+
+
+def format_tree(
+    node: str,
+    children: dict[str, list[str]],
+    experiments_by_id: dict[str, dict],
+    prefix: str = "",
+    is_last: bool = True,
+    metric_name: str = "accuracy",
+) -> list[str]:
+    """Recursively format a tree node and its children."""
+    exp = experiments_by_id.get(node, {})
+    status = exp.get("status", "?")
+    metric_val = exp.get("metrics", {}).get(metric_name)
+    model_type = exp.get("config", {}).get("model_type", "?")
+    desc = exp.get("description", "")[:40]
+
+    status_marker = "kept" if status == "kept" else "discarded" if status == "discarded" else status
+    metric_str = f"{metric_name}={metric_val:.4f}" if isinstance(metric_val, (int, float)) else ""
+    status_icon = "+" if status == "kept" else "-" if status == "discarded" else "?"
+
+    connector = "`-- " if is_last else "|-- "
+    line = f"{prefix}{connector}[{status_icon}] {node} ({model_type}, {metric_str}) {desc}"
+
+    lines = [line]
+
+    child_prefix = prefix + ("    " if is_last else "|   ")
+    kids = children.get(node, [])
+    for i, child in enumerate(kids):
+        child_lines = format_tree(
+            child, children, experiments_by_id, child_prefix,
+            is_last=(i == len(kids) - 1), metric_name=metric_name,
+        )
+        lines.extend(child_lines)
+
+    return lines
+
+
+def show_tree(log_path: str, metric_name: str = "accuracy") -> str:
+    """Generate the full experiment tree as text."""
+    experiments = load_experiments(log_path)
+    if not experiments:
+        return "No experiments logged yet."
+
+    experiments_by_id = {e.get("experiment_id", ""): e for e in experiments}
+    children = build_tree(experiments)
+    roots = find_roots(experiments, children)
+
+    if not roots:
+        return "No root experiments found."
+
+    lines = ["Experiment Tree", "=" * 60]
+    for i, root in enumerate(roots):
+        root_lines = format_tree(
+            root, children, experiments_by_id,
+            prefix="", is_last=(i == len(roots) - 1),
+            metric_name=metric_name,
+        )
+        lines.extend(root_lines)
+
+    # Summary
+    total = len(experiments)
+    kept = sum(1 for e in experiments if e.get("status") == "kept")
+    max_depth = _max_depth(roots, children)
+    lines.extend([
+        "",
+        f"Total: {total} experiments, {kept} kept, depth={max_depth}",
+        f"[+] = kept, [-] = discarded, [?] = other",
+    ])
+
+    return "\n".join(lines)
+
+
+def _max_depth(roots: list[str], children: dict[str, list[str]], depth: int = 1) -> int:
+    """Compute maximum tree depth."""
+    max_d = depth
+    for root in roots:
+        kids = children.get(root, [])
+        if kids:
+            max_d = max(max_d, _max_depth(kids, children, depth + 1))
+    return max_d
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Show experiment dependency tree")
+    parser.add_argument("--log", default="experiments/log.jsonl")
+    parser.add_argument("--metric", default="accuracy", help="Metric to display")
+    args = parser.parse_args()
+
+    print(show_tree(args.log, args.metric))
+
+
+if __name__ == "__main__":
+    main()