claude-turing 1.0.0

package/templates/scripts/synthesize_decision.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+"""Decision packet synthesis for the autoresearch pipeline.
+
+After each experiment, synthesizes a structured verdict combining:
+- Run outcome and metrics
+- Comparison to current champion
+- A recommended next action
+
+Actions: promote, branch_followup, replicate, abandon, fix_and_retry,
+investigate_crash.
+
+Inspired by pauldebdeep9/autoresearch MemoryLab decision packets.
+
+Usage:
+    python scripts/synthesize_decision.py \\
+        --experiment exp-005 \\
+        --log experiments/log.jsonl \\
+        --config config.yaml
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+PROMISING_DELTA = 0.005  # 0.5% relative improvement = promising
+
+
+def load_experiment(log_path: str, experiment_id: str) -> dict | None:
+    """Load a specific experiment from the JSONL log."""
+    path = Path(log_path)
+    if not path.exists():
+        return None
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+                if entry.get("experiment_id") == experiment_id:
+                    return entry
+            except json.JSONDecodeError:
+                continue
+    return None
+
+
+def find_champion(log_path: str, metric: str, lower_is_better: bool) -> dict | None:
+    """Find the current best (champion) experiment."""
+    path = Path(log_path)
+    if not path.exists():
+        return None
+    best = None
+    best_val = float("inf") if lower_is_better else float("-inf")
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+                if entry.get("status") != "kept":
+                    continue
+                val = entry.get("metrics", {}).get(metric)
+                if val is None:
+                    continue
+                if (lower_is_better and val < best_val) or (not lower_is_better and val > best_val):
+                    best_val = val
+                    best = entry
+            except json.JSONDecodeError:
+                continue
+    return best
+
+
+def compute_delta(current: float, champion: float, lower_is_better: bool) -> float:
+    """Compute relative improvement over champion."""
+    if champion == 0:
+        return 1.0 if current != 0 else 0.0
+    if lower_is_better:
+        return (champion - current) / abs(champion)
+    else:
+        return (current - champion) / abs(champion)
+
+
+def classify_outcome(
+    experiment: dict,
+    champion: dict | None,
+    metric: str,
+    lower_is_better: bool,
+) -> tuple[str, float | None]:
+    """Classify the experiment outcome.
+
+    Returns (outcome, delta_to_champion).
+    Outcomes: new_champion, marginal_improvement, lateral, regression, crash.
+    """
+    status = experiment.get("status", "")
+    if status == "crash":
+        return "crash", None
+
+    current_val = experiment.get("metrics", {}).get(metric)
+    if current_val is None:
+        return "crash", None
+
+    if champion is None:
+        if status == "kept":
+            return "new_champion", None
+        return "regression", None
+
+    champion_val = champion.get("metrics", {}).get(metric)
+    if champion_val is None:
+        return "new_champion" if status == "kept" else "regression", None
+
+    delta = compute_delta(current_val, champion_val, lower_is_better)
+
+    if status == "kept" and delta > PROMISING_DELTA:
+        return "new_champion", delta
+    elif status == "kept" and delta > 0:
+        return "marginal_improvement", delta
+    elif abs(delta) <= PROMISING_DELTA:
+        return "lateral", delta
+    else:
+        return "regression", delta
+
+
+def recommend_action(
+    outcome: str,
+    experiment: dict,
+) -> tuple[str, str]:
+    """Recommend a next action based on the outcome.
+
+    Returns (action, rationale).
+    """
+    actions = {
+        "new_champion": (
+            "promote",
+            "New best result — update champion, consider replicating to confirm stability",
+        ),
+        "marginal_improvement": (
+            "branch_followup",
+            "Slight improvement — explore variations of this approach",
+        ),
+        "lateral": (
+            "abandon",
+            "No meaningful change — this direction is not productive",
+        ),
+        "regression": (
+            "abandon",
+            "Performance decreased — discard and try a different approach",
+        ),
+        "crash": (
+            "fix_and_retry" if experiment.get("description", "") else "investigate_crash",
+            "Experiment crashed — check error logs and retry with fixes" if experiment.get("description") else "Experiment crashed — investigate the cause before retrying",
+        ),
+    }
+    return actions.get(outcome, ("investigate_crash", "Unknown outcome"))
+
+
+def synthesize_packet(
+    experiment: dict,
+    champion: dict | None,
+    metric: str,
+    lower_is_better: bool,
+) -> dict:
+    """Synthesize a complete decision packet for an experiment.
+
+    Returns dict with: experiment_id, outcome, action, rationale,
+    delta, current_metric, champion_metric, champion_id.
+    """
+    outcome, delta = classify_outcome(experiment, champion, metric, lower_is_better)
+    action, rationale = recommend_action(outcome, experiment)
+
+    current_val = experiment.get("metrics", {}).get(metric)
+    champion_val = champion.get("metrics", {}).get(metric) if champion else None
+
+    packet = {
+        "experiment_id": experiment.get("experiment_id", "?"),
+        "outcome": outcome,
+        "action": action,
+        "rationale": rationale,
+        "delta": round(delta, 6) if delta is not None else None,
+        "current_metric": current_val,
+        "champion_metric": champion_val,
+        "champion_id": champion.get("experiment_id", "?") if champion else None,
+        "status": experiment.get("status", "?"),
+        "description": experiment.get("description", ""),
+        "family": experiment.get("family"),
+        "hypothesis_id": experiment.get("hypothesis_id"),
+    }
+
+    return packet
+
+
+def auto_queue_followup(
+    packet: dict,
+    hypotheses_path: str = "hypotheses.yaml",
+) -> str | None:
+    """Auto-queue a follow-up hypothesis based on the decision packet.
+
+    Actions that trigger auto-queuing:
+    - branch_followup: queue a follow-up hypothesis as agent/medium priority
+    - fix_and_retry: queue a retry hypothesis as agent/high priority
+
+    Returns the new hypothesis ID if queued, or None if no action taken.
+    """
+    action = packet.get("action", "")
+    exp_id = packet.get("experiment_id", "?")
+    description = packet.get("description", "")
+    family = packet.get("family")
+
+    # Import here to avoid circular dependency at module level
+    from scripts.manage_hypotheses import add_hypothesis
+
+    if action == "branch_followup":
+        desc = f"Follow up on {exp_id}: explore variations of '{description[:60]}'"
+        return add_hypothesis(
+            queue_path=hypotheses_path,
+            description=desc,
+            source="agent",
+            priority="medium",
+            parent_experiment=exp_id,
+        )
+    elif action == "fix_and_retry":
+        desc = f"Retry {exp_id} with fixes: '{description[:60]}' crashed — investigate and fix"
+        return add_hypothesis(
+            queue_path=hypotheses_path,
+            description=desc,
+            source="agent",
+            priority="high",
+            parent_experiment=exp_id,
+        )
+
+    return None
+
+
+def format_packet(packet: dict, metric_name: str) -> str:
+    """Format a decision packet for display."""
+    lines = [
+        f"Decision Packet: {packet['experiment_id']}",
+        f"  Outcome: {packet['outcome']}",
+        f"  Action: {packet['action']}",
+        f"  Rationale: {packet['rationale']}",
+        f"  {metric_name}: {packet['current_metric']} (champion: {packet['champion_metric']})",
+    ]
+    if packet["delta"] is not None:
+        lines.append(f"  Delta: {packet['delta']:+.4f} relative")
+    if packet["family"]:
+        lines.append(f"  Family: {packet['family']}")
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Synthesize decision packet")
+    parser.add_argument("--experiment", required=True, help="Experiment ID")
+    parser.add_argument("--log", default="experiments/log.jsonl")
+    parser.add_argument("--config", default="config.yaml")
+    parser.add_argument("--hypotheses", default="hypotheses.yaml", help="Hypothesis queue path")
+    parser.add_argument("--auto-queue", action="store_true", help="Auto-queue follow-up hypotheses")
+    parser.add_argument("--json", action="store_true", help="Output as JSON")
+    args = parser.parse_args()
+
+    config = {}
+    if Path(args.config).exists():
+        with open(args.config) as f:
+            config = yaml.safe_load(f) or {}
+
+    eval_cfg = config.get("evaluation", {})
+    metric = eval_cfg.get("primary_metric", "accuracy")
+    lower_is_better = eval_cfg.get("lower_is_better", False)
+
+    experiment = load_experiment(args.log, args.experiment)
+    if not experiment:
+        print(f"Experiment {args.experiment} not found.", file=sys.stderr)
+        sys.exit(1)
+
+    champion = find_champion(args.log, metric, lower_is_better)
+    # Don't compare champion to itself
+    if champion and champion.get("experiment_id") == experiment.get("experiment_id"):
+        champion = None
+
+    packet = synthesize_packet(experiment, champion, metric, lower_is_better)
+
+    if args.json:
+        print(json.dumps(packet, indent=2))
+    else:
+        print(format_packet(packet, metric))
+
+    # Auto-queue follow-up hypotheses if requested
+    if args.auto_queue:
+        hyp_id = auto_queue_followup(packet, args.hypotheses)
+        if hyp_id:
+            print(f"\n  Auto-queued: {hyp_id} ({packet['action']})")
+
+
+if __name__ == "__main__":
+    main()

package/templates/scripts/turing_io.py

@@ -0,0 +1,76 @@
+"""Shared data loading functions for the autoresearch pipeline.
+
+Consolidates the duplicated load_experiments, load_config, and
+load_hypotheses functions that were copy-pasted across 8+ scripts.
+Every script that reads experiment logs, config, or hypothesis
+queues should import from here.
+
+Usage:
+    from scripts.turing_io import load_experiments, load_config, load_hypotheses
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import yaml
+
+
+def load_experiments(log_path: str) -> list[dict]:
+    """Load experiments from JSONL log.
+
+    Args:
+        log_path: Path to the experiments JSONL file.
+
+    Returns:
+        List of experiment dicts. Empty list if file missing or empty.
+        Malformed lines are silently skipped.
+    """
+    path = Path(log_path)
+    if not path.exists():
+        return []
+    experiments = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                try:
+                    experiments.append(json.loads(line))
+                except json.JSONDecodeError:
+                    continue
+    return experiments
+
+
+def load_config(path: str = "config.yaml") -> dict:
+    """Load YAML config with fallback to empty dict.
+
+    Args:
+        path: Path to config YAML file. Defaults to "config.yaml".
+
+    Returns:
+        Parsed config dict, or empty dict if file missing or invalid.
+    """
+    p = Path(path)
+    if not p.exists():
+        return {}
+    with open(p) as f:
+        return yaml.safe_load(f) or {}
+
+
+def load_hypotheses(queue_path: str) -> list[dict]:
+    """Load hypothesis queue from YAML file.
+
+    Args:
+        queue_path: Path to the hypotheses YAML file.
+
+    Returns:
+        List of hypothesis dicts. Empty list if file missing, empty,
+        or not a list.
+    """
+    path = Path(queue_path)
+    if not path.exists() or path.stat().st_size == 0:
+        return []
+    with open(path) as f:
+        data = yaml.safe_load(f)
+    return data if isinstance(data, list) else []

package/templates/scripts/update_state.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+"""Structured experiment state manager for the autoresearch pipeline.
+
+Replaces free-text MEMORY.md with a validated experiment_state.yaml
+that the agent reads and writes in a schema-enforced format. The state
+is machine-readable and programmatically queryable.
+
+MEMORY.md is kept as a human-readable companion, generated from the
+structured state.
+
+Usage:
+    python scripts/update_state.py init                    # Create empty state
+    python scripts/update_state.py show                    # Display current state
+    python scripts/update_state.py set-best <exp-id> <metrics_json>
+    python scripts/update_state.py add-observation <text> [--exp <exp-id>]
+    python scripts/update_state.py add-failure <text> --exp <exp-id> --reason <text>
+    python scripts/update_state.py add-direction <text> [--priority high|medium|low]
+    python scripts/update_state.py log-session <n_experiments> <best_metric>
+    python scripts/update_state.py generate-memory         # Write MEMORY.md from state
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+
+DEFAULT_STATE_PATH = "experiment_state.yaml"
+DEFAULT_MEMORY_PATH = ".claude/agent-memory/ml-researcher/MEMORY.md"
+
+EMPTY_STATE = {
+    "goal": "",
+    "primary_metric": "",
+    "metric_direction": "",
+    "best_result": None,
+    "observations": [],
+    "failed_approaches": [],
+    "promising_directions": [],
+    "session_history": [],
+}
+
+REQUIRED_KEYS = set(EMPTY_STATE.keys())
+
+
+def load_state(path: str) -> dict:
+    """Load experiment state from YAML, validating required keys."""
+    p = Path(path)
+    if not p.exists() or p.stat().st_size == 0:
+        return copy.deepcopy(EMPTY_STATE)
+    with open(p) as f:
+        state = yaml.safe_load(f)
+    if not isinstance(state, dict):
+        return copy.deepcopy(EMPTY_STATE)
+    # Ensure all required keys exist
+    for key in REQUIRED_KEYS:
+        if key not in state:
+            state[key] = copy.deepcopy(EMPTY_STATE[key])
+    return state
+
+
+def save_state(path: str, state: dict) -> None:
+    """Save experiment state to YAML."""
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    with open(p, "w") as f:
+        yaml.dump(state, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
+
+
+def validate_state(state: dict) -> list[str]:
+    """Validate state against schema. Returns list of error messages."""
+    errors = []
+    for key in REQUIRED_KEYS:
+        if key not in state:
+            errors.append(f"Missing required key: {key}")
+    if state.get("best_result") is not None:
+        br = state["best_result"]
+        if not isinstance(br, dict):
+            errors.append("best_result must be a dict or null")
+        elif "experiment_id" not in br or "metrics" not in br:
+            errors.append("best_result must have experiment_id and metrics")
+    if not isinstance(state.get("observations", []), list):
+        errors.append("observations must be a list")
+    if not isinstance(state.get("failed_approaches", []), list):
+        errors.append("failed_approaches must be a list")
+    if not isinstance(state.get("promising_directions", []), list):
+        errors.append("promising_directions must be a list")
+    if not isinstance(state.get("session_history", []), list):
+        errors.append("session_history must be a list")
+    return errors
+
+
+def set_best(state: dict, experiment_id: str, metrics: dict) -> dict:
+    """Update the best result in state."""
+    state["best_result"] = {
+        "experiment_id": experiment_id,
+        "metrics": metrics,
+        "updated_at": datetime.now(timezone.utc).isoformat(),
+    }
+    return state
+
+
+def add_observation(state: dict, text: str, experiment_id: str | None = None) -> dict:
+    """Add an observation to state."""
+    entry = {
+        "text": text,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+    if experiment_id:
+        entry["experiment_id"] = experiment_id
+    state["observations"].append(entry)
+    return state
+
+
+def add_failure(state: dict, description: str, experiment_id: str, reason: str) -> dict:
+    """Record a failed approach."""
+    state["failed_approaches"].append({
+        "description": description,
+        "experiment_id": experiment_id,
+        "reason": reason,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    })
+    return state
+
+
+def add_direction(state: dict, description: str, priority: str = "medium") -> dict:
+    """Add a promising direction."""
+    state["promising_directions"].append({
+        "description": description,
+        "priority": priority,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    })
+    return state
+
+
+def log_session(state: dict, n_experiments: int, best_metric: float | None) -> dict:
+    """Log a training session summary."""
+    state["session_history"].append({
+        "date": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
+        "experiments_run": n_experiments,
+        "best_metric": best_metric,
+    })
+    return state
+
+
+def generate_memory(state: dict, goal: str = "", metric: str = "", direction: str = "") -> str:
+    """Generate MEMORY.md content from structured state."""
+    goal = goal or state.get("goal", "")
+    metric = metric or state.get("primary_metric", "")
+    direction = direction or state.get("metric_direction", "")
+
+    lines = [
+        "# ML Researcher Memory",
+        "",
+        "## Goal",
+        "",
+        goal or "(not set)",
+        "",
+        f"Primary metric: {metric} ({direction} is better).",
+        "",
+        "## Best Result",
+        "",
+    ]
+
+    br = state.get("best_result")
+    if br:
+        metrics_str = ", ".join(f"{k}={v}" for k, v in br.get("metrics", {}).items())
+        lines.append(f"Experiment {br.get('experiment_id', '?')}: {metrics_str}")
+    else:
+        lines.append("No experiments completed yet.")
+
+    lines.extend(["", "## Observations", ""])
+    for obs in state.get("observations", [])[-10:]:  # Last 10
+        exp_tag = f" ({obs['experiment_id']})" if obs.get("experiment_id") else ""
+        lines.append(f"- {obs['text']}{exp_tag}")
+    if not state.get("observations"):
+        lines.append("(none yet)")
+
+    lines.extend(["", "## Failed Approaches", ""])
+    for fail in state.get("failed_approaches", []):
+        lines.append(f"- {fail['description']} ({fail['experiment_id']}): {fail['reason']}")
+    if not state.get("failed_approaches"):
+        lines.append("(none yet)")
+
+    lines.extend(["", "## Promising Directions", ""])
+    for d in state.get("promising_directions", []):
+        priority_tag = f" [{d['priority']}]" if d.get("priority") != "medium" else ""
+        lines.append(f"- {d['description']}{priority_tag}")
+    if not state.get("promising_directions"):
+        lines.append("(none yet)")
+
+    lines.extend(["", "## Session History", ""])
+    if state.get("session_history"):
+        lines.append("| Session | Experiments | Best Metric | Notes |")
+        lines.append("|---------|-------------|-------------|-------|")
+        for s in state["session_history"]:
+            bm = f"{s['best_metric']:.4f}" if s.get("best_metric") is not None else "N/A"
+            lines.append(f"| {s.get('date', '?')} | {s.get('experiments_run', 0)} | {bm} | |")
+    else:
+        lines.append("(no sessions yet)")
+
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(description="Manage experiment state")
+    parser.add_argument("--state", default=DEFAULT_STATE_PATH)
+    subparsers = parser.add_subparsers(dest="command")
+
+    subparsers.add_parser("init", help="Create empty state")
+    subparsers.add_parser("show", help="Display current state")
+    subparsers.add_parser("validate", help="Validate state schema")
+
+    sb = subparsers.add_parser("set-best")
+    sb.add_argument("experiment_id")
+    sb.add_argument("metrics_json")
+
+    obs = subparsers.add_parser("add-observation")
+    obs.add_argument("text")
+    obs.add_argument("--exp", default=None)
+
+    fail = subparsers.add_parser("add-failure")
+    fail.add_argument("text")
+    fail.add_argument("--exp", required=True)
+    fail.add_argument("--reason", required=True)
+
+    d = subparsers.add_parser("add-direction")
+    d.add_argument("text")
+    d.add_argument("--priority", default="medium", choices=["high", "medium", "low"])
+
+    sess = subparsers.add_parser("log-session")
+    sess.add_argument("n_experiments", type=int)
+    sess.add_argument("best_metric", type=float, nargs="?", default=None)
+
+    subparsers.add_parser("generate-memory", help="Write MEMORY.md from state")
+
+    args = parser.parse_args()
+    state = load_state(args.state)
+
+    if args.command == "init":
+        save_state(args.state, EMPTY_STATE)
+        print(f"Initialized empty state at {args.state}")
+
+    elif args.command == "show":
+        print(yaml.dump(state, default_flow_style=False, sort_keys=False))
+
+    elif args.command == "validate":
+        errors = validate_state(state)
+        if errors:
+            for e in errors:
+                print(f"  ERROR: {e}", file=sys.stderr)
+            sys.exit(1)
+        print("State is valid.")
+
+    elif args.command == "set-best":
+        metrics = json.loads(args.metrics_json)
+        set_best(state, args.experiment_id, metrics)
+        save_state(args.state, state)
+        print(f"Best result set to {args.experiment_id}")
+
+    elif args.command == "add-observation":
+        add_observation(state, args.text, args.exp)
+        save_state(args.state, state)
+        print("Observation added.")
+
+    elif args.command == "add-failure":
+        add_failure(state, args.text, args.exp, args.reason)
+        save_state(args.state, state)
+        print("Failed approach recorded.")
+
+    elif args.command == "add-direction":
+        add_direction(state, args.text, args.priority)
+        save_state(args.state, state)
+        print("Promising direction added.")
+
+    elif args.command == "log-session":
+        log_session(state, args.n_experiments, args.best_metric)
+        save_state(args.state, state)
+        print("Session logged.")
+
+    elif args.command == "generate-memory":
+        memory_content = generate_memory(state)
+        print(memory_content)
+
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()