@cleocode/skills 2.0.0

package/skills/ct-grade/scripts/run_all.py

@@ -0,0 +1,287 @@
+"""
+ct-grade v3 — Master Pipeline Runner
+
+Orchestrates the full ct-grade v3 pipeline:
+  1. Audit log analysis
+  2. Scenario note (agents run separately via SKILL.md)
+  3. A/B test
+  4. Token tracker
+  5. Report generation
+  6. Grade review server
+
+Usage:
+    python scripts/run_all.py [--full] [--skip-ab] [--port 3118]
+                              [--project-dir .] [--stop] [--no-browser]
+"""
+
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import time
+import webbrowser
+from datetime import datetime
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).parent.resolve()
+SKILL_DIR = SCRIPT_DIR.parent  # packages/ct-skills/skills/ct-grade/
+
+
+# ---------------------------------------------------------------------------
+# Server lifecycle
+# ---------------------------------------------------------------------------
+
+def stop_server(project_dir: str) -> None:
+    pid_file = Path(project_dir) / ".ct-grade-server.pid"
+    if not pid_file.exists():
+        print("No server PID file found. Server may not be running.")
+        return
+    pid = int(pid_file.read_text().strip())
+    try:
+        os.kill(pid, signal.SIGTERM)
+        pid_file.unlink()
+        print(f"Server stopped (PID {pid})")
+    except ProcessLookupError:
+        print(f"Process {pid} not found (already stopped)")
+        pid_file.unlink()
+    except Exception as e:
+        print(f"Error stopping server: {e}")
+
+
+def start_server(project_dir: str, output_dir: Path, port: int) -> int | None:
+    """Step 6: Start grade review server in background."""
+    viewer_script = SKILL_DIR / "eval-viewer" / "generate_grade_review.py"
+    if not viewer_script.exists():
+        print(
+            f"  WARNING: Viewer script not found at {viewer_script}. Skipping server start."
+        )
+        return None
+
+    print(f"\n[6/6] Starting Grade Review server on port {port}...")
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            str(viewer_script),
+            str(project_dir),
+            "--port",
+            str(port),
+            "--no-browser",
+            "--ab-dir",
+            str(output_dir),
+        ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True,  # detach from parent
+    )
+    pid_file = Path(project_dir) / ".ct-grade-server.pid"
+    pid_file.write_text(str(proc.pid))
+    return proc.pid
+
+
+# ---------------------------------------------------------------------------
+# Pipeline steps
+# ---------------------------------------------------------------------------
+
+def step_audit_analyze(project_dir: str, output_dir: Path) -> None:
+    """Step 1: Extract real per-op stats from tasks.db audit_log."""
+    print("\n[1/6] Analyzing audit log...")
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT_DIR / "audit_analyzer.py"),
+            "--project-dir",
+            str(project_dir),
+            "--output-dir",
+            str(output_dir),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        print(f"  WARNING: audit_analyzer failed: {result.stderr[:200]}")
+    else:
+        print("  Done.")
+
+
+def step_scenario_note(full_mode: bool) -> None:
+    """Step 2: Print info about scenario agents (not spawned here)."""
+    print("\n[2/6] Scenario runners:")
+    if full_mode:
+        print(
+            "  Full mode: S1–S5 scenarios are delegated to ct-grade scenario-runner agents."
+        )
+    else:
+        print(
+            "  Fast mode: S4+S5 scenarios are delegated to ct-grade scenario-runner agents."
+        )
+    print(
+        "  Run scenarios separately via SKILL.md orchestration (skill invocation)."
+    )
+
+
+def step_ab_test(project_dir: str, output_dir: Path, full_mode: bool = False) -> None:
+    """Step 3: Run A/B test (smoke in fast mode, parity in full mode)."""
+    test_set = "parity" if full_mode else "smoke"
+    runs = "3"
+    print(f"\n[3/6] Running A/B test (--test-set {test_set}, --runs {runs})...")
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT_DIR / "run_ab_test.py"),
+            "--test-set",
+            test_set,
+            "--runs",
+            runs,
+            "--project-dir",
+            str(project_dir),
+            "--output-dir",
+            str(output_dir / "ab-results"),
+        ],
+        capture_output=False,  # show live output
+        text=True,
+    )
+    if result.returncode != 0:
+        print("  WARNING: A/B test completed with errors.")
+
+
+def step_token_tracker(project_dir: str, output_dir: Path, grades_file: Path) -> None:
+    """Step 4: Enrich grade data with token estimates."""
+    print("\n[4/6] Running token tracker...")
+    args = [
+        sys.executable,
+        str(SCRIPT_DIR / "token_tracker.py"),
+        "--project-dir",
+        str(project_dir),
+        "--output",
+        str(output_dir / "token-summary.json"),
+    ]
+    if grades_file.exists():
+        args += ["--grades-file", str(grades_file)]
+    result = subprocess.run(args, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(f"  WARNING: token_tracker failed: {result.stderr[:200]}")
+    else:
+        print("  Done.")
+
+
+def step_generate_report(output_dir: Path) -> None:
+    """Step 5: Generate markdown report."""
+    print("\n[5/6] Generating report...")
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT_DIR / "generate_report.py"),
+            "--run-dir",
+            str(output_dir),
+            "--mode",
+            "ab",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        print(f"  WARNING: generate_report failed: {result.stderr[:200]}")
+    else:
+        print(f"  Done. Report: {output_dir / 'report.md'}")
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="ct-grade v3 — Master Pipeline Runner",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--full",
+        action="store_true",
+        help="Run all 5 scenarios (S1–S5) + parity A/B tests. Default: fast mode (S4+S5 + smoke A/B).",
+    )
+    parser.add_argument(
+        "--skip-ab",
+        action="store_true",
+        help="Skip the A/B test step.",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=3118,
+        help="Port for grade review server (default: 3118).",
+    )
+    parser.add_argument(
+        "--project-dir",
+        default=".",
+        help="CLEO project root (default: current directory).",
+    )
+    parser.add_argument(
+        "--stop",
+        action="store_true",
+        help="Kill existing server from .ct-grade-server.pid and exit.",
+    )
+    parser.add_argument(
+        "--no-browser",
+        action="store_true",
+        help="Don't auto-open browser after starting server.",
+    )
+    args = parser.parse_args()
+
+    project_dir = str(Path(args.project_dir).resolve())
+
+    # --stop: kill running server and exit
+    if args.stop:
+        stop_server(project_dir)
+        return
+
+    # Set up timestamped output directory
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    output_dir = Path(project_dir) / "ab-results" / timestamp
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    mode_label = "Full (S1–S5 + parity A/B)" if args.full else "Fast (S4+S5 + smoke A/B)"
+
+    # Print header
+    print("╔═══════════════════════════════════════╗")
+    print("║   ct-grade v3 — Grade Review System   ║")
+    print("╚═══════════════════════════════════════╝")
+    print()
+    print(f"  Mode     : {mode_label}")
+    print(f"  Output   : ab-results/{timestamp}")
+    print(f"  Project  : {project_dir}")
+
+    grades_file = Path(project_dir) / ".cleo" / "metrics" / "GRADES.jsonl"
+
+    # Step 1: Audit analysis
+    step_audit_analyze(project_dir, output_dir)
+
+    # Step 2: Scenario note
+    step_scenario_note(args.full)
+
+    # Step 3: A/B test (optional)
+    if not args.skip_ab:
+        step_ab_test(project_dir, output_dir, full_mode=args.full)
+    else:
+        print("\n[3/6] Skipping A/B test (--skip-ab).")
+
+    # Step 4: Token tracker
+    step_token_tracker(project_dir, output_dir, grades_file)
+
+    # Step 5: Report generation
+    step_generate_report(output_dir)
+
+    # Step 6: Start server
+    pid = start_server(project_dir, output_dir, args.port)
+
+    # Open browser after brief pause for server to bind
+    if not args.no_browser and pid is not None:
+        time.sleep(0.5)
+        webbrowser.open(f"http://localhost:{args.port}")
+
+    print(f"\nGrade Review live at http://localhost:{args.port}")
+    print("Stop with: python scripts/run_all.py --stop")
+
+
+if __name__ == "__main__":
+    main()

package/skills/ct-grade/scripts/setup_run.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""
+setup_run.py — Set up an A/B test run directory and print the execution plan.
+
+Usage:
+    python setup_run.py --mode scenario --scenario s4 --interface both --runs 3 --output-dir ./ab_results/run-001
+
+Outputs:
+    - Creates run directory structure
+    - Writes run-manifest.json
+    - Prints step-by-step execution plan for Claude to follow
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+VALID_MODES = ["scenario", "ab", "blind"]
+VALID_SCENARIOS = ["s1", "s2", "s3", "s4", "s5", "all"]
+VALID_INTERFACES = ["mcp", "cli", "both"]
+
+SCENARIO_LABELS = {
+    "s1": "Fresh Discovery",
+    "s2": "Task Creation Hygiene",
+    "s3": "Error Recovery",
+    "s4": "Full Lifecycle",
+    "s5": "Multi-Domain Analysis",
+}
+
+DEFAULT_DOMAINS = ["tasks", "session"]
+
+
+def expand_scenarios(scenario_arg):
+    if scenario_arg == "all":
+        return ["s1", "s2", "s3", "s4", "s5"]
+    return [s.strip() for s in scenario_arg.split(",") if s.strip() in SCENARIO_LABELS]
+
+
+def expand_interfaces(interface_arg):
+    if interface_arg == "both":
+        return ["mcp", "cli"]
+    return [interface_arg]
+
+
+def create_dir(path):
+    os.makedirs(path, exist_ok=True)
+    return path
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Set up a ct-grade A/B test run")
+    parser.add_argument("--mode", default="scenario", choices=VALID_MODES)
+    parser.add_argument("--scenario", default="all")
+    parser.add_argument("--interface", default="both", choices=VALID_INTERFACES)
+    parser.add_argument("--domains", default="tasks,session")
+    parser.add_argument("--runs", type=int, default=3)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument("--project-dir", default=".")
+    args = parser.parse_args()
+
+    scenarios = expand_scenarios(args.scenario)
+    interfaces = expand_interfaces(args.interface)
+    domains = [d.strip() for d in args.domains.split(",")]
+
+    if not scenarios:
+        print(f"ERROR: No valid scenarios in '{args.scenario}'. Use: {', '.join(VALID_SCENARIOS)}", file=sys.stderr)
+        sys.exit(1)
+
+    run_dir = args.output_dir
+    create_dir(run_dir)
+
+    # For ab/blind mode, each domain is a "slot"
+    slots = scenarios if args.mode == "scenario" else domains
+
+    # Create directory structure
+    for slot in slots:
+        for iface in interfaces:
+            arm_label = "arm-A" if iface == interfaces[0] else "arm-B"
+            for run in range(1, args.runs + 1):
+                slot_dir = os.path.join(run_dir, slot, f"run-{run:02d}", arm_label)
+                create_dir(slot_dir)
+                # Create placeholder timing.json
+                timing = {
+                    "arm": arm_label,
+                    "interface": iface,
+                    "slot": slot,
+                    "run": run,
+                    "executor_start": None,
+                    "executor_end": None,
+                    "executor_duration_seconds": None,
+                    "total_tokens": None,
+                    "duration_ms": None,
+                }
+                timing_path = os.path.join(slot_dir, "timing.json")
+                with open(timing_path, "w") as f:
+                    json.dump(timing, f, indent=2)
+
+    # Write run-manifest.json
+    manifest = {
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "mode": args.mode,
+        "scenarios": scenarios,
+        "interfaces": interfaces,
+        "domains": domains,
+        "runs_per_configuration": args.runs,
+        "project_dir": os.path.abspath(args.project_dir),
+        "run_dir": os.path.abspath(run_dir),
+        "arms": {
+            "A": {"interface": interfaces[0], "label": f"{interfaces[0].upper()} interface"},
+            "B": {"interface": interfaces[1] if len(interfaces) > 1 else interfaces[0],
+                  "label": f"{interfaces[-1].upper()} interface"},
+        },
+        "slots": slots,
+        "status": "setup_complete",
+    }
+    manifest_path = os.path.join(run_dir, "run-manifest.json")
+    with open(manifest_path, "w") as f:
+        json.dump(manifest, f, indent=2)
+
+    # Print execution plan
+    print(f"\n{'='*60}")
+    print(f"ct-grade A/B Run Setup Complete")
+    print(f"{'='*60}")
+    print(f"Mode:       {args.mode}")
+    print(f"Scenarios:  {', '.join(scenarios)}")
+    print(f"Interfaces: {', '.join(interfaces)}")
+    print(f"Runs each:  {args.runs}")
+    print(f"Output:     {os.path.abspath(run_dir)}")
+    print(f"{'='*60}\n")
+
+    print("EXECUTION PLAN\n")
+    print("Spawn each arm as a parallel Agent task in the same turn.\n")
+
+    step = 1
+    for slot in slots:
+        slot_label = SCENARIO_LABELS.get(slot, slot)
+        print(f"## Slot: {slot} — {slot_label}\n")
+        for run in range(1, args.runs + 1):
+            for idx, iface in enumerate(interfaces):
+                arm_label = "arm-A" if idx == 0 else "arm-B"
+                arm_dir = os.path.join(os.path.abspath(run_dir), slot, f"run-{run:02d}", arm_label)
+                print(f"Step {step}: Spawn Agent — {arm_label} ({iface}) | slot={slot} | run={run}")
+                print(f"  Agent file:  agents/scenario-runner.md")
+                print(f"  SCENARIO:    {slot}")
+                print(f"  INTERFACE:   {iface}")
+                print(f"  OUTPUT_DIR:  {arm_dir}")
+                print(f"  RUN_NUMBER:  {run}")
+                print(f"  CRITICAL: Capture total_tokens + duration_ms from task notification")
+                print(f"            and update {arm_dir}/timing.json immediately.\n")
+                step += 1
+
+            # After both arms complete for this run
+            comp_dir = os.path.join(os.path.abspath(run_dir), slot, f"run-{run:02d}")
+            print(f"Step {step}: Spawn blind-comparator Agent")
+            print(f"  Agent file:  agents/blind-comparator.md")
+            print(f"  OUTPUT_A:    {comp_dir}/arm-A/")
+            print(f"  OUTPUT_B:    {comp_dir}/arm-B/")
+            print(f"  SCENARIO:    {slot}")
+            print(f"  OUTPUT_PATH: {comp_dir}/comparison.json\n")
+            step += 1
+
+    print(f"Step {step}: Aggregate token data")
+    print(f"  python scripts/token_tracker.py --run-dir {os.path.abspath(run_dir)}\n")
+    step += 1
+
+    print(f"Step {step}: Generate final report")
+    print(f"  python scripts/generate_report.py --run-dir {os.path.abspath(run_dir)} --mode {args.mode}\n")
+    step += 1
+
+    print(f"Step {step}: (Optional) Spawn analysis-reporter Agent for deep synthesis")
+    print(f"  Agent file:  agents/analysis-reporter.md")
+    print(f"  RUN_DIR:     {os.path.abspath(run_dir)}\n")
+
+    print(f"{'='*60}")
+    print(f"Manifest: {manifest_path}")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()