@cleocode/cleo 2026.3.20 → 2026.3.22

package/packages/ct-skills/skills/ct-grade-v2-1/scripts/token_tracker.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+token_tracker.py — Aggregate token usage stats from a completed A/B run.
+
+Usage:
+    python token_tracker.py --run-dir ./ab_results/run-001
+
+Reads all timing.json files in the run directory and produces token-summary.json
+with per-arm statistics.
+
+Output: <run-dir>/token-summary.json
+"""
+
+import argparse
+import json
+import os
+import sys
+import math
+from pathlib import Path
+
+
+def find_timing_files(run_dir):
+    """Find all timing.json files under run_dir."""
+    return list(Path(run_dir).rglob("timing.json"))
+
+
+def load_timing(path):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"  WARN: Could not read {path}: {e}", file=sys.stderr)
+        return None
+
+
+def mean(values):
+    return sum(values) / len(values) if values else 0
+
+
+def stddev(values):
+    if len(values) < 2:
+        return 0
+    m = mean(values)
+    return math.sqrt(sum((x - m) ** 2 for x in values) / (len(values) - 1))
+
+
+def stats(values):
+    if not values:
+        return {"mean": None, "stddev": None, "min": None, "max": None, "count": 0}
+    return {
+        "mean": round(mean(values), 1),
+        "stddev": round(stddev(values), 1),
+        "min": min(values),
+        "max": max(values),
+        "count": len(values),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Aggregate token stats from ct-grade A/B run")
+    parser.add_argument("--run-dir", required=True)
+    parser.add_argument("--output", default=None, help="Output path (default: <run-dir>/token-summary.json)")
+    args = parser.parse_args()
+
+    run_dir = args.run_dir
+    if not os.path.isdir(run_dir):
+        print(f"ERROR: Run dir not found: {run_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    timing_files = find_timing_files(run_dir)
+    if not timing_files:
+        print(f"ERROR: No timing.json files found in {run_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Found {len(timing_files)} timing.json files")
+
+    # Group by arm
+    by_arm = {}
+    by_interface = {}
+    missing_tokens = []
+
+    for tpath in timing_files:
+        data = load_timing(tpath)
+        if data is None:
+            continue
+
+        arm = data.get("arm", "unknown")
+        iface = data.get("interface", "unknown")
+        tokens = data.get("total_tokens")
+        duration = data.get("duration_ms")
+
+        if arm not in by_arm:
+            by_arm[arm] = {"tokens": [], "duration_ms": [], "interface": iface, "files": []}
+        if iface not in by_interface:
+            by_interface[iface] = {"tokens": [], "duration_ms": [], "files": []}
+
+        by_arm[arm]["files"].append(str(tpath))
+
+        if tokens is not None:
+            by_arm[arm]["tokens"].append(tokens)
+            by_interface[iface]["tokens"].append(tokens)
+        else:
+            missing_tokens.append(str(tpath))
+
+        if duration is not None:
+            by_arm[arm]["duration_ms"].append(duration)
+            by_interface[iface]["duration_ms"].append(duration)
+
+    # Build summary
+    arm_stats = {}
+    for arm, data in sorted(by_arm.items()):
+        arm_stats[arm] = {
+            "interface": data["interface"],
+            "file_count": len(data["files"]),
+            "total_tokens": stats(data["tokens"]),
+            "duration_ms": stats(data["duration_ms"]),
+        }
+
+    iface_stats = {}
+    for iface, data in sorted(by_interface.items()):
+        iface_stats[iface] = {
+            "file_count": len(data["files"]),
+            "total_tokens": stats(data["tokens"]),
+            "duration_ms": stats(data["duration_ms"]),
+        }
+
+    # Compute delta between arms (A vs B)
+    delta = {}
+    if "arm-A" in arm_stats and "arm-B" in arm_stats:
+        a_mean = arm_stats["arm-A"]["total_tokens"].get("mean") or 0
+        b_mean = arm_stats["arm-B"]["total_tokens"].get("mean") or 0
+        if b_mean > 0:
+            delta = {
+                "mean_tokens": round(a_mean - b_mean, 1),
+                "percent": f"{((a_mean - b_mean) / b_mean * 100):+.1f}%",
+                "note": f"Arm A uses {abs(a_mean - b_mean):.0f} {'more' if a_mean > b_mean else 'fewer'} tokens on average",
+            }
+
+    summary = {
+        "run_dir": os.path.abspath(run_dir),
+        "timing_files_found": len(timing_files),
+        "timing_files_missing_tokens": len(missing_tokens),
+        "by_arm": arm_stats,
+        "by_interface": iface_stats,
+        "delta_A_vs_B": delta,
+        "warnings": (
+            [f"MISSING total_tokens in {len(missing_tokens)} files — fill these from task notifications"]
+            if missing_tokens else []
+        ),
+    }
+
+    output_path = args.output or os.path.join(run_dir, "token-summary.json")
+    with open(output_path, "w") as f:
+        json.dump(summary, f, indent=2)
+
+    # Print summary
+    print(f"\nToken Summary")
+    print(f"{'='*50}")
+    for arm, s in arm_stats.items():
+        t = s["total_tokens"]
+        if t["mean"] is not None:
+            print(f"  {arm} ({s['interface']}): {t['mean']:.0f} tokens (±{t['stddev']:.0f}, n={t['count']})")
+        else:
+            print(f"  {arm} ({s['interface']}): NO TOKEN DATA (fill timing.json from task notifications)")
+    if delta:
+        print(f"\n  Delta (A-B): {delta['percent']} ({delta['mean_tokens']:+.0f} tokens)")
+        print(f"  {delta['note']}")
+    if missing_tokens:
+        print(f"\n  WARNING: {len(missing_tokens)} files missing total_tokens")
+        print(f"  These must be filled from Claude Code task notification data.")
+    print(f"\nWritten: {output_path}")
+
+
+if __name__ == "__main__":
+    main()

package/packages/ct-skills/skills/ct-orchestrator/SKILL.md

@@ -1,34 +1,6 @@
 ---
 name: ct-orchestrator
-description: |
-  Pipeline-aware orchestration skill for managing complex workflows through subagent delegation.
-  Use when the user asks to "orchestrate", "orchestrator mode", "run as orchestrator",
-  "delegate to subagents", "coordinate agents", "spawn subagents", "multi-agent workflow",
-  "context-protected workflow", "agent farm", "HITL orchestration", "pipeline management",
-  or needs to manage complex workflows by delegating work to subagents while protecting
-  the main context window. Enforces ORC-001 through ORC-009 constraints. Provider-neutral —
-  works with any agent runtime that supports prompt-based delegation.
-version: 4.0.0
-tier: 0
-core: true
-category: core
-protocol: agent-protocol
-mvi_scope: orchestrator
-requires_tiers:
-  - minimal
-  - standard
-  - orchestrator
-dependencies: []
-sharedResources:
-  - subagent-protocol-base
-  - task-system-integration
-compatibility:
-  - claude-code
-  - cursor
-  - windsurf
-  - gemini-cli
-  - opencode
-  - codex-cli
+description: "Pipeline-aware orchestration skill for managing complex workflows through subagent delegation. Use when the user asks to \"orchestrate\", \"orchestrator mode\", \"run as orchestrator\", \"delegate to subagents\", \"coordinate agents\", \"spawn subagents\", \"multi-agent workflow\", \"context-protected workflow\", \"agent farm\", \"HITL orchestration\", \"pipeline management\", or needs to manage complex workflows by delegating work to subagents while protecting the main context window. Enforces ORC-001 through ORC-009 constraints. Provider-neutral."
 license: MIT
 ---
 

package/packages/ct-skills/skills/ct-orchestrator/manifest-entry.json

@@ -0,0 +1,19 @@
+{
+  "name": "ct-orchestrator",
+  "version": "4.0.0",
+  "description": "Pipeline-aware orchestration skill for managing complex workflows through subagent delegation.",
+  "path": "packages/ct-skills/skills/ct-orchestrator",
+  "status": "active",
+  "tier": 0,
+  "core": true,
+  "category": "core",
+  "protocol": "agent-protocol",
+  "mvi_scope": "orchestrator",
+  "requires_tiers": ["minimal", "standard", "orchestrator"],
+  "dependencies": [],
+  "sharedResources": ["subagent-protocol-base", "task-system-integration"],
+  "compatibility": ["claude-code", "cursor", "windsurf", "gemini-cli", "opencode", "codex-cli"],
+  "token_budget": 8000,
+  "capabilities": ["multi-agent-coordination", "wave-planning", "context-protection", "hitl-orchestration"],
+  "constraints": ["requires-agent-runtime"]
+}

package/packages/ct-skills/skills/ct-skill-creator/SKILL.md

@@ -1,18 +1,6 @@
 ---
 name: ct-skill-creator
 description: Guide for creating effective skills. This skill should be used when users want to create a new skill (or update an existing skill) that extends Claude's capabilities with specialized knowledge, workflows, or tool integrations.
-version: 1.0.0
-tier: 3
-core: false
-category: meta
-protocol: null
-dependencies: []
-sharedResources: []
-compatibility:
-  - claude-code
-  - cursor
-  - windsurf
-  - gemini-cli
 license: MIT
 ---
 

package/packages/ct-skills/skills/ct-skill-creator/agents/analyzer.md

@@ -0,0 +1,276 @@
+# Post-hoc Analyzer Agent
+
+Analyze blind comparison results to understand WHY the winner won and generate improvement suggestions.
+
+## Role
+
+After the blind comparator determines a winner, the Post-hoc Analyzer "unblinds" the results by examining the skills and transcripts. The goal is to extract actionable insights: what made the winner better, and how can the loser be improved?
+
+## Inputs
+
+You receive these parameters in your prompt:
+
+- **winner**: "A" or "B" (from blind comparison)
+- **winner_skill_path**: Path to the skill that produced the winning output
+- **winner_transcript_path**: Path to the execution transcript for the winner
+- **loser_skill_path**: Path to the skill that produced the losing output
+- **loser_transcript_path**: Path to the execution transcript for the loser
+- **comparison_result_path**: Path to the blind comparator's output JSON
+- **output_path**: Where to save the analysis results
+
+## Process
+
+### Step 1: Read Comparison Result
+
+1. Read the blind comparator's output at comparison_result_path
+2. Note the winning side (A or B), the reasoning, and any scores
+3. Understand what the comparator valued in the winning output
+
+### Step 2: Read Both Skills
+
+1. Read the winner skill's SKILL.md and key referenced files
+2. Read the loser skill's SKILL.md and key referenced files
+3. Identify structural differences:
+   - Instructions clarity and specificity
+   - Script/tool usage patterns
+   - Example coverage
+   - Edge case handling
+
+### Step 3: Read Both Transcripts
+
+1. Read the winner's transcript
+2. Read the loser's transcript
+3. Compare execution patterns:
+   - How closely did each follow their skill's instructions?
+   - What tools were used differently?
+   - Where did the loser diverge from optimal behavior?
+   - Did either encounter errors or make recovery attempts?
+
+### Step 4: Analyze Instruction Following
+
+For each transcript, evaluate:
+- Did the agent follow the skill's explicit instructions?
+- Did the agent use the skill's provided tools/scripts?
+- Were there missed opportunities to leverage skill content?
+- Did the agent add unnecessary steps not in the skill?
+
+Score instruction following 1-10 and note specific issues.
+
+### Step 5: Identify Winner Strengths
+
+Determine what made the winner better:
+- Clearer instructions that led to better behavior?
+- Better scripts/tools that produced better output?
+- More comprehensive examples that guided edge cases?
+- Better error handling guidance?
+
+Be specific. Quote from skills/transcripts where relevant.
+
+### Step 6: Identify Loser Weaknesses
+
+Determine what held the loser back:
+- Ambiguous instructions that led to suboptimal choices?
+- Missing tools/scripts that forced workarounds?
+- Gaps in edge case coverage?
+- Poor error handling that caused failures?
+
+### Step 7: Generate Improvement Suggestions
+
+Based on the analysis, produce actionable suggestions for improving the loser skill:
+- Specific instruction changes to make
+- Tools/scripts to add or modify
+- Examples to include
+- Edge cases to address
+
+Prioritize by impact. Focus on changes that would have changed the outcome.
+
+### Step 8: Write Analysis Results
+
+Save structured analysis to `{output_path}`.
+
+## Output Format
+
+Write a JSON file with this structure:
+
+```json
+{
+  "comparison_summary": {
+    "winner": "A",
+    "winner_skill": "path/to/winner/skill",
+    "loser_skill": "path/to/loser/skill",
+    "comparator_reasoning": "Brief summary of why comparator chose winner"
+  },
+  "winner_strengths": [
+    "Clear step-by-step instructions for handling multi-page documents",
+    "Included validation script that caught formatting errors",
+    "Explicit guidance on fallback behavior when OCR fails"
+  ],
+  "loser_weaknesses": [
+    "Vague instruction 'process the document appropriately' led to inconsistent behavior",
+    "No script for validation, agent had to improvise and made errors",
+    "No guidance on OCR failure, agent gave up instead of trying alternatives"
+  ],
+  "instruction_following": {
+    "winner": {
+      "score": 9,
+      "issues": [
+        "Minor: skipped optional logging step"
+      ]
+    },
+    "loser": {
+      "score": 6,
+      "issues": [
+        "Did not use the skill's formatting template",
+        "Invented own approach instead of following step 3",
+        "Missed the 'always validate output' instruction"
+      ]
+    }
+  },
+  "improvement_suggestions": [
+    {
+      "priority": "high",
+      "category": "instructions",
+      "suggestion": "Replace 'process the document appropriately' with explicit steps: 1) Extract text, 2) Identify sections, 3) Format per template",
+      "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior"
+    },
+    {
+      "priority": "high",
+      "category": "tools",
+      "suggestion": "Add validate_output.py script similar to winner skill's validation approach",
+      "expected_impact": "Would catch formatting errors before final output"
+    },
+    {
+      "priority": "medium",
+      "category": "error_handling",
+      "suggestion": "Add fallback instructions: 'If OCR fails, try: 1) different resolution, 2) image preprocessing, 3) manual extraction'",
+      "expected_impact": "Would prevent early failure on difficult documents"
+    }
+  ],
+  "transcript_insights": {
+    "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script -> Fixed 2 issues -> Produced output",
+    "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods -> No validation -> Output had errors"
+  }
+}
+```
+
+## Guidelines
+
+- **Be specific**: Quote from skills and transcripts, don't just say "instructions were unclear"
+- **Be actionable**: Suggestions should be concrete changes, not vague advice
+- **Focus on skill improvements**: The goal is to improve the losing skill, not critique the agent
+- **Prioritize by impact**: Which changes would most likely have changed the outcome?
+- **Consider causation**: Did the skill weakness actually cause the worse output, or is it incidental?
+- **Stay objective**: Analyze what happened, don't editorialize
+- **Think about generalization**: Would this improvement help on other evals too?
+
+## Categories for Suggestions
+
+Use these categories to organize improvement suggestions:
+
+| Category | Description |
+|----------|-------------|
+| `instructions` | Changes to the skill's prose instructions |
+| `tools` | Scripts, templates, or utilities to add/modify |
+| `examples` | Example inputs/outputs to include |
+| `error_handling` | Guidance for handling failures |
+| `structure` | Reorganization of skill content |
+| `references` | External docs or resources to add |
+
+## Priority Levels
+
+- **high**: Would likely change the outcome of this comparison
+- **medium**: Would improve quality but may not change win/loss
+- **low**: Nice to have, marginal improvement
+
+---
+
+# Analyzing Benchmark Results
+
+When analyzing benchmark results, the analyzer's purpose is to **surface patterns and anomalies** across multiple runs, not suggest skill improvements.
+
+## Role
+
+Review all benchmark run results and generate freeform notes that help the user understand skill performance. Focus on patterns that wouldn't be visible from aggregate metrics alone.
+
+## Inputs
+
+You receive these parameters in your prompt:
+
+- **benchmark_data_path**: Path to the in-progress benchmark.json with all run results
+- **skill_path**: Path to the skill being benchmarked
+- **output_path**: Where to save the notes (as JSON array of strings)
+
+## Process
+
+### Step 1: Read Benchmark Data
+
+1. Read the benchmark.json containing all run results
+2. Note the configurations tested (with_skill, without_skill)
+3. Understand the run_summary aggregates already calculated
+
+### Step 2: Analyze Per-Assertion Patterns
+
+For each expectation across all runs:
+- Does it **always pass** in both configurations? (may not differentiate skill value)
+- Does it **always fail** in both configurations? (may be broken or beyond capability)
+- Does it **always pass with skill but fail without**? (skill clearly adds value here)
+- Does it **always fail with skill but pass without**? (skill may be hurting)
+- Is it **highly variable**? (flaky expectation or non-deterministic behavior)
+
+### Step 3: Analyze Cross-Eval Patterns
+
+Look for patterns across evals:
+- Are certain eval types consistently harder/easier?
+- Do some evals show high variance while others are stable?
+- Are there surprising results that contradict expectations?
+
+### Step 4: Analyze Metrics Patterns
+
+Look at time_seconds, tokens, tool_calls:
+- Does the skill significantly increase execution time?
+- Is there high variance in resource usage?
+- Are there outlier runs that skew the aggregates?
+
+### Step 5: Generate Notes
+
+Write freeform observations as a list of strings. Each note should:
+- State a specific observation
+- Be grounded in the data (not speculation)
+- Help the user understand something the aggregate metrics don't show
+
+Examples:
+- "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value"
+- "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure that may be flaky"
+- "Without-skill runs consistently fail on table extraction expectations (0% pass rate)"
+- "Skill adds 13s average execution time but improves pass rate by 50%"
+- "Token usage is 80% higher with skill, primarily due to script output parsing"
+- "All 3 without-skill runs for eval 1 produced empty output"
+
+### Step 6: Write Notes
+
+Save notes to `{output_path}` as a JSON array of strings:
+
+```json
+[
+  "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value",
+  "Eval 3 shows high variance (50% ± 40%) - run 2 had an unusual failure",
+  "Without-skill runs consistently fail on table extraction expectations",
+  "Skill adds 13s average execution time but improves pass rate by 50%"
+]
+```
+
+## Guidelines
+
+**DO:**
+- Report what you observe in the data
+- Be specific about which evals, expectations, or runs you're referring to
+- Note patterns that aggregate metrics would hide
+- Provide context that helps interpret the numbers
+
+**DO NOT:**
+- Suggest improvements to the skill (that's for the improvement step, not benchmarking)
+- Make subjective quality judgments ("the output was good/bad")
+- Speculate about causes without evidence
+- Repeat information already in the run_summary aggregates
+
+See [references/schemas.md](../references/schemas.md) for the complete analysis.json and benchmark.json schema definitions.