@cleocode/skills 2026.5.82 → 2026.5.84

package/skills/ct-grade-v2-1/grade-viewer/build_op_stats.py

@@ -1,174 +0,0 @@
-#!/usr/bin/env python3
-"""
-build_op_stats.py — Aggregate operations.jsonl files from grade runs into per-operation statistics.
-
-Reads all operations.jsonl files under --grade-runs-dir and computes per-operation stats
-split by interface (mcp/cli). Output is a JSON object keyed by "domain.operation".
-
-Usage:
-    python build_op_stats.py [options]
-
-Options:
-    --grade-runs-dir PATH    Directory containing grade run subdirectories
-                             (default: .cleo/metrics/grade-runs relative to cwd)
-    --output PATH            Output JSON file path
-                             (default: .cleo/metrics/per_operation_stats.json)
-    --pretty                 Pretty-print JSON output (default: compact)
-    --verbose                Print progress to stderr
-
-Output format (per key "domain.operation"):
-{
-    "mcp_calls": 42,
-    "cli_calls": 10,
-    "total_mcp_ms": 1234.5,
-    "total_cli_ms": 456.7,
-    "avg_mcp_ms": 29.4,
-    "avg_cli_ms": 45.7,
-    "runs_seen": 3
-}
-
-Also importable as a module:
-    from build_op_stats import compute_stats
-    stats = compute_stats(grade_runs_dir="/path/to/grade-runs")
-"""
-
-import argparse
-import json
-import sys
-from pathlib import Path
-
-
-def compute_stats(grade_runs_dir, verbose=False):
-    """
-    Aggregate operations.jsonl files under grade_runs_dir.
-
-    Returns dict keyed by "domain.operation" with accumulated stats.
-    """
-    runs_dir = Path(grade_runs_dir)
-    stats = {}
-    files_processed = 0
-    lines_processed = 0
-
-    if not runs_dir.exists():
-        if verbose:
-            print(f"[build_op_stats] Grade runs dir not found: {runs_dir}", file=sys.stderr)
-        return stats
-
-    for ops_file in sorted(runs_dir.rglob('operations.jsonl')):
-        files_processed += 1
-        if verbose:
-            print(f"[build_op_stats] Processing: {ops_file}", file=sys.stderr)
-
-        for line in ops_file.read_text(errors='replace').splitlines():
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                entry = json.loads(line)
-            except json.JSONDecodeError:
-                continue
-
-            domain = entry.get('domain', 'unknown')
-            operation = entry.get('operation', 'unknown')
-            key = f"{domain}.{operation}"
-            interface = entry.get('interface', 'mcp')
-            duration = float(entry.get('duration_ms', 0) or 0)
-
-            if key not in stats:
-                stats[key] = {
-                    'mcp_calls': 0,
-                    'cli_calls': 0,
-                    'total_mcp_ms': 0.0,
-                    'total_cli_ms': 0.0,
-                    'avg_mcp_ms': 0.0,
-                    'avg_cli_ms': 0.0,
-                    'runs_seen': set(),
-                }
-
-            # Track which run directory this came from
-            # ops_file is e.g. .../grade-runs/run-20260308/s1/run-01/arm-mcp/operations.jsonl
-            # run_id is the first path component relative to runs_dir (e.g. "run-20260308")
-            run_id = ops_file.relative_to(runs_dir).parts[0]
-            stats[key]['runs_seen'].add(run_id)
-
-            if interface == 'cli':
-                stats[key]['cli_calls'] += 1
-                stats[key]['total_cli_ms'] += duration
-            else:
-                stats[key]['mcp_calls'] += 1
-                stats[key]['total_mcp_ms'] += duration
-
-            lines_processed += 1
-
-    # Compute averages and convert sets to counts
-    for key, v in stats.items():
-        mc = v['mcp_calls']
-        cc = v['cli_calls']
-        v['avg_mcp_ms'] = round(v['total_mcp_ms'] / mc, 2) if mc > 0 else 0.0
-        v['avg_cli_ms'] = round(v['total_cli_ms'] / cc, 2) if cc > 0 else 0.0
-        v['total_mcp_ms'] = round(v['total_mcp_ms'], 2)
-        v['total_cli_ms'] = round(v['total_cli_ms'], 2)
-        v['runs_seen'] = len(v['runs_seen'])
-
-    if verbose:
-        print(f"[build_op_stats] Processed {files_processed} files, {lines_processed} lines → {len(stats)} unique operations", file=sys.stderr)
-
-    return stats
-
-
-def find_cleo_dir(start='.'):
-    """Walk up from start to find directory containing .cleo/tasks.db."""
-    p = Path(start).resolve()
-    while p != p.parent:
-        if (p / '.cleo' / 'tasks.db').exists():
-            return p
-        p = p.parent
-    return Path(start).resolve()
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description='Aggregate grade run operations.jsonl files into per-operation stats.'
-    )
-    parser.add_argument(
-        '--grade-runs-dir',
-        default=None,
-        help='Directory containing grade run subdirectories (default: .cleo/metrics/grade-runs)'
-    )
-    parser.add_argument(
-        '--output',
-        default=None,
-        help='Output JSON path (default: .cleo/metrics/per_operation_stats.json)'
-    )
-    parser.add_argument(
-        '--pretty',
-        action='store_true',
-        help='Pretty-print JSON output'
-    )
-    parser.add_argument(
-        '--verbose',
-        action='store_true',
-        help='Print progress to stderr'
-    )
-    args = parser.parse_args()
-
-    workspace = find_cleo_dir('.')
-
-    grade_runs_dir = args.grade_runs_dir or str(workspace / '.cleo' / 'metrics' / 'grade-runs')
-    output_path = args.output or str(workspace / '.cleo' / 'metrics' / 'per_operation_stats.json')
-
-    stats = compute_stats(grade_runs_dir, verbose=args.verbose)
-
-    indent = 2 if args.pretty else None
-    output_json = json.dumps(stats, indent=indent)
-
-    out = Path(output_path)
-    out.parent.mkdir(parents=True, exist_ok=True)
-    out.write_text(output_json)
-
-    print(f"Wrote {len(stats)} operation stats to {output_path}")
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())

package/skills/ct-grade-v2-1/grade-viewer/eval-analysis.json

@@ -1,41 +0,0 @@
-{
-  "total_grades": 31,
-  "score_distribution": {
-    "F (0)": 7,
-    "D (45-59)": 10,
-    "C (60-74)": 5,
-    "B (75-89)": 8,
-    "A (90+)": 1
-  },
-  "score_stats": {
-    "mean": 64.6,
-    "min": 50,
-    "max": 95,
-    "grades_with_data": 24,
-    "zero_score_count": 7
-  },
-  "dimension_averages": {
-    "sessionDiscipline": 5.8,
-    "discoveryEfficiency": 9.0,
-    "taskHygiene": 15.4,
-    "errorProtocol": 15.3,
-    "disclosureUse": 4.5
-  },
-  "flag_frequency": {
-    "No admin.help calls": 21,
-    "session.list never called": 18,
-    "No MCP query calls": 13,
-    "session.end never called": 12,
-    "No audit entries": 7,
-    "tasks.list used (prefer find)": 5,
-    "Subtasks without exists check": 1,
-    "Duplicate task creates": 1
-  },
-  "avg_audit_entries": 9.5,
-  "token_estimate": {
-    "avg_per_session_chars": 0,
-    "avg_per_session_tokens": 1425.0,
-    "method": "entry_count * 150 proxy",
-    "note": "OTEL not enabled; enable with CLAUDE_CODE_ENABLE_TELEMETRY=1 for real counts"
-  }
-}

package/skills/ct-grade-v2-1/grade-viewer/eval-report.md

@@ -1,37 +0,0 @@
-# CLEO Grade v2.1 — Comparative Analysis Report
-
-**Generated:** 2026-03-07 23:47 UTC  
-**Source:** `/tmp/ct-grade-eval`
-
-> **DEPRECATED**: This report was generated when MCP was still supported. MCP has been removed.
-> All operations now use the CLI exclusively. These results are retained for historical reference only.
-
----
-
-## Historical: MCP vs CLI Blind A/B Results
-
-**Overall winner: MCP**  
-
-| Metric | Value |
-|--------|-------|
-| Total runs | 3 |
-| MCP wins | 3 (100.0%) |
-| CLI wins | 0 (0.0%) |
-| Ties | 0 |
-| Avg token delta (MCP–CLI) | +416.0 tokens |
-| Interpretation | MCP uses more tokens on average |
-
-### Per-Operation Results
-
-| Operation | MCP wins | CLI wins | Ties | Token delta | MCP chars | CLI chars | MCP ms | CLI ms |
-|-----------|----------|----------|------|-------------|-----------|-----------|--------|--------|
-| `admin.version` **MCP** | 3 | 0 | 0 | +416t | 1664 | 0 | 930ms | 786ms |
-
-### Recommendations
-
-- **MCP adds significant token overhead.** Consider whether MCP envelope verbosity can be reduced for high-frequency operations.
-- **MCP output quality is consistently higher.** Reinforces MCP-first agent protocol recommendation.
-
----
-
-*Report generated by ct-grade v2.1 `generate_report.py`*