codingbuddy-rules 4.5.0 → 5.1.0

package/.ai-rules/skills/skill-creator/scripts/run_loop.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""Run a description optimization loop for skill trigger evaluation.
+
+Loads trigger_eval.json, splits cases into 60/40 train/test sets,
+and iterates to optimize the skill description for better trigger accuracy.
+
+LLM calls are replaced with CLI guidance — the script logs scores and
+prompts the user to manually refine the description between iterations.
+
+Usage:
+    python run_loop.py <trigger-eval-json> --skill-name <name> [--iterations N] [--seed S]
+
+Example:
+    python run_loop.py workspace/trigger_eval.json --skill-name tdd --iterations 5
+
+Requirements:
+    Python 3.8+ (standard library only)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+
+def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Description optimization loop for skill trigger evaluation. "
+            "Splits trigger_eval.json into train/test sets and guides "
+            "iterative description refinement."
+        ),
+        epilog=(
+            "Example:\n"
+            "  python run_loop.py workspace/trigger_eval.json "
+            "--skill-name tdd --iterations 5\n\n"
+            "The script will guide you through each iteration, prompting\n"
+            "you to run `recommend_skills` manually and enter results."
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "trigger_eval_json",
+        type=str,
+        help="Path to trigger_eval.json file",
+    )
+    parser.add_argument(
+        "--skill-name",
+        required=True,
+        help="Target skill name in kebab-case",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=5,
+        help="Number of optimization iterations (default: 5)",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for train/test split (default: 42)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=None,
+        help="Directory to write iteration logs (default: same as trigger_eval.json)",
+    )
+    return parser.parse_args(argv)
+
+
+def _load_trigger_eval(path: Path) -> List[Dict[str, Any]]:
+    """Load and validate trigger_eval.json."""
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+
+    if not isinstance(data, list) or len(data) == 0:
+        raise ValueError(
+            "trigger_eval.json must be a non-empty array of test cases"
+        )
+
+    for i, case in enumerate(data):
+        if "query" not in case or "should_trigger" not in case:
+            raise ValueError(
+                f"Case {i} missing required fields: 'query' and 'should_trigger'"
+            )
+
+    return data
+
+
+def _split_train_test(
+    cases: List[Dict[str, Any]], seed: int
+) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+    """Split cases into 60% train / 40% test with deterministic shuffle."""
+    rng = random.Random(seed)
+    shuffled = list(cases)
+    rng.shuffle(shuffled)
+    split_idx = max(1, int(len(shuffled) * 0.6))
+    return shuffled[:split_idx], shuffled[split_idx:]
+
+
+def _compute_metrics(
+    results: List[Dict[str, Any]],
+) -> Dict[str, float]:
+    """Compute precision, recall, F1 from trigger results.
+
+    Each result dict has:
+        - should_trigger: bool (ground truth)
+        - triggered: bool (actual result from user input)
+    """
+    tp = sum(1 for r in results if r["should_trigger"] and r["triggered"])
+    fp = sum(1 for r in results if not r["should_trigger"] and r["triggered"])
+    fn = sum(1 for r in results if r["should_trigger"] and not r["triggered"])
+    tn = sum(
+        1 for r in results if not r["should_trigger"] and not r["triggered"]
+    )
+
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+    f1 = (
+        2 * precision * recall / (precision + recall)
+        if (precision + recall) > 0
+        else 0.0
+    )
+    accuracy = (tp + tn) / len(results) if results else 0.0
+
+    return {
+        "precision": round(precision, 4),
+        "recall": round(recall, 4),
+        "f1": round(f1, 4),
+        "accuracy": round(accuracy, 4),
+        "tp": tp,
+        "fp": fp,
+        "fn": fn,
+        "tn": tn,
+    }
+
+
+def _prompt_user_results(
+    cases: List[Dict[str, Any]], skill_name: str, label: str
+) -> List[Dict[str, Any]]:
+    """Prompt user to manually run recommend_skills and report results.
+
+    Since LLM calls cannot be made from this script, we guide the user
+    to evaluate each query and enter whether the skill was triggered.
+    """
+    print(f"\n{'=' * 60}")
+    print(f"  Evaluating {label} set ({len(cases)} cases)")
+    print(f"  Target skill: {skill_name}")
+    print(f"{'=' * 60}")
+    print()
+    print("For each query below, run:")
+    print(f"  recommend_skills(prompt=<query>)")
+    print(f"and check if '{skill_name}' appears in the results.")
+    print()
+
+    results: List[Dict[str, Any]] = []
+    for i, case in enumerate(cases):
+        query = case["query"]
+        expected = case["should_trigger"]
+        print(f"  [{i + 1}/{len(cases)}] Query: {query}")
+        print(f"           Expected: {'TRIGGER' if expected else 'NO TRIGGER'}")
+
+        while True:
+            response = input(
+                "           Result? (y=triggered / n=not triggered / s=skip): "
+            ).strip().lower()
+            if response in ("y", "n", "s"):
+                break
+            print("           Invalid input. Enter y, n, or s.")
+
+        if response == "s":
+            print("           -> Skipped")
+            continue
+
+        triggered = response == "y"
+        match = triggered == expected
+        results.append(
+            {
+                "query": query,
+                "should_trigger": expected,
+                "triggered": triggered,
+                "match": match,
+            }
+        )
+        print(f"           -> {'MATCH' if match else 'MISMATCH'}")
+
+    return results
+
+
+def _print_metrics(metrics: Dict[str, float], label: str) -> None:
+    """Pretty-print evaluation metrics."""
+    print(f"\n--- {label} Metrics ---")
+    print(f"  Precision: {metrics['precision']:.2%}")
+    print(f"  Recall:    {metrics['recall']:.2%}")
+    print(f"  F1 Score:  {metrics['f1']:.2%}")
+    print(f"  Accuracy:  {metrics['accuracy']:.2%}")
+    print(
+        f"  (TP={metrics['tp']} FP={metrics['fp']} "
+        f"FN={metrics['fn']} TN={metrics['tn']})"
+    )
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = parse_args(argv)
+    trigger_path = Path(args.trigger_eval_json).resolve()
+
+    if not trigger_path.is_file():
+        print(f"[ERROR] File not found: {trigger_path}", file=sys.stderr)
+        return 1
+
+    output_dir = Path(args.output_dir) if args.output_dir else trigger_path.parent
+    output_dir = output_dir.resolve()
+
+    try:
+        cases = _load_trigger_eval(trigger_path)
+    except (json.JSONDecodeError, ValueError) as exc:
+        print(f"[ERROR] Failed to load trigger_eval.json: {exc}", file=sys.stderr)
+        return 1
+
+    train_set, test_set = _split_train_test(cases, args.seed)
+    print(f"Loaded {len(cases)} cases: {len(train_set)} train / {len(test_set)} test")
+    print(f"Skill: {args.skill_name}")
+    print(f"Iterations: {args.iterations}")
+    print(f"Seed: {args.seed}")
+
+    iteration_log: List[Dict[str, Any]] = []
+
+    for iteration in range(1, args.iterations + 1):
+        print(f"\n{'#' * 60}")
+        print(f"  ITERATION {iteration}/{args.iterations}")
+        print(f"{'#' * 60}")
+
+        # Step 1: Evaluate on train set
+        print("\n[Step 1] Evaluate current description on TRAIN set")
+        train_results = _prompt_user_results(
+            train_set, args.skill_name, f"Iteration {iteration} TRAIN"
+        )
+        train_metrics = _compute_metrics(train_results)
+        _print_metrics(train_metrics, f"Iteration {iteration} TRAIN")
+
+        # Step 2: Guide description refinement
+        print(f"\n[Step 2] Refine the skill description")
+        print("  Based on the train results above, update the skill's")
+        print("  'description' field in SKILL.md frontmatter to improve")
+        print("  trigger accuracy.")
+        print()
+        print("  Mismatched cases to focus on:")
+        mismatches = [r for r in train_results if not r.get("match", True)]
+        if mismatches:
+            for m in mismatches:
+                direction = "should trigger but didn't" if m["should_trigger"] else "triggered but shouldn't"
+                print(f"    - \"{m['query']}\" ({direction})")
+        else:
+            print("    (none — perfect score on train set)")
+        print()
+        input("  Press Enter when description has been updated...")
+
+        # Step 3: Evaluate on test set
+        print("\n[Step 3] Evaluate updated description on TEST set")
+        test_results = _prompt_user_results(
+            test_set, args.skill_name, f"Iteration {iteration} TEST"
+        )
+        test_metrics = _compute_metrics(test_results)
+        _print_metrics(test_metrics, f"Iteration {iteration} TEST")
+
+        # Log iteration
+        entry = {
+            "iteration": iteration,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "train": {
+                "metrics": train_metrics,
+                "total_cases": len(train_results),
+            },
+            "test": {
+                "metrics": test_metrics,
+                "total_cases": len(test_results),
+            },
+        }
+        iteration_log.append(entry)
+
+        # Check convergence
+        if test_metrics["f1"] >= 1.0:
+            print("\n[INFO] Perfect F1 on test set. Stopping early.")
+            break
+
+    # Write iteration log
+    log_path = output_dir / f"optimization_log_{args.skill_name}.json"
+    with open(log_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "skill_name": args.skill_name,
+                "seed": args.seed,
+                "train_size": len(train_set),
+                "test_size": len(test_set),
+                "iterations": iteration_log,
+            },
+            f,
+            indent=2,
+            ensure_ascii=False,
+        )
+    print(f"\n[OK] Optimization log written: {log_path}")
+
+    # Final summary
+    print(f"\n{'=' * 60}")
+    print("  OPTIMIZATION SUMMARY")
+    print(f"{'=' * 60}")
+    print(f"\n  {'Iter':>4}  {'Train F1':>10}  {'Test F1':>10}  {'Test Acc':>10}")
+    print(f"  {'----':>4}  {'--------':>10}  {'-------':>10}  {'--------':>10}")
+    for entry in iteration_log:
+        it = entry["iteration"]
+        tf1 = entry["train"]["metrics"]["f1"]
+        sf1 = entry["test"]["metrics"]["f1"]
+        sacc = entry["test"]["metrics"]["accuracy"]
+        print(f"  {it:>4}  {tf1:>10.2%}  {sf1:>10.2%}  {sacc:>10.2%}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.ai-rules/skills/systematic-debugging/SKILL.md

@@ -1,6 +1,7 @@
 ---
 name: systematic-debugging
 description: Use when encountering any bug, test failure, or unexpected behavior, before proposing fixes
+allowed-tools: Read, Grep, Glob, Bash
 ---
 
 # Systematic Debugging

package/.ai-rules/skills/tech-debt/SKILL.md

@@ -1,6 +1,7 @@
 ---
 name: tech-debt
 description: Use when identifying, prioritizing, and planning resolution of technical debt. Provides structured assessment, ROI-based prioritization, and incremental paydown strategies.
+allowed-tools: Read, Grep, Glob
 ---
 
 # Tech Debt

package/.ai-rules/skills/test-coverage-gate/SKILL.md

@@ -0,0 +1,303 @@
+---
+name: test-coverage-gate
+description: >-
+  Use before shipping, creating PRs, or merging to enforce minimum test coverage thresholds.
+  Covers line, branch, and function coverage. Supports vitest, jest, c8, and istanbul.
+  Blocks shipping when coverage falls below configurable thresholds (default 80%).
+disable-model-invocation: true
+argument-hint: "[--threshold=N] [--type=line|branch|function|all]"
+---
+
+# Test Coverage Gate
+
+## Overview
+
+Untested code is unverified code. Shipping without coverage data is shipping blind.
+
+This skill enforces minimum test coverage thresholds before code leaves your machine. It detects your coverage tool, runs the analysis, compares results against thresholds, and blocks shipping if coverage is insufficient.
+
+**Core principle:** No code ships without meeting coverage thresholds. Period.
+
+**Iron Law:**
+```
+NO SHIP WITHOUT COVERAGE GATE PASSING
+If coverage is below threshold, the PR does not get created.
+```
+
+## When to Use
+
+**Always, before:**
+- Running `/ship` to create a PR
+- Merging feature branches
+- Tagging releases
+- Deploying to staging or production
+
+**Especially when:**
+- Adding new features (new code must be tested)
+- Refactoring (coverage must not drop)
+- Fixing bugs (regression test required)
+- Touching critical paths (auth, payments, data processing)
+
+**Skip only when:**
+- Documentation-only changes (no source code modified)
+- Configuration file changes (non-code)
+- Generated code explicitly excluded from coverage
+
+## Configuration
+
+### Default Thresholds
+
+```
+Line Coverage:     80%
+Branch Coverage:   80%
+Function Coverage: 80%
+```
+
+### Custom Thresholds
+
+Override defaults per project via `codingbuddy.config.json`, `package.json`, or tool-native config:
+
+```jsonc
+// codingbuddy.config.json
+{
+  "coverage": {
+    "thresholds": {
+      "lines": 80,
+      "branches": 80,
+      "functions": 80
+    },
+    "tool": "auto",        // "vitest" | "jest" | "c8" | "istanbul" | "auto"
+    "excludePatterns": [
+      "**/*.config.*",
+      "**/generated/**",
+      "**/migrations/**"
+    ]
+  }
+}
+```
+
+```jsonc
+// package.json (alternative)
+{
+  "coverageGate": {
+    "lines": 80,
+    "branches": 80,
+    "functions": 80
+  }
+}
+```
+
+### Per-Tool Native Config
+
+If thresholds are already defined in your test tool's config, the gate respects them:
+
+| Tool | Config Location |
+|------|----------------|
+| Vitest | `vitest.config.ts` → `test.coverage.thresholds` |
+| Jest | `jest.config.ts` → `coverageThreshold.global` |
+| c8 | `.c8rc.json` or `package.json` → `c8` |
+| Istanbul/nyc | `.nycrc` or `package.json` → `nyc` |
+
+**Priority order:** CLI flags > `codingbuddy.config.json` > tool-native config > defaults (80%)
+
+## Supported Tools
+
+### Auto-Detection
+
+The gate detects your coverage tool automatically:
+
+```
+1. Check package.json devDependencies:
+   - @vitest/coverage-v8 or @vitest/coverage-istanbul → vitest
+   - jest + jest coverage config → jest
+   - c8 → c8
+   - nyc or istanbul → istanbul
+2. Check for tool-specific config files:
+   - vitest.config.ts → vitest
+   - jest.config.ts → jest
+   - .c8rc.json → c8
+   - .nycrc → istanbul
+3. Fallback: prompt user to specify tool
+```
+
+### Tool Commands
+
+| Tool | Coverage Command |
+|------|-----------------|
+| **Vitest** | `npx vitest run --coverage --reporter=json` |
+| **Jest** | `npx jest --coverage --coverageReporters=json-summary` |
+| **c8** | `npx c8 --reporter=json-summary npm test` |
+| **Istanbul/nyc** | `npx nyc --reporter=json-summary npm test` |
+
+## Workflow
+
+### Phase 1: Detect Coverage Tool
+
+```
+- [ ] Scan package.json devDependencies for coverage packages
+- [ ] Check for tool-specific config files
+- [ ] Resolve coverage tool (or prompt if ambiguous)
+- [ ] Verify coverage tool is installed
+```
+
+If the tool is not installed:
+```bash
+# Suggest installation
+echo "Coverage tool not found. Install with:"
+echo "  yarn add -D @vitest/coverage-v8   # for vitest"
+echo "  yarn add -D jest                   # for jest"
+echo "  yarn add -D c8                     # for c8"
+echo "  yarn add -D nyc                    # for istanbul/nyc"
+```
+
+### Phase 2: Run Coverage Analysis
+
+```
+- [ ] Execute coverage command for detected tool
+- [ ] Capture JSON coverage output
+- [ ] Parse coverage summary (lines, branches, functions)
+```
+
+**Expected output format (json-summary):**
+```json
+{
+  "total": {
+    "lines": { "total": 500, "covered": 420, "pct": 84.0 },
+    "branches": { "total": 120, "covered": 96, "pct": 80.0 },
+    "functions": { "total": 80, "covered": 68, "pct": 85.0 }
+  }
+}
+```
+
+### Phase 3: Compare Against Thresholds
+
+```
+- [ ] Load thresholds (CLI > config > tool-native > defaults)
+- [ ] Compare each metric against its threshold
+- [ ] Classify result: PASS or FAIL per metric
+```
+
+**Comparison logic:**
+```
+For each metric in [lines, branches, functions]:
+  actual = coverage_report.total[metric].pct
+  threshold = resolved_thresholds[metric]
+  result = actual >= threshold ? PASS : FAIL
+```
+
+### Phase 4: Gate Decision
+
+#### PASS — All metrics meet thresholds
+
+```
+Coverage Gate: PASS
+
+  Lines:     84.0% >= 80% ✅
+  Branches:  80.0% >= 80% ✅
+  Functions: 85.0% >= 80% ✅
+
+Proceed with shipping.
+```
+
+#### FAIL — One or more metrics below threshold
+
+```
+Coverage Gate: FAIL ❌
+
+  Lines:     84.0% >= 80% ✅
+  Branches:  72.0% >= 80% ❌ (-8.0%)
+  Functions: 85.0% >= 80% ✅
+
+BLOCKED: Coverage below threshold.
+
+Uncovered areas requiring attention:
+  - src/auth/oauth.ts: branches 45% (missing: error paths lines 23-31, 55-60)
+  - src/utils/parser.ts: branches 60% (missing: edge cases lines 88-95)
+
+Action required:
+  1. Add tests for uncovered branches listed above
+  2. Re-run coverage gate
+  3. Ship only after all metrics pass
+```
+
+**On failure:**
+- Block `/ship` from creating the PR
+- List specific files and lines with low coverage
+- Suggest which tests to write
+- Never allow override without explicit `--skip-coverage` flag
+
+## Integration with /ship
+
+### Pre-Ship Hook
+
+The coverage gate runs **before** any PR creation step in `/ship`:
+
+```
+/ship workflow:
+  1. Lint check
+  2. Type check (tsc --noEmit)
+  3. Run tests
+  4. ▶ COVERAGE GATE ◀  ← This skill
+  5. Build verification
+  6. Create branch + commit + push
+  7. Create PR
+```
+
+If the coverage gate fails at step 4, steps 5-7 are **not executed**.
+
+### Skip Flag
+
+For exceptional cases (hotfixes, documentation):
+```
+/ship --skip-coverage
+```
+
+When `--skip-coverage` is used:
+- Log a warning: `⚠️ Coverage gate skipped. Reason must be documented in PR.`
+- Add `[skip-coverage]` label to the PR
+- Require reason in PR description
+
+## Failure Recovery
+
+### Common Issues
+
+| Issue | Solution |
+|-------|----------|
+| Coverage tool not installed | Install via `yarn add -D <tool>` |
+| No coverage config found | Add `coverage` section to test tool config |
+| Coverage report not generated | Verify test command produces JSON output |
+| Threshold too strict for legacy code | Set per-directory thresholds, raise gradually |
+| New files have 0% coverage | Write tests before shipping (TDD) |
+
+### Gradual Adoption
+
+For projects starting below 80%:
+
+```jsonc
+// Start with current baseline, ratchet up over time
+{
+  "coverage": {
+    "thresholds": {
+      "lines": 60,       // Current: 58% → target 80%
+      "branches": 50,    // Current: 48% → target 80%
+      "functions": 65     // Current: 63% → target 80%
+    },
+    "ratchet": true       // Never allow coverage to decrease
+  }
+}
+```
+
+**Ratchet mode:** When enabled, the threshold automatically increases to match the highest coverage ever achieved. Coverage can only go up, never down.
+
+## Red Flags
+
+These thoughts mean STOP — you're rationalizing:
+
+| Thought | Reality |
+|---------|---------|
+| "Coverage doesn't matter for this change" | Every change matters. Untested code breaks. |
+| "I'll add tests later" | Later never comes. Test now. |
+| "80% is too high for this project" | Lower the threshold explicitly, don't skip the gate. |
+| "This is just a hotfix" | Hotfixes need tests too — use `--skip-coverage` and document why. |
+| "The tests are slow" | Slow tests > no tests. Optimize later. |
+| "It's only a config change" | Config changes can break things. Verify coverage didn't drop. |