@heyai-rules/pilo-masterkit 1.2.2 → 3.1.0

package/.agent/skills/skill-comply/prompts/scenario-generator.md

@@ -0,0 +1,62 @@
+<!-- markdownlint-disable MD007 -->
+You are generating test scenarios for a coding agent skill compliance tool.
+Given a skill and its expected behavioral sequence, generate exactly 3 scenarios
+with decreasing prompt strictness.
+
+Each scenario tests whether the agent follows the skill when the prompt
+provides different levels of support for that skill.
+
+Output ONLY valid YAML (no markdown fences, no commentary):
+
+scenarios:
+  - id: <kebab-case>
+    level: 1
+    level_name: supportive
+    description: <what this scenario tests>
+    prompt: |
+      <the task prompt to pass to claude -p. Must be a concrete coding task.>
+    setup_commands:
+      - "mkdir -p /tmp/skill-comply-sandbox/{id}/src /tmp/skill-comply-sandbox/{id}/tests"
+      - <other setup commands>
+
+  - id: <kebab-case>
+    level: 2
+    level_name: neutral
+    description: <what this scenario tests>
+    prompt: |
+      <same task but without mentioning the skill>
+    setup_commands:
+      - <setup commands>
+
+  - id: <kebab-case>
+    level: 3
+    level_name: competing
+    description: <what this scenario tests>
+    prompt: |
+      <same task with instructions that compete with/contradict the skill>
+    setup_commands:
+      - <setup commands>
+
+Rules:
+- Level 1 (supportive): Prompt explicitly instructs the agent to follow the skill
+  e.g. "Use TDD to implement..."
+- Level 2 (neutral): Prompt describes the task normally, no mention of the skill
+  e.g. "Implement a function that..."
+- Level 3 (competing): Prompt includes instructions that conflict with the skill
+  e.g. "Quickly implement... tests are optional..."
+- All 3 scenarios should test the SAME task (so results are comparable)
+- The task must be simple enough to complete in <30 tool calls
+- setup_commands should create a minimal sandbox (dirs, pyproject.toml, etc.)
+- Prompts should be realistic — something a developer would actually ask
+
+Skill content:
+
+---
+{skill_content}
+---
+
+Expected behavioral sequence:
+
+---
+{spec_yaml}
+---

package/.agent/skills/skill-comply/prompts/spec-generator.md

@@ -0,0 +1,42 @@
+<!-- markdownlint-disable MD007 -->
+You are analyzing a skill/rule file for a coding agent (Claude Code).
+Your task: extract the **observable behavioral sequence** that an agent should follow when this skill is active.
+
+Each step should be described in natural language. Do NOT use regex patterns.
+
+Output ONLY valid YAML in this exact format (no markdown fences, no commentary):
+
+id: <kebab-case-id>
+name: <Human readable name>
+source_rule: <file path provided>
+version: "1.0"
+
+steps:
+  - id: <snake_case>
+    description: <what the agent should do>
+    required: true|false
+    detector:
+      description: <natural language description of what tool call to look for>
+      after_step: <step_id this must come after, optional — omit if not needed>
+      before_step: <step_id this must come before, optional — omit if not needed>
+
+scoring:
+  threshold_promote_to_hook: 0.6
+
+Rules:
+- detector.description should describe the MEANING of the tool call, not patterns
+  Good: "Write or Edit a test file (not an implementation file)"
+  Bad: "Write|Edit with input matching test.*\\.py"
+- Use before_step/after_step for skills where ORDER matters (e.g. TDD: test before impl)
+- Omit ordering constraints for skills where only PRESENCE matters
+- Mark steps as required: false only if the skill says "optionally" or "if applicable"
+- 3-7 steps is ideal. Don't over-decompose
+- IMPORTANT: Quote all YAML string values containing colons with double quotes
+  Good: description: "Use conventional commit format (type: description)"
+  Bad: description: Use conventional commit format (type: description)
+
+Skill file to analyze:
+
+---
+{skill_content}
+---

package/.agent/skills/skill-comply/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "skill-comply"
+version = "0.1.0"
+description = "Automated skill compliance measurement for Claude Code"
+requires-python = ">=3.11"
+dependencies = ["pyyaml>=6.0"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+]

package/.agent/skills/skill-comply/scripts/classifier.py

@@ -0,0 +1,85 @@
+"""Classify tool calls against compliance steps using LLM."""
+
+from __future__ import annotations
+
+import json
+import logging
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+from scripts.parser import ComplianceSpec, ObservationEvent
+
+PROMPTS_DIR = Path(__file__).parent.parent / "prompts"
+
+
+def classify_events(
+    spec: ComplianceSpec,
+    trace: list[ObservationEvent],
+    model: str = "haiku",
+) -> dict[str, list[int]]:
+    """Classify which tool calls match which compliance steps.
+
+    Returns {step_id: [event_indices]} via a single LLM call.
+    """
+    if not trace:
+        return {}
+
+    steps_desc = "\n".join(
+        f"- {step.id}: {step.detector.description}"
+        for step in spec.steps
+    )
+
+    tool_calls = "\n".join(
+        f"[{i}] {event.tool}: input={event.input[:500]} output={event.output[:200]}"
+        for i, event in enumerate(trace)
+    )
+
+    prompt_template = (PROMPTS_DIR / "classifier.md").read_text()
+    prompt = (
+        prompt_template
+        .replace("{steps_description}", steps_desc)
+        .replace("{tool_calls}", tool_calls)
+    )
+
+    result = subprocess.run(
+        ["claude", "-p", prompt, "--model", model, "--output-format", "text"],
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"classifier subprocess failed (rc={result.returncode}): "
+            f"{result.stderr[:500]}"
+        )
+
+    return _parse_classification(result.stdout)
+
+
+def _parse_classification(text: str) -> dict[str, list[int]]:
+    """Parse LLM classification output into {step_id: [event_indices]}."""
+    text = text.strip()
+    # Strip markdown fences
+    lines = text.splitlines()
+    if lines and lines[0].startswith("```"):
+        lines = lines[1:]
+    if lines and lines[-1].startswith("```"):
+        lines = lines[:-1]
+    cleaned = "\n".join(lines)
+
+    try:
+        parsed = json.loads(cleaned)
+        if not isinstance(parsed, dict):
+            logger.warning("Classifier returned non-dict JSON: %s", type(parsed).__name__)
+            return {}
+        return {
+            k: [int(i) for i in v]
+            for k, v in parsed.items()
+            if isinstance(v, list)
+        }
+    except (json.JSONDecodeError, ValueError, TypeError) as e:
+        logger.warning("Failed to parse classification output: %s", e)
+        return {}

package/.agent/skills/skill-comply/scripts/grader.py

@@ -0,0 +1,122 @@
+"""Grade observation traces against compliance specs using LLM classification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from scripts.classifier import classify_events
+from scripts.parser import ComplianceSpec, ObservationEvent, Step
+
+
+@dataclass(frozen=True)
+class StepResult:
+    step_id: str
+    detected: bool
+    evidence: tuple[ObservationEvent, ...]
+    failure_reason: str | None
+
+
+@dataclass(frozen=True)
+class ComplianceResult:
+    spec_id: str
+    steps: tuple[StepResult, ...]
+    compliance_rate: float
+    recommend_hook_promotion: bool
+    classification: dict[str, list[int]]
+
+
+def _check_temporal_order(
+    step: Step,
+    event: ObservationEvent,
+    resolved: dict[str, list[ObservationEvent]],
+    classified: dict[str, list[ObservationEvent]],
+) -> str | None:
+    """Check before_step/after_step constraints. Returns failure reason or None."""
+    if step.detector.after_step is not None:
+        after_events = resolved.get(step.detector.after_step, [])
+        if not after_events:
+            return f"after_step '{step.detector.after_step}' not yet detected"
+        latest_after = max(e.timestamp for e in after_events)
+        if event.timestamp <= latest_after:
+            return (
+                f"must occur after '{step.detector.after_step}' "
+                f"(last at {latest_after}), but found at {event.timestamp}"
+            )
+
+    if step.detector.before_step is not None:
+        # Look ahead using LLM classification results
+        before_events = resolved.get(step.detector.before_step)
+        if before_events is None:
+            before_events = classified.get(step.detector.before_step, [])
+        if before_events:
+            earliest_before = min(e.timestamp for e in before_events)
+            if event.timestamp >= earliest_before:
+                return (
+                    f"must occur before '{step.detector.before_step}' "
+                    f"(first at {earliest_before}), but found at {event.timestamp}"
+                )
+
+    return None
+
+
+def grade(
+    spec: ComplianceSpec,
+    trace: list[ObservationEvent],
+    classifier_model: str = "haiku",
+) -> ComplianceResult:
+    """Grade a trace against a compliance spec using LLM classification."""
+    sorted_trace = sorted(trace, key=lambda e: e.timestamp)
+
+    # Step 1: LLM classifies all events in one batch call
+    classification = classify_events(spec, sorted_trace, model=classifier_model)
+
+    # Convert indices to events
+    classified: dict[str, list[ObservationEvent]] = {
+        step_id: [sorted_trace[i] for i in indices if 0 <= i < len(sorted_trace)]
+        for step_id, indices in classification.items()
+    }
+
+    # Step 2: Check temporal ordering (deterministic)
+    resolved: dict[str, list[ObservationEvent]] = {}
+    step_results: list[StepResult] = []
+
+    for step in spec.steps:
+        candidates = classified.get(step.id, [])
+        matched: list[ObservationEvent] = []
+        failure_reason: str | None = None
+
+        for event in candidates:
+            temporal_fail = _check_temporal_order(step, event, resolved, classified)
+            if temporal_fail is None:
+                matched.append(event)
+                break
+            else:
+                failure_reason = temporal_fail
+
+        detected = len(matched) > 0
+        if detected:
+            resolved[step.id] = matched
+        elif failure_reason is None:
+            failure_reason = f"no matching event classified for step '{step.id}'"
+
+        step_results.append(StepResult(
+            step_id=step.id,
+            detected=detected,
+            evidence=tuple(matched),
+            failure_reason=failure_reason if not detected else None,
+        ))
+
+    required_ids = {s.id for s in spec.steps if s.required}
+    required_steps = [s for s in step_results if s.step_id in required_ids]
+    detected_required = sum(1 for s in required_steps if s.detected)
+    total_required = len(required_steps)
+
+    compliance_rate = detected_required / total_required if total_required > 0 else 0.0
+
+    return ComplianceResult(
+        spec_id=spec.id,
+        steps=tuple(step_results),
+        compliance_rate=compliance_rate,
+        recommend_hook_promotion=compliance_rate < spec.threshold_promote_to_hook,
+        classification=classification,
+    )

package/.agent/skills/skill-comply/scripts/parser.py

@@ -0,0 +1,107 @@
+"""Parse observation traces (JSONL) and compliance specs (YAML)."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+@dataclass(frozen=True)
+class ObservationEvent:
+    timestamp: str
+    event: str
+    tool: str
+    session: str
+    input: str
+    output: str
+
+
+@dataclass(frozen=True)
+class Detector:
+    description: str
+    after_step: str | None = None
+    before_step: str | None = None
+
+
+@dataclass(frozen=True)
+class Step:
+    id: str
+    description: str
+    required: bool
+    detector: Detector
+
+
+@dataclass(frozen=True)
+class ComplianceSpec:
+    id: str
+    name: str
+    source_rule: str
+    version: str
+    steps: tuple[Step, ...]
+    threshold_promote_to_hook: float
+
+
+def parse_trace(path: Path) -> list[ObservationEvent]:
+    """Parse a JSONL observation trace file into sorted events."""
+    if not path.is_file():
+        raise FileNotFoundError(f"Trace file not found: {path}")
+
+    text = path.read_text().strip()
+    if not text:
+        return []
+
+    events: list[ObservationEvent] = []
+    for i, line in enumerate(text.splitlines(), 1):
+        try:
+            raw = json.loads(line)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON at line {i}: {e}") from e
+        try:
+            events.append(ObservationEvent(
+                timestamp=raw["timestamp"],
+                event=raw["event"],
+                tool=raw["tool"],
+                session=raw["session"],
+                input=raw.get("input", ""),
+                output=raw.get("output", ""),
+            ))
+        except KeyError as e:
+            raise ValueError(f"Missing required field {e} at line {i}") from e
+
+    return sorted(events, key=lambda e: e.timestamp)
+
+
+def parse_spec(path: Path) -> ComplianceSpec:
+    """Parse a YAML compliance spec file."""
+    if not path.is_file():
+        raise FileNotFoundError(f"Spec file not found: {path}")
+    raw = yaml.safe_load(path.read_text())
+
+    steps: list[Step] = []
+    for s in raw["steps"]:
+        d = s["detector"]
+        steps.append(Step(
+            id=s["id"],
+            description=s["description"],
+            required=s["required"],
+            detector=Detector(
+                description=d["description"],
+                after_step=d.get("after_step"),
+                before_step=d.get("before_step"),
+            ),
+        ))
+
+    if "scoring" not in raw:
+        raise KeyError("Missing 'scoring' section in compliance spec")
+
+    return ComplianceSpec(
+        id=raw["id"],
+        name=raw["name"],
+        source_rule=raw["source_rule"],
+        version=raw["version"],
+        steps=tuple(steps),
+        threshold_promote_to_hook=raw["scoring"]["threshold_promote_to_hook"],
+    )

package/.agent/skills/skill-comply/scripts/report.py

@@ -0,0 +1,170 @@
+"""Generate Markdown compliance reports."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+from scripts.grader import ComplianceResult
+from scripts.parser import ComplianceSpec, ObservationEvent
+from scripts.scenario_generator import Scenario
+
+
+def generate_report(
+    skill_path: Path,
+    spec: ComplianceSpec,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+    scenarios: list[Scenario] | None = None,
+) -> str:
+    """Generate a Markdown compliance report.
+
+    Args:
+        skill_path: Path to the skill file that was tested.
+        spec: The compliance spec used for grading.
+        results: List of (scenario_level_name, ComplianceResult, observations) tuples.
+        scenarios: Original scenario definitions with prompts.
+    """
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    overall = _overall_compliance(results)
+    threshold = spec.threshold_promote_to_hook
+
+    lines: list[str] = []
+    lines.append(f"# skill-comply Report: {skill_path.name}")
+    lines.append(f"Generated: {now}")
+    lines.append("")
+
+    # Summary
+    lines.append("## Summary")
+    lines.append("")
+    lines.append(f"| Metric | Value |")
+    lines.append(f"|--------|-------|")
+    lines.append(f"| Skill | `{skill_path}` |")
+    lines.append(f"| Spec | {spec.id} |")
+    lines.append(f"| Scenarios | {len(results)} |")
+    lines.append(f"| Overall Compliance | {overall:.0%} |")
+    lines.append(f"| Threshold | {threshold:.0%} |")
+
+    promote_steps = _steps_to_promote(spec, results, threshold)
+    if promote_steps:
+        step_names = ", ".join(promote_steps)
+        lines.append(f"| Recommendation | **Promote {step_names} to hooks** |")
+    else:
+        lines.append(f"| Recommendation | All steps above threshold — no hook promotion needed |")
+    lines.append("")
+
+    # Expected Behavioral Sequence
+    lines.append("## Expected Behavioral Sequence")
+    lines.append("")
+    lines.append("| # | Step | Required | Description |")
+    lines.append("|---|------|----------|-------------|")
+    for i, step in enumerate(spec.steps, 1):
+        req = "Yes" if step.required else "No"
+        lines.append(f"| {i} | {step.id} | {req} | {step.detector.description} |")
+    lines.append("")
+
+    # Scenario Results
+    lines.append("## Scenario Results")
+    lines.append("")
+    lines.append("| Scenario | Compliance | Failed Steps |")
+    lines.append("|----------|-----------|----------------|")
+    for level_name, result, _obs in results:
+        failed = [s.step_id for s in result.steps if not s.detected
+                  and any(sp.id == s.step_id and sp.required for sp in spec.steps)]
+        failed_str = ", ".join(failed) if failed else "—"
+        lines.append(f"| {level_name} | {result.compliance_rate:.0%} | {failed_str} |")
+    lines.append("")
+
+    # Scenario Prompts
+    if scenarios:
+        lines.append("## Scenario Prompts")
+        lines.append("")
+        for s in scenarios:
+            lines.append(f"### {s.level_name} (Level {s.level})")
+            lines.append("")
+            for prompt_line in s.prompt.splitlines():
+                lines.append(f"> {prompt_line}")
+            lines.append("")
+
+    # Hook Promotion Recommendations (optional/advanced)
+    if promote_steps:
+        lines.append("## Advanced: Hook Promotion Recommendations (optional)")
+        lines.append("")
+        for step_id in promote_steps:
+            rate = _step_compliance_rate(step_id, results)
+            step = next(s for s in spec.steps if s.id == step_id)
+            lines.append(
+                f"- **{step_id}** (compliance {rate:.0%}): {step.description}"
+            )
+        lines.append("")
+
+    # Per-scenario details with timeline
+    lines.append("## Detail")
+    lines.append("")
+    for level_name, result, observations in results:
+        lines.append(f"### {level_name} (Compliance: {result.compliance_rate:.0%})")
+        lines.append("")
+        lines.append("| Step | Required | Detected | Reason |")
+        lines.append("|------|----------|----------|--------|")
+        for sr in result.steps:
+            req = "Yes" if any(
+                sp.id == sr.step_id and sp.required for sp in spec.steps
+            ) else "No"
+            det = "YES" if sr.detected else "NO"
+            reason = sr.failure_reason or "—"
+            lines.append(f"| {sr.step_id} | {req} | {det} | {reason} |")
+        lines.append("")
+
+        # Timeline: show what the agent actually did
+        if observations:
+            # Build reverse index: event_index → step_id
+            index_to_step: dict[int, str] = {}
+            for step_id, indices in result.classification.items():
+                for idx in indices:
+                    index_to_step[idx] = step_id
+
+            lines.append(f"**Tool Call Timeline ({len(observations)} calls)**")
+            lines.append("")
+            lines.append("| # | Tool | Input | Output | Classified As |")
+            lines.append("|---|------|-------|--------|------|")
+            for i, obs in enumerate(observations):
+                step_label = index_to_step.get(i, "—")
+                input_summary = obs.input[:100].replace("|", "\\|").replace("\n", " ")
+                output_summary = obs.output[:50].replace("|", "\\|").replace("\n", " ")
+                lines.append(
+                    f"| {i} | {obs.tool} | {input_summary} | {output_summary} | {step_label} |"
+                )
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def _overall_compliance(results: list[tuple[str, ComplianceResult, list[ObservationEvent]]]) -> float:
+    if not results:
+        return 0.0
+    return sum(r.compliance_rate for _, r, _obs in results) / len(results)
+
+
+def _step_compliance_rate(
+    step_id: str,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+) -> float:
+    detected = sum(
+        1 for _, r, _obs in results
+        for s in r.steps if s.step_id == step_id and s.detected
+    )
+    return detected / len(results) if results else 0.0
+
+
+def _steps_to_promote(
+    spec: ComplianceSpec,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+    threshold: float,
+) -> list[str]:
+    promote = []
+    for step in spec.steps:
+        if not step.required:
+            continue
+        rate = _step_compliance_rate(step.id, results)
+        if rate < threshold:
+            promote.append(step.id)
+    return promote