@heyai-rules/pilo-masterkit 2.1.0 → 3.1.0

package/.agent/skills/skill-comply/scripts/run.py

@@ -1,127 +1,127 @@
-"""CLI entry point for skill-comply."""
-
-from __future__ import annotations
-
-import argparse
-import logging
-import sys
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-from scripts.grader import grade
-from scripts.report import generate_report
-from scripts.runner import run_scenario
-from scripts.scenario_generator import generate_scenarios
-from scripts.spec_generator import generate_spec
-
-logger = logging.getLogger(__name__)
-
-
-def main() -> None:
-    logging.basicConfig(level=logging.INFO, format="%(message)s")
-
-    parser = argparse.ArgumentParser(
-        description="skill-comply: Measure skill compliance rates",
-    )
-    parser.add_argument(
-        "skill",
-        type=Path,
-        help="Path to skill/rule file to test",
-    )
-    parser.add_argument(
-        "--model",
-        default="sonnet",
-        help="Model for scenario execution (default: sonnet)",
-    )
-    parser.add_argument(
-        "--gen-model",
-        default="haiku",
-        help="Model for spec/scenario generation (default: haiku)",
-    )
-    parser.add_argument(
-        "--dry-run",
-        action="store_true",
-        help="Generate spec and scenarios without executing",
-    )
-    parser.add_argument(
-        "--output",
-        type=Path,
-        default=None,
-        help="Output report path (default: results/<skill-name>.md)",
-    )
-
-    args = parser.parse_args()
-
-    if not args.skill.is_file():
-        logger.error("Error: Skill file not found: %s", args.skill)
-        sys.exit(1)
-
-    results_dir = Path(__file__).parent.parent / "results"
-    results_dir.mkdir(exist_ok=True)
-
-    # Step 1: Generate compliance spec
-    logger.info("[1/4] Generating compliance spec from %s...", args.skill.name)
-    spec = generate_spec(args.skill, model=args.gen_model)
-    logger.info("       %d steps extracted", len(spec.steps))
-
-    # Step 2: Generate scenarios
-    spec_yaml = yaml.dump({
-        "steps": [
-            {"id": s.id, "description": s.description, "required": s.required}
-            for s in spec.steps
-        ]
-    })
-    logger.info("[2/4] Generating scenarios (3 prompt strictness levels)...")
-    scenarios = generate_scenarios(args.skill, spec_yaml, model=args.gen_model)
-    logger.info("       %d scenarios generated", len(scenarios))
-
-    for s in scenarios:
-        logger.info("       - %s: %s", s.level_name, s.description[:60])
-
-    if args.dry_run:
-        logger.info("\n[dry-run] Spec and scenarios generated. Skipping execution.")
-        logger.info("\nSpec: %s (%d steps)", spec.id, len(spec.steps))
-        for step in spec.steps:
-            marker = "*" if step.required else " "
-            logger.info("  [%s] %s: %s", marker, step.id, step.description)
-        return
-
-    # Step 3: Execute scenarios
-    logger.info("[3/4] Executing scenarios (model=%s)...", args.model)
-    graded_results: list[tuple[str, Any, list[Any]]] = []
-
-    for scenario in scenarios:
-        logger.info("       Running %s...", scenario.level_name)
-        run = run_scenario(scenario, model=args.model)
-        result = grade(spec, list(run.observations))
-        graded_results.append((scenario.level_name, result, list(run.observations)))
-        logger.info("       %s: %.0f%%", scenario.level_name, result.compliance_rate * 100)
-
-    # Step 4: Generate report
-    skill_name = args.skill.parent.name if args.skill.stem == "SKILL" else args.skill.stem
-    output_path = args.output or results_dir / f"{skill_name}.md"
-    logger.info("[4/4] Generating report...")
-
-    report = generate_report(args.skill, spec, graded_results, scenarios=scenarios)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(report)
-    logger.info("       Report saved to %s", output_path)
-
-    # Summary
-    if not graded_results:
-        logger.warning("No scenarios were executed.")
-        return
-    overall = sum(r.compliance_rate for _, r, _obs in graded_results) / len(graded_results)
-    logger.info("\n%s", "=" * 50)
-    logger.info("Overall Compliance: %.0f%%", overall * 100)
-    if overall < spec.threshold_promote_to_hook:
-        logger.info(
-            "Recommendation: Some steps have low compliance. "
-            "Consider promoting them to hooks. See the report for details."
-        )
-
-
-if __name__ == "__main__":
-    main()
+"""CLI entry point for skill-comply."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from scripts.grader import grade
+from scripts.report import generate_report
+from scripts.runner import run_scenario
+from scripts.scenario_generator import generate_scenarios
+from scripts.spec_generator import generate_spec
+
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+    parser = argparse.ArgumentParser(
+        description="skill-comply: Measure skill compliance rates",
+    )
+    parser.add_argument(
+        "skill",
+        type=Path,
+        help="Path to skill/rule file to test",
+    )
+    parser.add_argument(
+        "--model",
+        default="sonnet",
+        help="Model for scenario execution (default: sonnet)",
+    )
+    parser.add_argument(
+        "--gen-model",
+        default="haiku",
+        help="Model for spec/scenario generation (default: haiku)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Generate spec and scenarios without executing",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="Output report path (default: results/<skill-name>.md)",
+    )
+
+    args = parser.parse_args()
+
+    if not args.skill.is_file():
+        logger.error("Error: Skill file not found: %s", args.skill)
+        sys.exit(1)
+
+    results_dir = Path(__file__).parent.parent / "results"
+    results_dir.mkdir(exist_ok=True)
+
+    # Step 1: Generate compliance spec
+    logger.info("[1/4] Generating compliance spec from %s...", args.skill.name)
+    spec = generate_spec(args.skill, model=args.gen_model)
+    logger.info("       %d steps extracted", len(spec.steps))
+
+    # Step 2: Generate scenarios
+    spec_yaml = yaml.dump({
+        "steps": [
+            {"id": s.id, "description": s.description, "required": s.required}
+            for s in spec.steps
+        ]
+    })
+    logger.info("[2/4] Generating scenarios (3 prompt strictness levels)...")
+    scenarios = generate_scenarios(args.skill, spec_yaml, model=args.gen_model)
+    logger.info("       %d scenarios generated", len(scenarios))
+
+    for s in scenarios:
+        logger.info("       - %s: %s", s.level_name, s.description[:60])
+
+    if args.dry_run:
+        logger.info("\n[dry-run] Spec and scenarios generated. Skipping execution.")
+        logger.info("\nSpec: %s (%d steps)", spec.id, len(spec.steps))
+        for step in spec.steps:
+            marker = "*" if step.required else " "
+            logger.info("  [%s] %s: %s", marker, step.id, step.description)
+        return
+
+    # Step 3: Execute scenarios
+    logger.info("[3/4] Executing scenarios (model=%s)...", args.model)
+    graded_results: list[tuple[str, Any, list[Any]]] = []
+
+    for scenario in scenarios:
+        logger.info("       Running %s...", scenario.level_name)
+        run = run_scenario(scenario, model=args.model)
+        result = grade(spec, list(run.observations))
+        graded_results.append((scenario.level_name, result, list(run.observations)))
+        logger.info("       %s: %.0f%%", scenario.level_name, result.compliance_rate * 100)
+
+    # Step 4: Generate report
+    skill_name = args.skill.parent.name if args.skill.stem == "SKILL" else args.skill.stem
+    output_path = args.output or results_dir / f"{skill_name}.md"
+    logger.info("[4/4] Generating report...")
+
+    report = generate_report(args.skill, spec, graded_results, scenarios=scenarios)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report)
+    logger.info("       Report saved to %s", output_path)
+
+    # Summary
+    if not graded_results:
+        logger.warning("No scenarios were executed.")
+        return
+    overall = sum(r.compliance_rate for _, r, _obs in graded_results) / len(graded_results)
+    logger.info("\n%s", "=" * 50)
+    logger.info("Overall Compliance: %.0f%%", overall * 100)
+    if overall < spec.threshold_promote_to_hook:
+        logger.info(
+            "Recommendation: Some steps have low compliance. "
+            "Consider promoting them to hooks. See the report for details."
+        )
+
+
+if __name__ == "__main__":
+    main()

package/.agent/skills/skill-comply/scripts/runner.py

@@ -1,161 +1,161 @@
-"""Run scenarios via claude -p and parse tool calls from stream-json output."""
-
-from __future__ import annotations
-
-import json
-import re
-import shlex
-import shutil
-import subprocess
-from dataclasses import dataclass
-from pathlib import Path
-
-from scripts.parser import ObservationEvent
-from scripts.scenario_generator import Scenario
-
-SANDBOX_BASE = Path("/tmp/skill-comply-sandbox")
-ALLOWED_MODELS = frozenset({"haiku", "sonnet", "opus"})
-
-
-@dataclass(frozen=True)
-class ScenarioRun:
-    scenario: Scenario
-    observations: tuple[ObservationEvent, ...]
-    sandbox_dir: Path
-
-
-def run_scenario(
-    scenario: Scenario,
-    model: str = "sonnet",
-    max_turns: int = 30,
-    timeout: int = 300,
-) -> ScenarioRun:
-    """Execute a scenario and extract tool calls from stream-json output."""
-    if model not in ALLOWED_MODELS:
-        raise ValueError(f"Unknown model: {model!r}. Allowed: {ALLOWED_MODELS}")
-
-    sandbox_dir = _safe_sandbox_dir(scenario.id)
-    _setup_sandbox(sandbox_dir, scenario)
-
-    result = subprocess.run(
-        [
-            "claude", "-p", scenario.prompt,
-            "--model", model,
-            "--max-turns", str(max_turns),
-            "--add-dir", str(sandbox_dir),
-            "--allowedTools", "Read,Write,Edit,Bash,Glob,Grep",
-            "--output-format", "stream-json",
-            "--verbose",
-        ],
-        capture_output=True,
-        text=True,
-        timeout=timeout,
-        cwd=sandbox_dir,
-    )
-
-    if result.returncode != 0:
-        raise RuntimeError(
-            f"claude -p failed (rc={result.returncode}): {result.stderr[:500]}"
-        )
-
-    observations = _parse_stream_json(result.stdout)
-
-    return ScenarioRun(
-        scenario=scenario,
-        observations=tuple(observations),
-        sandbox_dir=sandbox_dir,
-    )
-
-
-def _safe_sandbox_dir(scenario_id: str) -> Path:
-    """Sanitize scenario ID and ensure path stays within sandbox base."""
-    safe_id = re.sub(r"[^a-zA-Z0-9\-_]", "_", scenario_id)
-    path = SANDBOX_BASE / safe_id
-    # Validate path stays within sandbox base (raises ValueError on traversal)
-    path.resolve().relative_to(SANDBOX_BASE.resolve())
-    return path
-
-
-def _setup_sandbox(sandbox_dir: Path, scenario: Scenario) -> None:
-    """Create sandbox directory and run setup commands."""
-    if sandbox_dir.exists():
-        shutil.rmtree(sandbox_dir)
-    sandbox_dir.mkdir(parents=True)
-
-    subprocess.run(["git", "init"], cwd=sandbox_dir, capture_output=True)
-
-    for cmd in scenario.setup_commands:
-        parts = shlex.split(cmd)
-        subprocess.run(parts, cwd=sandbox_dir, capture_output=True)
-
-
-def _parse_stream_json(stdout: str) -> list[ObservationEvent]:
-    """Parse claude -p stream-json output into ObservationEvents.
-
-    Stream-json format:
-    - type=assistant with content[].type=tool_use → tool call (name, input)
-    - type=user with content[].type=tool_result → tool result (output)
-    """
-    events: list[ObservationEvent] = []
-    pending: dict[str, dict] = {}
-    event_counter = 0
-
-    for line in stdout.strip().splitlines():
-        try:
-            msg = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-
-        msg_type = msg.get("type")
-
-        if msg_type == "assistant":
-            content = msg.get("message", {}).get("content", [])
-            for block in content:
-                if block.get("type") == "tool_use":
-                    tool_use_id = block.get("id", "")
-                    tool_input = block.get("input", {})
-                    input_str = (
-                        json.dumps(tool_input)[:5000]
-                        if isinstance(tool_input, dict)
-                        else str(tool_input)[:5000]
-                    )
-                    pending[tool_use_id] = {
-                        "tool": block.get("name", "unknown"),
-                        "input": input_str,
-                        "order": event_counter,
-                    }
-                    event_counter += 1
-
-        elif msg_type == "user":
-            content = msg.get("message", {}).get("content", [])
-            if isinstance(content, list):
-                for block in content:
-                    tool_use_id = block.get("tool_use_id", "")
-                    if tool_use_id in pending:
-                        info = pending.pop(tool_use_id)
-                        output_content = block.get("content", "")
-                        if isinstance(output_content, list):
-                            output_str = json.dumps(output_content)[:5000]
-                        else:
-                            output_str = str(output_content)[:5000]
-
-                        events.append(ObservationEvent(
-                            timestamp=f"T{info['order']:04d}",
-                            event="tool_complete",
-                            tool=info["tool"],
-                            session=msg.get("session_id", "unknown"),
-                            input=info["input"],
-                            output=output_str,
-                        ))
-
-    for _tool_use_id, info in pending.items():
-        events.append(ObservationEvent(
-            timestamp=f"T{info['order']:04d}",
-            event="tool_complete",
-            tool=info["tool"],
-            session="unknown",
-            input=info["input"],
-            output="",
-        ))
-
-    return sorted(events, key=lambda e: e.timestamp)
+"""Run scenarios via claude -p and parse tool calls from stream-json output."""
+
+from __future__ import annotations
+
+import json
+import re
+import shlex
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from scripts.parser import ObservationEvent
+from scripts.scenario_generator import Scenario
+
+SANDBOX_BASE = Path("/tmp/skill-comply-sandbox")
+ALLOWED_MODELS = frozenset({"haiku", "sonnet", "opus"})
+
+
+@dataclass(frozen=True)
+class ScenarioRun:
+    scenario: Scenario
+    observations: tuple[ObservationEvent, ...]
+    sandbox_dir: Path
+
+
+def run_scenario(
+    scenario: Scenario,
+    model: str = "sonnet",
+    max_turns: int = 30,
+    timeout: int = 300,
+) -> ScenarioRun:
+    """Execute a scenario and extract tool calls from stream-json output."""
+    if model not in ALLOWED_MODELS:
+        raise ValueError(f"Unknown model: {model!r}. Allowed: {ALLOWED_MODELS}")
+
+    sandbox_dir = _safe_sandbox_dir(scenario.id)
+    _setup_sandbox(sandbox_dir, scenario)
+
+    result = subprocess.run(
+        [
+            "claude", "-p", scenario.prompt,
+            "--model", model,
+            "--max-turns", str(max_turns),
+            "--add-dir", str(sandbox_dir),
+            "--allowedTools", "Read,Write,Edit,Bash,Glob,Grep",
+            "--output-format", "stream-json",
+            "--verbose",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        cwd=sandbox_dir,
+    )
+
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"claude -p failed (rc={result.returncode}): {result.stderr[:500]}"
+        )
+
+    observations = _parse_stream_json(result.stdout)
+
+    return ScenarioRun(
+        scenario=scenario,
+        observations=tuple(observations),
+        sandbox_dir=sandbox_dir,
+    )
+
+
+def _safe_sandbox_dir(scenario_id: str) -> Path:
+    """Sanitize scenario ID and ensure path stays within sandbox base."""
+    safe_id = re.sub(r"[^a-zA-Z0-9\-_]", "_", scenario_id)
+    path = SANDBOX_BASE / safe_id
+    # Validate path stays within sandbox base (raises ValueError on traversal)
+    path.resolve().relative_to(SANDBOX_BASE.resolve())
+    return path
+
+
+def _setup_sandbox(sandbox_dir: Path, scenario: Scenario) -> None:
+    """Create sandbox directory and run setup commands."""
+    if sandbox_dir.exists():
+        shutil.rmtree(sandbox_dir)
+    sandbox_dir.mkdir(parents=True)
+
+    subprocess.run(["git", "init"], cwd=sandbox_dir, capture_output=True)
+
+    for cmd in scenario.setup_commands:
+        parts = shlex.split(cmd)
+        subprocess.run(parts, cwd=sandbox_dir, capture_output=True)
+
+
+def _parse_stream_json(stdout: str) -> list[ObservationEvent]:
+    """Parse claude -p stream-json output into ObservationEvents.
+
+    Stream-json format:
+    - type=assistant with content[].type=tool_use → tool call (name, input)
+    - type=user with content[].type=tool_result → tool result (output)
+    """
+    events: list[ObservationEvent] = []
+    pending: dict[str, dict] = {}
+    event_counter = 0
+
+    for line in stdout.strip().splitlines():
+        try:
+            msg = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        msg_type = msg.get("type")
+
+        if msg_type == "assistant":
+            content = msg.get("message", {}).get("content", [])
+            for block in content:
+                if block.get("type") == "tool_use":
+                    tool_use_id = block.get("id", "")
+                    tool_input = block.get("input", {})
+                    input_str = (
+                        json.dumps(tool_input)[:5000]
+                        if isinstance(tool_input, dict)
+                        else str(tool_input)[:5000]
+                    )
+                    pending[tool_use_id] = {
+                        "tool": block.get("name", "unknown"),
+                        "input": input_str,
+                        "order": event_counter,
+                    }
+                    event_counter += 1
+
+        elif msg_type == "user":
+            content = msg.get("message", {}).get("content", [])
+            if isinstance(content, list):
+                for block in content:
+                    tool_use_id = block.get("tool_use_id", "")
+                    if tool_use_id in pending:
+                        info = pending.pop(tool_use_id)
+                        output_content = block.get("content", "")
+                        if isinstance(output_content, list):
+                            output_str = json.dumps(output_content)[:5000]
+                        else:
+                            output_str = str(output_content)[:5000]
+
+                        events.append(ObservationEvent(
+                            timestamp=f"T{info['order']:04d}",
+                            event="tool_complete",
+                            tool=info["tool"],
+                            session=msg.get("session_id", "unknown"),
+                            input=info["input"],
+                            output=output_str,
+                        ))
+
+    for _tool_use_id, info in pending.items():
+        events.append(ObservationEvent(
+            timestamp=f"T{info['order']:04d}",
+            event="tool_complete",
+            tool=info["tool"],
+            session="unknown",
+            input=info["input"],
+            output="",
+        ))
+
+    return sorted(events, key=lambda e: e.timestamp)