@funnycode/myclaude 0.1.76 → 0.1.77

package/seed/marketplaces/ecc/skills/skill-comply/scripts/report.py

@@ -1,170 +1,170 @@
-"""Generate Markdown compliance reports."""
-
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from pathlib import Path
-
-from scripts.grader import ComplianceResult
-from scripts.parser import ComplianceSpec, ObservationEvent
-from scripts.scenario_generator import Scenario
-
-
-def generate_report(
-    skill_path: Path,
-    spec: ComplianceSpec,
-    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
-    scenarios: list[Scenario] | None = None,
-) -> str:
-    """Generate a Markdown compliance report.
-
-    Args:
-        skill_path: Path to the skill file that was tested.
-        spec: The compliance spec used for grading.
-        results: List of (scenario_level_name, ComplianceResult, observations) tuples.
-        scenarios: Original scenario definitions with prompts.
-    """
-    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    overall = _overall_compliance(results)
-    threshold = spec.threshold_promote_to_hook
-
-    lines: list[str] = []
-    lines.append(f"# skill-comply Report: {skill_path.name}")
-    lines.append(f"Generated: {now}")
-    lines.append("")
-
-    # Summary
-    lines.append("## Summary")
-    lines.append("")
-    lines.append(f"| Metric | Value |")
-    lines.append(f"|--------|-------|")
-    lines.append(f"| Skill | `{skill_path}` |")
-    lines.append(f"| Spec | {spec.id} |")
-    lines.append(f"| Scenarios | {len(results)} |")
-    lines.append(f"| Overall Compliance | {overall:.0%} |")
-    lines.append(f"| Threshold | {threshold:.0%} |")
-
-    promote_steps = _steps_to_promote(spec, results, threshold)
-    if promote_steps:
-        step_names = ", ".join(promote_steps)
-        lines.append(f"| Recommendation | **Promote {step_names} to hooks** |")
-    else:
-        lines.append(f"| Recommendation | All steps above threshold — no hook promotion needed |")
-    lines.append("")
-
-    # Expected Behavioral Sequence
-    lines.append("## Expected Behavioral Sequence")
-    lines.append("")
-    lines.append("| # | Step | Required | Description |")
-    lines.append("|---|------|----------|-------------|")
-    for i, step in enumerate(spec.steps, 1):
-        req = "Yes" if step.required else "No"
-        lines.append(f"| {i} | {step.id} | {req} | {step.detector.description} |")
-    lines.append("")
-
-    # Scenario Results
-    lines.append("## Scenario Results")
-    lines.append("")
-    lines.append("| Scenario | Compliance | Failed Steps |")
-    lines.append("|----------|-----------|----------------|")
-    for level_name, result, _obs in results:
-        failed = [s.step_id for s in result.steps if not s.detected
-                  and any(sp.id == s.step_id and sp.required for sp in spec.steps)]
-        failed_str = ", ".join(failed) if failed else "—"
-        lines.append(f"| {level_name} | {result.compliance_rate:.0%} | {failed_str} |")
-    lines.append("")
-
-    # Scenario Prompts
-    if scenarios:
-        lines.append("## Scenario Prompts")
-        lines.append("")
-        for s in scenarios:
-            lines.append(f"### {s.level_name} (Level {s.level})")
-            lines.append("")
-            for prompt_line in s.prompt.splitlines():
-                lines.append(f"> {prompt_line}")
-            lines.append("")
-
-    # Hook Promotion Recommendations (optional/advanced)
-    if promote_steps:
-        lines.append("## Advanced: Hook Promotion Recommendations (optional)")
-        lines.append("")
-        for step_id in promote_steps:
-            rate = _step_compliance_rate(step_id, results)
-            step = next(s for s in spec.steps if s.id == step_id)
-            lines.append(
-                f"- **{step_id}** (compliance {rate:.0%}): {step.description}"
-            )
-        lines.append("")
-
-    # Per-scenario details with timeline
-    lines.append("## Detail")
-    lines.append("")
-    for level_name, result, observations in results:
-        lines.append(f"### {level_name} (Compliance: {result.compliance_rate:.0%})")
-        lines.append("")
-        lines.append("| Step | Required | Detected | Reason |")
-        lines.append("|------|----------|----------|--------|")
-        for sr in result.steps:
-            req = "Yes" if any(
-                sp.id == sr.step_id and sp.required for sp in spec.steps
-            ) else "No"
-            det = "YES" if sr.detected else "NO"
-            reason = sr.failure_reason or "—"
-            lines.append(f"| {sr.step_id} | {req} | {det} | {reason} |")
-        lines.append("")
-
-        # Timeline: show what the agent actually did
-        if observations:
-            # Build reverse index: event_index → step_id
-            index_to_step: dict[int, str] = {}
-            for step_id, indices in result.classification.items():
-                for idx in indices:
-                    index_to_step[idx] = step_id
-
-            lines.append(f"**Tool Call Timeline ({len(observations)} calls)**")
-            lines.append("")
-            lines.append("| # | Tool | Input | Output | Classified As |")
-            lines.append("|---|------|-------|--------|------|")
-            for i, obs in enumerate(observations):
-                step_label = index_to_step.get(i, "—")
-                input_summary = obs.input[:100].replace("|", "\\|").replace("\n", " ")
-                output_summary = obs.output[:50].replace("|", "\\|").replace("\n", " ")
-                lines.append(
-                    f"| {i} | {obs.tool} | {input_summary} | {output_summary} | {step_label} |"
-                )
-            lines.append("")
-
-    return "\n".join(lines)
-
-
-def _overall_compliance(results: list[tuple[str, ComplianceResult, list[ObservationEvent]]]) -> float:
-    if not results:
-        return 0.0
-    return sum(r.compliance_rate for _, r, _obs in results) / len(results)
-
-
-def _step_compliance_rate(
-    step_id: str,
-    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
-) -> float:
-    detected = sum(
-        1 for _, r, _obs in results
-        for s in r.steps if s.step_id == step_id and s.detected
-    )
-    return detected / len(results) if results else 0.0
-
-
-def _steps_to_promote(
-    spec: ComplianceSpec,
-    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
-    threshold: float,
-) -> list[str]:
-    promote = []
-    for step in spec.steps:
-        if not step.required:
-            continue
-        rate = _step_compliance_rate(step.id, results)
-        if rate < threshold:
-            promote.append(step.id)
-    return promote
+"""Generate Markdown compliance reports."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+from scripts.grader import ComplianceResult
+from scripts.parser import ComplianceSpec, ObservationEvent
+from scripts.scenario_generator import Scenario
+
+
+def generate_report(
+    skill_path: Path,
+    spec: ComplianceSpec,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+    scenarios: list[Scenario] | None = None,
+) -> str:
+    """Generate a Markdown compliance report.
+
+    Args:
+        skill_path: Path to the skill file that was tested.
+        spec: The compliance spec used for grading.
+        results: List of (scenario_level_name, ComplianceResult, observations) tuples.
+        scenarios: Original scenario definitions with prompts.
+    """
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    overall = _overall_compliance(results)
+    threshold = spec.threshold_promote_to_hook
+
+    lines: list[str] = []
+    lines.append(f"# skill-comply Report: {skill_path.name}")
+    lines.append(f"Generated: {now}")
+    lines.append("")
+
+    # Summary
+    lines.append("## Summary")
+    lines.append("")
+    lines.append(f"| Metric | Value |")
+    lines.append(f"|--------|-------|")
+    lines.append(f"| Skill | `{skill_path}` |")
+    lines.append(f"| Spec | {spec.id} |")
+    lines.append(f"| Scenarios | {len(results)} |")
+    lines.append(f"| Overall Compliance | {overall:.0%} |")
+    lines.append(f"| Threshold | {threshold:.0%} |")
+
+    promote_steps = _steps_to_promote(spec, results, threshold)
+    if promote_steps:
+        step_names = ", ".join(promote_steps)
+        lines.append(f"| Recommendation | **Promote {step_names} to hooks** |")
+    else:
+        lines.append(f"| Recommendation | All steps above threshold — no hook promotion needed |")
+    lines.append("")
+
+    # Expected Behavioral Sequence
+    lines.append("## Expected Behavioral Sequence")
+    lines.append("")
+    lines.append("| # | Step | Required | Description |")
+    lines.append("|---|------|----------|-------------|")
+    for i, step in enumerate(spec.steps, 1):
+        req = "Yes" if step.required else "No"
+        lines.append(f"| {i} | {step.id} | {req} | {step.detector.description} |")
+    lines.append("")
+
+    # Scenario Results
+    lines.append("## Scenario Results")
+    lines.append("")
+    lines.append("| Scenario | Compliance | Failed Steps |")
+    lines.append("|----------|-----------|----------------|")
+    for level_name, result, _obs in results:
+        failed = [s.step_id for s in result.steps if not s.detected
+                  and any(sp.id == s.step_id and sp.required for sp in spec.steps)]
+        failed_str = ", ".join(failed) if failed else "—"
+        lines.append(f"| {level_name} | {result.compliance_rate:.0%} | {failed_str} |")
+    lines.append("")
+
+    # Scenario Prompts
+    if scenarios:
+        lines.append("## Scenario Prompts")
+        lines.append("")
+        for s in scenarios:
+            lines.append(f"### {s.level_name} (Level {s.level})")
+            lines.append("")
+            for prompt_line in s.prompt.splitlines():
+                lines.append(f"> {prompt_line}")
+            lines.append("")
+
+    # Hook Promotion Recommendations (optional/advanced)
+    if promote_steps:
+        lines.append("## Advanced: Hook Promotion Recommendations (optional)")
+        lines.append("")
+        for step_id in promote_steps:
+            rate = _step_compliance_rate(step_id, results)
+            step = next(s for s in spec.steps if s.id == step_id)
+            lines.append(
+                f"- **{step_id}** (compliance {rate:.0%}): {step.description}"
+            )
+        lines.append("")
+
+    # Per-scenario details with timeline
+    lines.append("## Detail")
+    lines.append("")
+    for level_name, result, observations in results:
+        lines.append(f"### {level_name} (Compliance: {result.compliance_rate:.0%})")
+        lines.append("")
+        lines.append("| Step | Required | Detected | Reason |")
+        lines.append("|------|----------|----------|--------|")
+        for sr in result.steps:
+            req = "Yes" if any(
+                sp.id == sr.step_id and sp.required for sp in spec.steps
+            ) else "No"
+            det = "YES" if sr.detected else "NO"
+            reason = sr.failure_reason or "—"
+            lines.append(f"| {sr.step_id} | {req} | {det} | {reason} |")
+        lines.append("")
+
+        # Timeline: show what the agent actually did
+        if observations:
+            # Build reverse index: event_index → step_id
+            index_to_step: dict[int, str] = {}
+            for step_id, indices in result.classification.items():
+                for idx in indices:
+                    index_to_step[idx] = step_id
+
+            lines.append(f"**Tool Call Timeline ({len(observations)} calls)**")
+            lines.append("")
+            lines.append("| # | Tool | Input | Output | Classified As |")
+            lines.append("|---|------|-------|--------|------|")
+            for i, obs in enumerate(observations):
+                step_label = index_to_step.get(i, "—")
+                input_summary = obs.input[:100].replace("|", "\\|").replace("\n", " ")
+                output_summary = obs.output[:50].replace("|", "\\|").replace("\n", " ")
+                lines.append(
+                    f"| {i} | {obs.tool} | {input_summary} | {output_summary} | {step_label} |"
+                )
+            lines.append("")
+
+    return "\n".join(lines)
+
+
+def _overall_compliance(results: list[tuple[str, ComplianceResult, list[ObservationEvent]]]) -> float:
+    if not results:
+        return 0.0
+    return sum(r.compliance_rate for _, r, _obs in results) / len(results)
+
+
+def _step_compliance_rate(
+    step_id: str,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+) -> float:
+    detected = sum(
+        1 for _, r, _obs in results
+        for s in r.steps if s.step_id == step_id and s.detected
+    )
+    return detected / len(results) if results else 0.0
+
+
+def _steps_to_promote(
+    spec: ComplianceSpec,
+    results: list[tuple[str, ComplianceResult, list[ObservationEvent]]],
+    threshold: float,
+) -> list[str]:
+    promote = []
+    for step in spec.steps:
+        if not step.required:
+            continue
+        rate = _step_compliance_rate(step.id, results)
+        if rate < threshold:
+            promote.append(step.id)
+    return promote

package/seed/marketplaces/ecc/skills/skill-comply/scripts/run.py

@@ -1,127 +1,127 @@
-"""CLI entry point for skill-comply."""
-
-from __future__ import annotations
-
-import argparse
-import logging
-import sys
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-from scripts.grader import grade
-from scripts.report import generate_report
-from scripts.runner import run_scenario
-from scripts.scenario_generator import generate_scenarios
-from scripts.spec_generator import generate_spec
-
-logger = logging.getLogger(__name__)
-
-
-def main() -> None:
-    logging.basicConfig(level=logging.INFO, format="%(message)s")
-
-    parser = argparse.ArgumentParser(
-        description="skill-comply: Measure skill compliance rates",
-    )
-    parser.add_argument(
-        "skill",
-        type=Path,
-        help="Path to skill/rule file to test",
-    )
-    parser.add_argument(
-        "--model",
-        default="sonnet",
-        help="Model for scenario execution (default: sonnet)",
-    )
-    parser.add_argument(
-        "--gen-model",
-        default="haiku",
-        help="Model for spec/scenario generation (default: haiku)",
-    )
-    parser.add_argument(
-        "--dry-run",
-        action="store_true",
-        help="Generate spec and scenarios without executing",
-    )
-    parser.add_argument(
-        "--output",
-        type=Path,
-        default=None,
-        help="Output report path (default: results/<skill-name>.md)",
-    )
-
-    args = parser.parse_args()
-
-    if not args.skill.is_file():
-        logger.error("Error: Skill file not found: %s", args.skill)
-        sys.exit(1)
-
-    results_dir = Path(__file__).parent.parent / "results"
-    results_dir.mkdir(exist_ok=True)
-
-    # Step 1: Generate compliance spec
-    logger.info("[1/4] Generating compliance spec from %s...", args.skill.name)
-    spec = generate_spec(args.skill, model=args.gen_model)
-    logger.info("       %d steps extracted", len(spec.steps))
-
-    # Step 2: Generate scenarios
-    spec_yaml = yaml.dump({
-        "steps": [
-            {"id": s.id, "description": s.description, "required": s.required}
-            for s in spec.steps
-        ]
-    })
-    logger.info("[2/4] Generating scenarios (3 prompt strictness levels)...")
-    scenarios = generate_scenarios(args.skill, spec_yaml, model=args.gen_model)
-    logger.info("       %d scenarios generated", len(scenarios))
-
-    for s in scenarios:
-        logger.info("       - %s: %s", s.level_name, s.description[:60])
-
-    if args.dry_run:
-        logger.info("\n[dry-run] Spec and scenarios generated. Skipping execution.")
-        logger.info("\nSpec: %s (%d steps)", spec.id, len(spec.steps))
-        for step in spec.steps:
-            marker = "*" if step.required else " "
-            logger.info("  [%s] %s: %s", marker, step.id, step.description)
-        return
-
-    # Step 3: Execute scenarios
-    logger.info("[3/4] Executing scenarios (model=%s)...", args.model)
-    graded_results: list[tuple[str, Any, list[Any]]] = []
-
-    for scenario in scenarios:
-        logger.info("       Running %s...", scenario.level_name)
-        run = run_scenario(scenario, model=args.model)
-        result = grade(spec, list(run.observations))
-        graded_results.append((scenario.level_name, result, list(run.observations)))
-        logger.info("       %s: %.0f%%", scenario.level_name, result.compliance_rate * 100)
-
-    # Step 4: Generate report
-    skill_name = args.skill.parent.name if args.skill.stem == "SKILL" else args.skill.stem
-    output_path = args.output or results_dir / f"{skill_name}.md"
-    logger.info("[4/4] Generating report...")
-
-    report = generate_report(args.skill, spec, graded_results, scenarios=scenarios)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(report)
-    logger.info("       Report saved to %s", output_path)
-
-    # Summary
-    if not graded_results:
-        logger.warning("No scenarios were executed.")
-        return
-    overall = sum(r.compliance_rate for _, r, _obs in graded_results) / len(graded_results)
-    logger.info("\n%s", "=" * 50)
-    logger.info("Overall Compliance: %.0f%%", overall * 100)
-    if overall < spec.threshold_promote_to_hook:
-        logger.info(
-            "Recommendation: Some steps have low compliance. "
-            "Consider promoting them to hooks. See the report for details."
-        )
-
-
-if __name__ == "__main__":
-    main()
+"""CLI entry point for skill-comply."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from scripts.grader import grade
+from scripts.report import generate_report
+from scripts.runner import run_scenario
+from scripts.scenario_generator import generate_scenarios
+from scripts.spec_generator import generate_spec
+
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+    parser = argparse.ArgumentParser(
+        description="skill-comply: Measure skill compliance rates",
+    )
+    parser.add_argument(
+        "skill",
+        type=Path,
+        help="Path to skill/rule file to test",
+    )
+    parser.add_argument(
+        "--model",
+        default="sonnet",
+        help="Model for scenario execution (default: sonnet)",
+    )
+    parser.add_argument(
+        "--gen-model",
+        default="haiku",
+        help="Model for spec/scenario generation (default: haiku)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Generate spec and scenarios without executing",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="Output report path (default: results/<skill-name>.md)",
+    )
+
+    args = parser.parse_args()
+
+    if not args.skill.is_file():
+        logger.error("Error: Skill file not found: %s", args.skill)
+        sys.exit(1)
+
+    results_dir = Path(__file__).parent.parent / "results"
+    results_dir.mkdir(exist_ok=True)
+
+    # Step 1: Generate compliance spec
+    logger.info("[1/4] Generating compliance spec from %s...", args.skill.name)
+    spec = generate_spec(args.skill, model=args.gen_model)
+    logger.info("       %d steps extracted", len(spec.steps))
+
+    # Step 2: Generate scenarios
+    spec_yaml = yaml.dump({
+        "steps": [
+            {"id": s.id, "description": s.description, "required": s.required}
+            for s in spec.steps
+        ]
+    })
+    logger.info("[2/4] Generating scenarios (3 prompt strictness levels)...")
+    scenarios = generate_scenarios(args.skill, spec_yaml, model=args.gen_model)
+    logger.info("       %d scenarios generated", len(scenarios))
+
+    for s in scenarios:
+        logger.info("       - %s: %s", s.level_name, s.description[:60])
+
+    if args.dry_run:
+        logger.info("\n[dry-run] Spec and scenarios generated. Skipping execution.")
+        logger.info("\nSpec: %s (%d steps)", spec.id, len(spec.steps))
+        for step in spec.steps:
+            marker = "*" if step.required else " "
+            logger.info("  [%s] %s: %s", marker, step.id, step.description)
+        return
+
+    # Step 3: Execute scenarios
+    logger.info("[3/4] Executing scenarios (model=%s)...", args.model)
+    graded_results: list[tuple[str, Any, list[Any]]] = []
+
+    for scenario in scenarios:
+        logger.info("       Running %s...", scenario.level_name)
+        run = run_scenario(scenario, model=args.model)
+        result = grade(spec, list(run.observations))
+        graded_results.append((scenario.level_name, result, list(run.observations)))
+        logger.info("       %s: %.0f%%", scenario.level_name, result.compliance_rate * 100)
+
+    # Step 4: Generate report
+    skill_name = args.skill.parent.name if args.skill.stem == "SKILL" else args.skill.stem
+    output_path = args.output or results_dir / f"{skill_name}.md"
+    logger.info("[4/4] Generating report...")
+
+    report = generate_report(args.skill, spec, graded_results, scenarios=scenarios)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report)
+    logger.info("       Report saved to %s", output_path)
+
+    # Summary
+    if not graded_results:
+        logger.warning("No scenarios were executed.")
+        return
+    overall = sum(r.compliance_rate for _, r, _obs in graded_results) / len(graded_results)
+    logger.info("\n%s", "=" * 50)
+    logger.info("Overall Compliance: %.0f%%", overall * 100)
+    if overall < spec.threshold_promote_to_hook:
+        logger.info(
+            "Recommendation: Some steps have low compliance. "
+            "Consider promoting them to hooks. See the report for details."
+        )
+
+
+if __name__ == "__main__":
+    main()