@event4u/agent-config 5.4.1 → 5.6.0

package/scripts/skill_preview.py

@@ -0,0 +1,179 @@
+"""Skill preview — non-destructive "what will this skill do?" summary.
+
+Phase 5 of `road-to-leaner-core-and-discovery`. Reads a skill's declared intent
+(frontmatter + `## Steps` body) and renders a plain-language summary BEFORE the
+skill runs. Read-only, no network, no execution.
+
+NOT a sandbox: it surfaces declared intent, it does not run the skill or prove
+side-effect-freeness (contract: docs/contracts/skill-dry-run.md). For
+`execution: manual` skills (the default) it states "instructional only".
+
+Usage:
+    python3 scripts/skill_preview.py <name> [--technical] [--format text|json]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+SKILLS_DIR = REPO_ROOT / ".agent-src" / "skills"
+
+_CMD_RE = re.compile(r"`(python3?|bash|node|php|npm|task|pytest)\s+[^`]+`")
+_PATH_RE = re.compile(r"`([\w./-]+\.(?:py|sh|md|json|yml|yaml|ts|js|php))`")
+
+
+class PreviewError(Exception):
+    """Raised for a missing or malformed SKILL.md — rendered, never crashed on."""
+
+
+def _split_frontmatter(text: str) -> tuple[dict, str]:
+    if not text.startswith("---"):
+        raise PreviewError("SKILL.md has no YAML frontmatter (missing leading `---`).")
+    end = text.find("\n---", 3)
+    if end == -1:
+        raise PreviewError("SKILL.md frontmatter is not closed (missing terminating `---`).")
+    try:
+        fm = yaml.safe_load(text[3:end]) or {}
+    except yaml.YAMLError as exc:
+        raise PreviewError(f"SKILL.md frontmatter is not valid YAML: {exc}")
+    if not isinstance(fm, dict):
+        raise PreviewError("SKILL.md frontmatter did not parse to a mapping.")
+    return fm, text[end + 4:]
+
+
+def _steps(body: str) -> list[str]:
+    out: list[str] = []
+    in_steps = False
+    for line in body.splitlines():
+        if re.match(r"^##\s+Steps\b", line, re.IGNORECASE):
+            in_steps = True
+            continue
+        if in_steps and re.match(r"^##\s+\S", line):  # next top-level section
+            break
+        if in_steps:
+            m = re.match(r"^###\s+(.*)", line)
+            if m:
+                out.append(m.group(1).strip())
+    return out
+
+
+def _targets(body: str) -> tuple[list[str], list[str]]:
+    cmds = sorted({m.group(0).strip("`") for m in _CMD_RE.finditer(body)})
+    paths = sorted({m.group(1) for m in _PATH_RE.finditer(body)})
+    return cmds, paths
+
+
+def load_preview(name: str) -> dict:
+    skill_dir = SKILLS_DIR / name
+    sk = skill_dir / "SKILL.md"
+    if not sk.is_file():
+        try:
+            shown = sk.relative_to(REPO_ROOT)
+        except ValueError:
+            shown = sk
+        raise PreviewError(f"no skill named {name!r} (looked for {shown}).")
+    fm, body = _split_frontmatter(sk.read_text(encoding="utf-8", errors="replace"))
+    execution = fm.get("execution") or {}
+    if not isinstance(execution, dict):
+        execution = {}
+    cmds, paths = _targets(body)
+    return {
+        "name": fm.get("name") or name,
+        "description": (fm.get("description") or "").strip(),
+        "domain": fm.get("domain") or "",
+        "execution_type": (execution.get("type") or "manual"),
+        "handler": (execution.get("handler") or "none"),
+        "allowed_tools": execution.get("allowed_tools") or [],
+        "command": execution.get("command") or [],
+        "steps": _steps(body),
+        "commands_named": cmds,
+        "paths_named": paths,
+    }
+
+
+def render_plain(p: dict) -> str:
+    lines = [f"# Preview — `{p['name']}`", ""]
+    if p["description"]:
+        lines += [p["description"], ""]
+    etype = p["execution_type"]
+    if etype == "manual":
+        lines.append("**Execution: instructional only.** This skill does not run anything "
+                     "automatically — it guides the agent step by step.")
+    elif etype == "assisted":
+        lines.append(f"**Execution: assisted** (handler `{p['handler']}`). It will *propose* actions "
+                     "for you to approve — it never executes silently.")
+    else:
+        lines.append(f"**Execution: {etype}** (handler `{p['handler']}`). It can run actions; review the "
+                     "declared tools and commands below before allowing it.")
+    lines.append("")
+    if p["steps"]:
+        lines.append("This skill will walk these steps:")
+        lines += [f"- {s}" for s in p["steps"]]
+        lines.append("")
+    if p["allowed_tools"]:
+        lines.append(f"Declared tools: {', '.join(p['allowed_tools'])}")
+    if p["command"]:
+        lines.append(f"Declared command: `{' '.join(str(c) for c in p['command'])}`")
+    if p["commands_named"]:
+        lines.append("Commands it may run:")
+        lines += [f"- `{c}`" for c in p["commands_named"]]
+    if p["paths_named"]:
+        lines.append("Files / scripts it references:")
+        lines += [f"- `{f}`" for f in p["paths_named"]]
+    if not (p["allowed_tools"] or p["command"] or p["commands_named"] or p["paths_named"]):
+        lines.append("_No tools, commands, or file targets declared — pure guidance._")
+    lines.append("")
+    lines.append("> Preview shows declared intent only — it does not run the skill or guarantee "
+                 "side-effect-freeness. Contract: docs/contracts/skill-dry-run.md")
+    return "\n".join(lines)
+
+
+def render_technical(p: dict) -> str:
+    lines = [f"# Preview (technical) — {p['name']}", "", "## Frontmatter (execution)", "```yaml"]
+    lines.append(f"execution_type: {p['execution_type']}")
+    lines.append(f"handler: {p['handler']}")
+    lines.append(f"allowed_tools: {p['allowed_tools']}")
+    if p["command"]:
+        lines.append(f"command: {p['command']}")
+    lines += ["```", "", "## Declared steps"]
+    lines += [f"{i+1}. {s}" for i, s in enumerate(p["steps"])] or ["(none)"]
+    if p["commands_named"]:
+        lines += ["", "## Commands named in body"] + [f"- `{c}`" for c in p["commands_named"]]
+    if p["paths_named"]:
+        lines += ["", "## Paths named in body"] + [f"- `{f}`" for f in p["paths_named"]]
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="Preview a skill's declared intent (read-only, no execution).")
+    ap.add_argument("name", help="Skill name (directory under .agent-src/skills/).")
+    ap.add_argument("--technical", action="store_true", help="Show raw frontmatter + step list.")
+    ap.add_argument("--format", choices=("text", "json"), default="text")
+    args = ap.parse_args(argv)
+
+    try:
+        preview = load_preview(args.name)
+    except PreviewError as exc:
+        if args.format == "json":
+            print(json.dumps({"error": str(exc), "name": args.name}, indent=2))
+        else:
+            print(f"❌  Cannot preview {args.name!r}: {exc}", file=sys.stderr)
+        return 2
+
+    if args.format == "json":
+        print(json.dumps(preview, indent=2))
+    elif args.technical:
+        print(render_technical(preview))
+    else:
+        print(render_plain(preview))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/trigger_coverage.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Trigger-coverage suite (roadmap Phase 2.1 / 2.2).
+
+The deterministic *must-load* floor for the lean-initial-context migration.
+Before any auto-tier rule body is demoted to a router-resolved pointer
+(Phase 3), this suite proves the router still fires that rule on
+representative task phrasings — so a needed rule can never silently fail
+to surface.
+
+Cases live in `tests/eval/trigger-coverage.yaml` and have the shape:
+
+    - id: secrets-edit
+      prompt: "add a webhook secret to the billing service auth flow"
+      expect: [security-sensitive-stop]   # MUST be in the fired set
+
+Matching is deterministic against `dist/router.json` (NOT the semantic
+production router — this is a reproducible floor that catches a removed
+trigger in CI):
+
+- kernel rules always fire (always-on layer).
+- a tier rule fires iff any of its triggers matches the prompt:
+  - `keyword` → case-insensitive substring.
+  - `intent`  → every alpha word (len>2) of the intent phrase appears as a
+    token in the prompt (so "structural decision" fires on a prompt that
+    contains both "structural" and "decision").
+
+A case fails when an expected rule is NOT in the fired set. Exit 1 on any
+miss → the merge that would have shrunk the rule is blocked (2.2).
+
+Usage:
+    python3 scripts/trigger_coverage.py            # run, human report
+    python3 scripts/trigger_coverage.py --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+ROUTER = REPO_ROOT / "dist" / "router.json"
+CORPUS = REPO_ROOT / "tests" / "eval" / "trigger-coverage.yaml"
+
+try:
+    import yaml
+except ImportError:  # pragma: no cover
+    sys.stderr.write("error: PyYAML required (pip install pyyaml)\n")
+    sys.exit(2)
+
+_WORD = re.compile(r"[a-z][a-z0-9_]+")
+
+
+def _tokens(text: str) -> set[str]:
+    return {w for w in _WORD.findall(text.lower()) if len(w) > 2}
+
+
+def load_router() -> dict:
+    return json.loads(ROUTER.read_text(encoding="utf-8"))
+
+
+def fired_rules(prompt: str, router: dict) -> set[str]:
+    """Return every rule id the router would surface for `prompt`."""
+    low = prompt.lower()
+    toks = _tokens(prompt)
+    fired: set[str] = set(router.get("kernel", []))
+    for tier in ("tier_1", "tier_2"):
+        for entry in router.get(tier, []):
+            for trig in entry.get("triggers", []):
+                if "keyword" in trig:
+                    if trig["keyword"].lower() in low:
+                        fired.add(entry["id"])
+                        break
+                elif "intent" in trig:
+                    words = _tokens(trig["intent"])
+                    if words and words <= toks:
+                        fired.add(entry["id"])
+                        break
+    return fired
+
+
+def run(corpus: list[dict], router: dict) -> tuple[list[dict], int]:
+    results = []
+    misses = 0
+    for case in corpus:
+        fired = fired_rules(case["prompt"], router)
+        expected = case.get("expect", [])
+        missing = [r for r in expected if r not in fired]
+        ok = not missing
+        if not ok:
+            misses += 1
+        results.append({"id": case["id"], "ok": ok, "missing": missing,
+                        "expect": expected})
+    return results, misses
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args(argv)
+
+    if not ROUTER.is_file():
+        sys.stderr.write(f"error: {ROUTER} missing — run compile_router first\n")
+        return 2
+    corpus = yaml.safe_load(CORPUS.read_text(encoding="utf-8")) or []
+    router = load_router()
+    results, misses = run(corpus, router)
+
+    if args.json:
+        print(json.dumps({"cases": len(results), "misses": misses,
+                          "results": results}, indent=2, sort_keys=True))
+    else:
+        for r in results:
+            mark = "✅" if r["ok"] else "❌"
+            detail = "" if r["ok"] else f"  MISSING: {', '.join(r['missing'])}"
+            print(f"  {mark}  {r['id']}{detail}")
+        print()
+        if misses:
+            print(f"❌  trigger-coverage: {misses}/{len(results)} case(s) failed — "
+                  "a required rule does not fire. Blocking.")
+        else:
+            print(f"✅  trigger-coverage: {len(results)}/{len(results)} pass")
+    return 1 if misses else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())