@event4u/agent-config 2.18.0 → 2.20.0

package/scripts/skill_tools/audit_user_type_coverage.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Block D · D3 — audit_user_type_coverage.
+
+Coverage audit for the user-type axis. User-types are **CLI-only** in v1
+(see `docs/contracts/adr-user-types-axis.md` and Phase 4 step 3 of
+`agents/roadmaps/step-6-user-types-axis.md`) — skills do NOT declare a
+`user-types:` frontmatter key, so persona-style citation counting does
+not apply. Instead this script:
+
+  - Inventories every user-type file in the source directory.
+  - Scans skills, commands, and `docs/` for `--user-type=<id>` mentions.
+  - Flags **orphan references** (CLI mention to a non-existent id) and
+    **never-referenced** user-types (file exists but nobody cites it).
+
+Inputs:
+  --user-types-dir DIR — directory holding user-type Markdown files
+  --search-root DIR    — root to recurse for `--user-type=<id>` mentions
+  --json               — machine-readable output
+
+Output: per-user-type reference count + status (ok / never-referenced /
+orphan). Exit code: 0 always (advisory, not a CI gate).
+
+Stdlib-only. ≤ 130 LOC. Sibling of `audit_persona_coverage.py`.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Set
+
+ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_USER_TYPES = ROOT / ".agent-src.uncompressed" / "user-types"
+DEFAULT_SEARCH_ROOT = ROOT / ".agent-src.uncompressed"
+REFERENCE_THRESHOLD = 1  # user-type with 0 references → flagged.
+
+# Matches `--user-type=<id>` in command markdown, skill prose, docs.
+_REFERENCE_RE = re.compile(r"--user-type=([\w-]+)")
+
+
+def _read_block(path: Path) -> str:
+    text = path.read_text(encoding="utf-8", errors="replace")
+    if not text.startswith("---"):
+        return ""
+    end = text.find("\n---", 3)
+    return text[3:end] if end != -1 else ""
+
+
+def _frontmatter_value(block: str, key: str) -> str | None:
+    m = re.search(rf"^{re.escape(key)}\s*:\s*(.+)$", block, re.MULTILINE)
+    if not m:
+        return None
+    val = m.group(1).strip()
+    if val.startswith('"') and val.endswith('"'):
+        val = val[1:-1]
+    return val
+
+
+def _load_user_types(user_types_dir: Path) -> Set[str]:
+    ids: Set[str] = set()
+    if not user_types_dir.is_dir():
+        return ids
+    for md in sorted(user_types_dir.glob("*.md")):
+        if md.name.lower() == "readme.md":
+            continue
+        block = _read_block(md)
+        slug = _frontmatter_value(block, "id") or md.stem
+        ids.add(slug)
+    # Walk one level deeper to skip `_template/` etc.
+    for md in sorted(user_types_dir.glob("*/*.md")):
+        if "_template" in md.parts:
+            continue
+        block = _read_block(md)
+        slug = _frontmatter_value(block, "id") or md.parent.name
+        ids.add(slug)
+    return ids
+
+
+def _count_references(search_root: Path, skip_dir: Path) -> Dict[str, int]:
+    counts: Dict[str, int] = {}
+    if not search_root.is_dir():
+        return counts
+    skip_resolved = skip_dir.resolve() if skip_dir.is_dir() else None
+    for md in search_root.rglob("*.md"):
+        # Don't count references inside the user-types dir itself
+        # (the README documents the flag in example form).
+        if skip_resolved and skip_resolved in md.resolve().parents:
+            continue
+        text = md.read_text(encoding="utf-8", errors="replace")
+        for slug in _REFERENCE_RE.findall(text):
+            counts[slug] = counts.get(slug, 0) + 1
+    return counts
+
+
+def audit(user_types_dir: Path, search_root: Path) -> List[Dict[str, object]]:
+    ids = _load_user_types(user_types_dir)
+    references = _count_references(search_root, user_types_dir)
+    rows: List[Dict[str, object]] = []
+    for slug in sorted(ids):
+        count = references.get(slug, 0)
+        status = "ok" if count >= REFERENCE_THRESHOLD else "never-referenced"
+        rows.append({"user_type": slug, "references": count,
+                     "threshold": REFERENCE_THRESHOLD, "status": status})
+    for slug in sorted(references.keys()):
+        if slug not in ids:
+            rows.append({"user_type": slug, "references": references[slug],
+                         "threshold": REFERENCE_THRESHOLD, "status": "orphan"})
+    return rows
+
+
+def _print_human(rows: List[Dict[str, object]]) -> None:
+    if not rows:
+        print("(no user-types found)")
+        return
+    width = max(len(str(r["user_type"])) for r in rows)
+    print(f"  {'user-type':<{width}}  refs   status")
+    print(f"  {'-' * width}  -----  ----------------")
+    for r in rows:
+        print(f"  {str(r['user_type']):<{width}}  "
+              f"{int(r['references']):>5}  {r['status']}")
+    flagged = [r for r in rows if r["status"] != "ok"]
+    if flagged:
+        print(f"\n  {len(flagged)} user-type(s) flagged "
+              f"(never-referenced or orphan).")
+
+
+def main(argv: List[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+    parser.add_argument("--user-types-dir", default=str(DEFAULT_USER_TYPES))
+    parser.add_argument("--search-root", default=str(DEFAULT_SEARCH_ROOT))
+    parser.add_argument("--json", action="store_true",
+                        help="emit JSON instead of text")
+    args = parser.parse_args(argv)
+    rows = audit(Path(args.user_types_dir), Path(args.search_root))
+    if args.json:
+        json.dump({"rows": rows}, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    else:
+        _print_human(rows)
+    return 0
+
+
+_SAMPLE = {"threshold": REFERENCE_THRESHOLD}
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/skill_usage_collect.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Collect skill-activation signal from Claude Code session jsonl.
+
+Implements step-2-skill-inventory-rationalization.md Phase 1 Step 2.
+Reads `~/.claude/projects/<project-slug>/*.jsonl` for the current repo,
+parses each turn for two signals:
+
+  - exposure: the skill slug appeared in an `attachment.type=skill_listing`
+    payload (catalog presented to the agent that turn).
+  - mention:  the assistant-text response in the same or following turn
+    referenced the slug in backticks with one of the anchor verbs
+    (using, via, per, route, dispatch, invoke, call) OR cited a SKILL.md
+    path under `.augment/skills/<slug>/`, `.claude/skills/<slug>/`, or
+    `.agent-src/skills/<slug>/`.
+
+Emits one JSONL record per (session, turn, slug, kind) to
+`agents/metrics/skill-usage.jsonl` (append-only, deduped on the
+(session_id, turn_idx, slug, kind) tuple).
+
+Privacy: `prompt_excerpt_hash` = SHA-256 of the first 200 chars of the
+user prompt that opened the turn. No raw user or assistant bodies are
+persisted. See `agents/audit-2026-05-14-north-star/skill-usage-sources.md`.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Iterable, Iterator
+
+REPO = Path(__file__).resolve().parent.parent
+OUT = REPO / "agents" / "metrics" / "skill-usage.jsonl"
+
+LISTING_LINE_RE = re.compile(r"^-\s+([a-z0-9][a-z0-9_-]+):\s", re.MULTILINE)
+ANCHOR_VERBS = ("using", "via", "per", "route", "routing", "dispatch", "dispatched", "invoke", "call")
+PATH_RE = re.compile(r"\.(?:augment|claude|agent-src)/skills/([a-z0-9][a-z0-9_-]+)/SKILL\.md")
+
+
+def project_slug(repo: Path) -> str:
+    return str(repo).replace("/", "-")
+
+
+def session_files(slug: str) -> list[Path]:
+    base = Path.home() / ".claude" / "projects" / slug
+    if not base.is_dir():
+        return []
+    return sorted(base.glob("*.jsonl"))
+
+
+def iter_turns(jsonl: Path) -> Iterator[dict]:
+    with jsonl.open("r", encoding="utf-8", errors="replace") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+
+def extract_listing(entry: dict) -> set[str]:
+    att = entry.get("attachment") or {}
+    if att.get("type") != "skill_listing":
+        return set()
+    content = att.get("content", "") or ""
+    return set(LISTING_LINE_RE.findall(content))
+
+
+def extract_text(entry: dict) -> str:
+    if entry.get("type") != "assistant":
+        return ""
+    msg = entry.get("message") or {}
+    content = msg.get("content")
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text")
+    return ""
+
+
+def find_mentions(text: str, known_slugs: Iterable[str]) -> set[str]:
+    hits: set[str] = set()
+    if not text:
+        return hits
+    hits.update(PATH_RE.findall(text))
+    for slug in known_slugs:
+        token = f"`{slug}`"
+        if token not in text:
+            continue
+        lower = text.lower()
+        for verb in ANCHOR_VERBS:
+            if f"{verb} {token}".lower() in lower or f"{verb} the {token}".lower() in lower:
+                hits.add(slug)
+                break
+    return hits
+
+
+def hash_prompt(text: str) -> str:
+    if not text:
+        return ""
+    return hashlib.sha256(text[:200].encode("utf-8", errors="replace")).hexdigest()[:16]
+
+
+def collect_session(jsonl: Path, all_known: set[str]) -> list[dict]:
+    session_id = jsonl.stem
+    records: list[dict] = []
+    last_prompt_hash = ""
+    listed: set[str] = set()
+    turn_idx = -1
+    for entry in iter_turns(jsonl):
+        etype = entry.get("type")
+        if etype == "user":
+            turn_idx += 1
+            msg = entry.get("message") or {}
+            body = msg.get("content") if isinstance(msg.get("content"), str) else ""
+            last_prompt_hash = hash_prompt(body or "")
+            continue
+        if etype == "attachment":
+            listed |= extract_listing(entry)
+            continue
+        if etype == "assistant":
+            text = extract_text(entry)
+            mentions = find_mentions(text, listed | all_known)
+            ts = entry.get("timestamp") or ""
+            for slug in sorted(listed):
+                records.append({"session_id": session_id, "turn_idx": turn_idx, "slug": slug,
+                                "kind": "exposure", "ts": ts, "prompt_excerpt_hash": last_prompt_hash})
+            for slug in sorted(mentions):
+                records.append({"session_id": session_id, "turn_idx": turn_idx, "slug": slug,
+                                "kind": "mention", "ts": ts, "prompt_excerpt_hash": last_prompt_hash})
+            listed = set()
+    return records
+
+
+def load_known_slugs(repo: Path) -> set[str]:
+    slugs: set[str] = set()
+    for root in (repo / ".augment" / "skills", repo / ".claude" / "skills", repo / ".agent-src" / "skills"):
+        if not root.is_dir():
+            continue
+        for skill_md in root.glob("*/SKILL.md"):
+            slugs.add(skill_md.parent.name)
+    return slugs
+
+
+def dedup_key(rec: dict) -> tuple:
+    return (rec["session_id"], rec["turn_idx"], rec["slug"], rec["kind"])
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--project-slug", help="Override the ~/.claude/projects slug (defaults to current repo)")
+    ap.add_argument("--out", type=Path, default=OUT, help="Output jsonl (default: agents/metrics/skill-usage.jsonl)")
+    ap.add_argument("--quiet", action="store_true", help="Suppress non-error output")
+    args = ap.parse_args()
+
+    slug = args.project_slug or project_slug(REPO)
+    files = session_files(slug)
+    if not files:
+        if not args.quiet:
+            print(f"no session files for slug {slug}", file=sys.stderr)
+        return 0
+    known = load_known_slugs(REPO)
+    seen: set[tuple] = set()
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    if args.out.exists():
+        for line in args.out.read_text(encoding="utf-8", errors="replace").splitlines():
+            try:
+                seen.add(dedup_key(json.loads(line)))
+            except (json.JSONDecodeError, KeyError):
+                continue
+    appended = 0
+    with args.out.open("a", encoding="utf-8") as fh:
+        for jsonl in files:
+            for rec in collect_session(jsonl, known):
+                k = dedup_key(rec)
+                if k in seen:
+                    continue
+                seen.add(k)
+                fh.write(json.dumps(rec, separators=(",", ":")) + "\n")
+                appended += 1
+    if not args.quiet:
+        print(f"✅  Wrote {appended} new record(s) to {args.out.relative_to(REPO)} ({len(seen)} total)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/skill_usage_report.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Aggregate `agents/metrics/skill-usage.jsonl` into a per-skill report.
+
+Implements step-2-skill-inventory-rationalization.md Phase 1 Step 3.
+Groups records by slug; emits `agents/metrics/skill-usage-report.md`
+with columns:
+
+  slug · exposures_total · mentions_total · exposures_30d · mentions_30d
+       · last_seen · status
+
+`status` ∈ { active, exposed-only, dead } per:
+
+  active        = mentions_30d ≥ 1
+  exposed-only  = exposures_30d ≥ 1 ∧ mentions_30d == 0
+  dead          = exposures_30d == 0
+
+The report is **a baseline, not a verdict**. Rationalization decisions
+live in Phase 2 (`skill-rationalization-candidates.md`).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+IN = REPO / "agents" / "metrics" / "skill-usage.jsonl"
+OUT = REPO / "agents" / "metrics" / "skill-usage-report.md"
+
+
+def parse_ts(raw: str) -> datetime | None:
+    if not raw:
+        return None
+    try:
+        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+def load_records(path: Path) -> list[dict]:
+    if not path.exists():
+        return []
+    records: list[dict] = []
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        if not line.strip():
+            continue
+        try:
+            records.append(json.loads(line))
+        except json.JSONDecodeError:
+            continue
+    return records
+
+
+def aggregate(records: list[dict], now: datetime, window_days: int = 30) -> dict[str, dict]:
+    cutoff = now - timedelta(days=window_days)
+    per: dict[str, dict] = defaultdict(lambda: {
+        "exposures_total": 0, "mentions_total": 0,
+        "exposures_30d": 0, "mentions_30d": 0,
+        "last_seen": None,
+    })
+    for rec in records:
+        slug = rec.get("slug")
+        kind = rec.get("kind")
+        if not slug or kind not in ("exposure", "mention"):
+            continue
+        ts = parse_ts(rec.get("ts") or "")
+        bucket = per[slug]
+        bucket[f"{kind}s_total"] += 1
+        if ts and ts >= cutoff:
+            bucket[f"{kind}s_30d"] += 1
+        if ts and (bucket["last_seen"] is None or ts > bucket["last_seen"]):
+            bucket["last_seen"] = ts
+    return per
+
+
+def status_for(row: dict) -> str:
+    if row["mentions_30d"] >= 1:
+        return "active"
+    if row["exposures_30d"] >= 1:
+        return "exposed-only"
+    return "dead"
+
+
+def all_known_slugs(repo: Path) -> set[str]:
+    slugs: set[str] = set()
+    for root in (repo / ".augment" / "skills", repo / ".claude" / "skills", repo / ".agent-src" / "skills"):
+        if not root.is_dir():
+            continue
+        for skill_md in root.glob("*/SKILL.md"):
+            slugs.add(skill_md.parent.name)
+    return slugs
+
+
+def render(per: dict[str, dict], known: set[str]) -> str:
+    rows = []
+    for slug in sorted(known | set(per)):
+        data = per.get(slug, {
+            "exposures_total": 0, "mentions_total": 0,
+            "exposures_30d": 0, "mentions_30d": 0, "last_seen": None,
+        })
+        rows.append({"slug": slug, **data, "status": status_for(data)})
+    rows.sort(key=lambda r: (r["status"] != "dead", -r["exposures_total"], r["slug"]))
+
+    counts = {"active": 0, "exposed-only": 0, "dead": 0}
+    for r in rows:
+        counts[r["status"]] += 1
+    total = len(rows)
+
+    lines = [
+        "# Skill Usage Report (baseline)",
+        "",
+        "> Generated by `scripts/skill_usage_report.py`. Source:",
+        "> `agents/metrics/skill-usage.jsonl` (collector emits per-turn",
+        "> exposure/mention records). See",
+        "> [`step-2-skill-inventory-rationalization.md`](../roadmaps/step-2-skill-inventory-rationalization.md)",
+        "> Phase 1.",
+        "",
+        f"**Window:** 30-day rolling \u00b7 **Skills tracked:** {total} \u00b7 "
+        f"**Active:** {counts['active']} \u00b7 **Exposed-only:** {counts['exposed-only']} \u00b7 "
+        f"**Dead:** {counts['dead']}",
+        "",
+        "| # | slug | status | exposures_30d | mentions_30d | exposures_total | mentions_total | last_seen |",
+        "|---|---|---|---|---|---|---|---|",
+    ]
+    for i, r in enumerate(rows, 1):
+        last = r["last_seen"].date().isoformat() if r["last_seen"] else "\u2014"
+        lines.append(
+            f"| {i} | `{r['slug']}` | {r['status']} | {r['exposures_30d']} | "
+            f"{r['mentions_30d']} | {r['exposures_total']} | {r['mentions_total']} | {last} |"
+        )
+    lines.append("")
+    lines.append("**Read-out:** rows tagged `dead` are first-cut archive candidates; "
+                 "rows tagged `exposed-only` are first-cut merge / rename candidates "
+                 "(catalog noise, agent never invokes). Phase 2 confirms with "
+                 "structural overlap before any deletion.")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--in", dest="inp", type=Path, default=IN)
+    ap.add_argument("--out", type=Path, default=OUT)
+    ap.add_argument("--window", type=int, default=30, help="Rolling window in days")
+    ap.add_argument("--quiet", action="store_true")
+    args = ap.parse_args()
+
+    records = load_records(args.inp)
+    now = datetime.now(timezone.utc)
+    per = aggregate(records, now, args.window)
+    known = all_known_slugs(REPO)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(render(per, known), encoding="utf-8")
+    if not args.quiet:
+        print(f"\u2705  Wrote {args.out.relative_to(REPO)} ({len(known | set(per))} skill(s))")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/smoke/kernel.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# scripts/smoke/kernel.sh — kernel-tier smoke (step-11 Phase 3 Step 2).
+#
+# Asserts:
+#   1. router.json lists exactly 9 kernel rules.
+#   2. Every kernel rule file exists at .agent-src/rules/<id>.md.
+#   3. 8 of 9 carry at least one Iron-Law fenced block.
+#      agent-authority is the dispatch index, exempt from the fence
+#      requirement (docs/contracts/smoke-contracts.md § 3.1).
+#   4. Kernel-bucket char budget breaches ≤ EXPECTED_BREACHES.
+#
+# Runtime ceiling: 30 s.
+# Output: table by default, baseline line on stdout last; SMOKE_QUIET=1
+# suppresses the table.
+# Contract: docs/contracts/smoke-contracts.md
+
+set -euo pipefail
+
+EXPECTED_KERNEL_COUNT=9
+EXPECTED_FENCE_CARRIERS=8
+EXPECTED_BREACHES=2
+EXEMPT_FROM_FENCE="agent-authority"
+
+quiet="${SMOKE_QUIET:-0}"
+fail=0
+
+log() { [ "$quiet" = "1" ] || printf '%s\n' "$*"; }
+
+# 1. kernel ids from router.json
+kernel_ids=$(python3 -c '
+import json
+d = json.load(open("router.json"))
+print("\n".join(d.get("kernel", [])))
+')
+kernel_count=$(printf '%s\n' "$kernel_ids" | grep -c .)
+
+log "## Kernel smoke"
+log ""
+log "| Check | Value |"
+log "|---|---:|"
+log "| router.json kernel count | $kernel_count |"
+
+if [ "$kernel_count" -ne "$EXPECTED_KERNEL_COUNT" ]; then
+  echo "❌ kernel count: $kernel_count (expected $EXPECTED_KERNEL_COUNT)"
+  fail=1
+fi
+
+# 2. every kernel rule has a file
+missing=0
+for id in $kernel_ids; do
+  if [ ! -f ".agent-src/rules/$id.md" ]; then
+    echo "❌ missing rule file: .agent-src/rules/$id.md"
+    missing=$((missing + 1))
+  fi
+done
+log "| Rule files present | $((kernel_count - missing))/$kernel_count |"
+if [ "$missing" -gt 0 ]; then fail=1; fi
+
+# 3. count Iron-Law fences per rule
+fence_carriers=0
+for id in $kernel_ids; do
+  if printf ' %s ' "$EXEMPT_FROM_FENCE" | grep -q " $id "; then
+    continue
+  fi
+  if [ -f ".agent-src/rules/$id.md" ]; then
+    fences=$(awk 'BEGIN{c=0;open=0} /^```/{ if(open==0){c++;open=1}else{open=0} } END{print c}' ".agent-src/rules/$id.md")
+    if [ "$fences" -ge 1 ]; then
+      fence_carriers=$((fence_carriers + 1))
+    else
+      echo "❌ no Iron-Law fence in .agent-src/rules/$id.md"
+      fail=1
+    fi
+  fi
+done
+log "| Iron-Law fence carriers | $fence_carriers/$((kernel_count - 1)) |"
+
+if [ "$fence_carriers" -lt "$EXPECTED_FENCE_CARRIERS" ]; then
+  echo "❌ fence carriers: $fence_carriers (expected $EXPECTED_FENCE_CARRIERS)"
+  fail=1
+fi
+
+# 4. kernel char-budget breach count (advisory: locked at current measured)
+breach_count=0
+if python3 scripts/measure_rule_budget.py --kernel-budget-check >/tmp/kernel-budget.$$ 2>&1; then
+  breach_count=0
+else
+  breach_count=$(grep -c "^  - " /tmp/kernel-budget.$$ || true)
+fi
+rm -f /tmp/kernel-budget.$$
+log "| Kernel-budget breaches | $breach_count (locked ≤ $EXPECTED_BREACHES) |"
+
+if [ "$breach_count" -gt "$EXPECTED_BREACHES" ]; then
+  echo "❌ kernel budget breaches: $breach_count > $EXPECTED_BREACHES (regression)"
+  fail=1
+fi
+
+# Baseline line — last line of stdout for CI summary parsing.
+log ""
+echo "BASELINE: $kernel_count kernel rules · $fence_carriers carry Iron-Law fences · 1 dispatch index · $breach_count budget breach(es)"
+
+exit $fail