@event4u/agent-config 6.1.0 → 7.0.0

package/src/scripts/score_skill_selection.py

@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-"""Selection-accuracy scorer (council file 05, Phase 2.2).
-
-Reads `tests/fixtures/skill_selection/fixtures.yml` and a predictions
-JSON (`{fixture_id: selected_skill_name}`), then computes:
-
-- (a) intended-skill hit rate — exact `intended` match
-- (b) correct-cluster hit rate — any member of the same cluster
-
-Per-cluster pass/fail uses the Round-3 protocol:
-    pass = (a) >= 0.90  OR  (b) >= 0.95
-    fail = (a) <  0.80  AND  (b) <  0.80   →  cluster needs `routes_to`
-
-Predictions source:
-- `--predictions <path>`: external JSON file (LLM run, eval harness, manual).
-- `--baseline`: built-in TF-IDF-style description-similarity baseline. The
-  baseline does NOT speak for any specific host tool; it estimates what
-  pure description-matching would do and provides a numeric floor.
-
-Output: human-readable summary on stdout + machine JSON to
-`agents/runtime/reports/skill-selection-accuracy.json` (or `--out`).
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import re
-import sys
-from collections import Counter, defaultdict
-from pathlib import Path
-
-import yaml
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-FIXTURES = REPO_ROOT / "tests" / "fixtures" / "skill_selection" / "fixtures.yml"
-CLUSTERS = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
-SKILLS_DIR = REPO_ROOT / ".agent-src.uncondensed" / "skills"
-DEFAULT_OUT = REPO_ROOT / "agents" / "reports" / "skill-selection-accuracy.json"
-
-PASS_A = 0.90
-PASS_B = 0.95
-FAIL_THRESHOLD = 0.80
-
-STOPWORDS = {
-    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a", "an",
-    "is", "in", "on", "by", "be", "at", "as", "it", "if", "are", "this",
-    "that", "from", "but", "not", "can", "any", "all", "no", "after",
-    "before", "during", "user", "agent", "code", "project", "via", "into",
-    "onto", "even", "without", "naming", "uses", "used", "using", "also",
-    "etc", "across", "between",
-}
-
-
-def tokenize(text: str) -> list[str]:
-    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
-    return [t for t in tokens if t not in STOPWORDS and not t.isdigit()]
-
-
-def load_skills() -> dict[str, str]:
-    out = {}
-    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
-        text = skill_md.read_text()
-        if not text.startswith("---"):
-            continue
-        parts = text.split("---", 2)
-        if len(parts) < 3:
-            continue
-        try:
-            fm = yaml.safe_load(parts[1]) or {}
-        except yaml.YAMLError:
-            continue
-        name = fm.get("name") or skill_md.parent.name
-        desc = (fm.get("description") or "").strip()
-        if desc:
-            out[name] = desc
-    return out
-
-
-def tfidf_vectors(docs: dict[str, str]) -> tuple[dict[str, dict[str, float]], dict[str, float]]:
-    n_docs = len(docs)
-    df: Counter[str] = Counter()
-    tokenized = {k: tokenize(v) for k, v in docs.items()}
-    for toks in tokenized.values():
-        for term in set(toks):
-            df[term] += 1
-    idf = {term: math.log((n_docs + 1) / (count + 1)) + 1 for term, count in df.items()}
-    vectors: dict[str, dict[str, float]] = {}
-    for name, toks in tokenized.items():
-        tf = Counter(toks)
-        vectors[name] = {term: tf[term] * idf.get(term, 0.0) for term in tf}
-    return vectors, idf
-
-
-def cosine(a: dict[str, float], b: dict[str, float]) -> float:
-    if not a or not b:
-        return 0.0
-    common = set(a) & set(b)
-    dot = sum(a[t] * b[t] for t in common)
-    na = math.sqrt(sum(v * v for v in a.values()))
-    nb = math.sqrt(sum(v * v for v in b.values()))
-    if na == 0 or nb == 0:
-        return 0.0
-    return dot / (na * nb)
-
-
-def baseline_predict(fixtures: list[dict], skills: dict[str, str]) -> dict[str, str]:
-    vectors, idf = tfidf_vectors(skills)
-    preds: dict[str, str] = {}
-    for fx in fixtures:
-        prompt_tokens = tokenize(fx["prompt"])
-        tf = Counter(prompt_tokens)
-        pv = {term: tf[term] * idf.get(term, 0.0) for term in tf}
-        best_name, best_score = "", -1.0
-        for name, vec in vectors.items():
-            score = cosine(pv, vec)
-            if score > best_score:
-                best_name, best_score = name, score
-        preds[fx["id"]] = best_name
-    return preds
-
-
-def score(fixtures: list[dict], clusters: list[dict], preds: dict[str, str]) -> dict:
-    # Look up cluster membership by intended-skill (robust to cluster_id renumbering).
-    by_member: dict[str, set[str]] = {}
-    for c in clusters:
-        members = set(c["members"])
-        for m in members:
-            by_member[m] = members
-    per_cluster = defaultdict(lambda: {"total": 0, "hits_a": 0, "hits_b": 0, "misses": [], "label": ""})
-    for fx in fixtures:
-        intended = fx["intended"]
-        members = by_member.get(intended, {intended})
-        # Stable label: sorted members joined — survives cluster_id renumbering.
-        cid = fx.get("cluster") or "+".join(sorted(members)[:2])
-        pred = preds.get(fx["id"], "")
-        rec = per_cluster[cid]
-        rec["total"] += 1
-        rec["label"] = ",".join(sorted(members))
-        if pred == intended:
-            rec["hits_a"] += 1
-        if pred in members:
-            rec["hits_b"] += 1
-        else:
-            rec["misses"].append({"id": fx["id"], "intended": intended, "predicted": pred})
-    results = []
-    for cid, rec in sorted(per_cluster.items()):
-        a = rec["hits_a"] / rec["total"]
-        b = rec["hits_b"] / rec["total"]
-        if a >= PASS_A or b >= PASS_B:
-            verdict = "pass"
-        elif a < FAIL_THRESHOLD and b < FAIL_THRESHOLD:
-            verdict = "fail-needs-routes_to"
-        else:
-            verdict = "mixed"
-        results.append({"cluster": cid, "n": rec["total"], "hit_a": round(a, 3),
-                        "hit_b": round(b, 3), "verdict": verdict, "misses": rec["misses"]})
-    total = sum(r["n"] for r in results)
-    overall_a = sum(r["hit_a"] * r["n"] for r in results) / total if total else 0.0
-    overall_b = sum(r["hit_b"] * r["n"] for r in results) / total if total else 0.0
-    return {"clusters": results,
-            "overall": {"n": total, "hit_a": round(overall_a, 3), "hit_b": round(overall_b, 3)}}
-
-
-def main() -> int:
-    p = argparse.ArgumentParser()
-    p.add_argument("--predictions", type=Path, help="JSON file: {fixture_id: skill_name}")
-    p.add_argument("--baseline", action="store_true", help="Use built-in TF-IDF baseline")
-    p.add_argument("--source", default="external", help="Label recorded in output")
-    p.add_argument("--out", type=Path, default=DEFAULT_OUT)
-    args = p.parse_args()
-
-    if not args.predictions and not args.baseline:
-        print("❌  Specify --predictions <file> or --baseline", file=sys.stderr)
-        return 2
-    fixtures = yaml.safe_load(FIXTURES.read_text())["fixtures"]
-    clusters = json.loads(CLUSTERS.read_text())["clusters"]
-    skills = load_skills()
-    if args.baseline:
-        preds = baseline_predict(fixtures, skills)
-        source = "tfidf-baseline"
-    else:
-        preds = json.loads(args.predictions.read_text())
-        source = args.source
-    report = score(fixtures, clusters, preds)
-    report["source"] = source
-    args.out.parent.mkdir(parents=True, exist_ok=True)
-    args.out.write_text(json.dumps(report, indent=2) + "\n")
-    print(f"✅  Wrote {args.out.relative_to(REPO_ROOT)}  (source={source})")
-    print(f"   overall: hit_a={report['overall']['hit_a']:.3f}  hit_b={report['overall']['hit_b']:.3f}  n={report['overall']['n']}")
-    for c in report["clusters"]:
-        print(f"   {c['cluster']:6}  n={c['n']:2}  hit_a={c['hit_a']:.2f}  hit_b={c['hit_b']:.2f}  {c['verdict']}")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/src/scripts/security_audit_config.py

@@ -1,153 +0,0 @@
-#!/usr/bin/env python3
-"""P3.1 — consumer-facing agent-config security audit (road-to-security-pillar.md).
-
-Points the Phase-1 detection logic at a *consumer's assembled* agent config —
-instruction files (CLAUDE.md, AGENTS.md, .cursor/rules, copilot-instructions),
-MCP configs (.mcp.json, .cursor/mcp.json, claude_desktop_config.json), settings
-+ hooks (.claude/settings.json), and installed skills — and emits an A–F score
-with a per-category breakdown mapped to the OWASP Top 10 for Agentic
-Applications (ASI).
-
-Detection is the same library as the self-audit gate (so there is one source of
-truth for the patterns) under the same false-positive containment convention.
-This is decision-support, not a guarantee: detection is probabilistic.
-
-Usage:
-  python3 src/scripts/security_audit_config.py [--root DIR] [--json]
-"""
-from __future__ import annotations
-
-import argparse
-import sys
-from pathlib import Path
-
-HERE = Path(__file__).resolve().parent
-sys.path.insert(0, str(HERE))
-from _lib import security_lint as sl  # noqa: E402
-import lint_hidden_unicode as p11  # noqa: E402
-import lint_instruction_smuggling as p12  # noqa: E402
-import lint_mcp_config_security as p13  # noqa: E402
-import lint_skill_frontmatter_safety as p14  # noqa: E402
-
-# Consumer config surfaces (globs relative to --root).
-SURFACES = [
-    "CLAUDE.md", "AGENTS.md", "GEMINI.md", ".clinerules", ".windsurfrules",
-    ".github/copilot-instructions.md",
-    ".cursor/rules/**/*", ".cursorrules",
-    ".claude/skills/**/SKILL.md", ".claude/commands/**/*.md",
-    ".claude/settings.json", ".claude/settings.local.json",
-    ".mcp.json", ".cursor/mcp.json", "claude_desktop_config.json",
-]
-
-# check id → (category, OWASP-ASI tag)
-CATEGORY = {
-    "hidden-unicode": ("Agents/Rules", "ASI01 Goal Hijack"),
-    "instruction-smuggling": ("Agents/Rules", "ASI01 Goal Hijack"),
-    "mcp-config-security": ("MCP", "ASI04 Supply Chain"),
-    "dangerous-frontmatter": ("Permissions", "ASI03 Privilege Abuse"),
-}
-SECRET_HINT = "secret"  # mcp finding mentioning a secret → Secrets category
-CATEGORIES = ["Secrets", "Permissions", "Hooks", "MCP", "Agents/Rules"]
-
-# Deduction per finding (full weight); weighted findings scale by their weight.
-_DEDUCT = {"HIGH": 25.0, "MED": 5.0, "LOW": 2.0}
-
-
-def _grade(score: float) -> str:
-    return ("A" if score >= 90 else "B" if score >= 80 else "C" if score >= 70
-            else "D" if score >= 60 else "F")
-
-
-def _category(f) -> str:
-    if f.check == "mcp-config-security" and SECRET_HINT in f.message.lower():
-        return "Secrets"
-    return CATEGORY.get(f.check, ("Agents/Rules", ""))[0]
-
-
-def _iter_targets(root: Path):
-    seen = set()
-    for pattern in SURFACES:
-        for p in root.glob(pattern):
-            if p.is_file() and p not in seen:
-                seen.add(p)
-                yield p
-
-
-def audit(root: Path) -> dict:
-    findings = []
-    for p in _iter_targets(root):
-        try:
-            sf = sl.scan_path(p, root)
-        except (UnicodeDecodeError, OSError):
-            continue
-        for mod in (p11, p12, p13, p14):
-            try:
-                findings.extend(mod._scan(sf))
-            except Exception:
-                pass
-
-    per_cat = {c: 100.0 for c in CATEGORIES}
-    cat_findings = {c: [] for c in CATEGORIES}
-    for f in findings:
-        cat = _category(f)
-        per_cat[cat] -= _DEDUCT.get(f.severity, 2.0) * float(f.weight)
-        cat_findings[cat].append(f)
-    for c in per_cat:
-        per_cat[c] = max(0.0, per_cat[c])
-
-    overall = round(sum(per_cat.values()) / len(CATEGORIES), 1)
-    return {
-        "root": str(root),
-        "overall_score": overall,
-        "overall_grade": _grade(overall),
-        "categories": {
-            c: {
-                "score": round(per_cat[c], 1),
-                "grade": _grade(per_cat[c]),
-                "owasp": next((CATEGORY[fl.check][1] for fl in cat_findings[c]
-                               if fl.check in CATEGORY), ""),
-                "findings": [
-                    {"path": fl.path, "line": fl.line, "check": fl.check,
-                     "severity": fl.severity, "message": fl.message,
-                     "weight": fl.weight}
-                    for fl in cat_findings[c]
-                ],
-            }
-            for c in CATEGORIES
-        },
-    }
-
-
-def _print(report: dict) -> None:
-    print(f"Agent-config security audit — {report['root']}")
-    print(f"Overall: {report['overall_grade']} ({report['overall_score']}/100)\n")
-    for c in CATEGORIES:
-        cat = report["categories"][c]
-        tag = f" · {cat['owasp']}" if cat["owasp"] else ""
-        print(f"  {cat['grade']}  {c:<12} {cat['score']:>5}/100{tag}")
-        for f in cat["findings"]:
-            loc = f"{f['path']}:{f['line']}" if f["line"] else f["path"]
-            w = "" if f["weight"] >= 1.0 else f" (weight {f['weight']:g})"
-            print(f"        [{f['severity']}] {loc}{w}: {f['message']}")
-    print("\n> Decision support, not a guarantee — detection is probabilistic. "
-          "Pair with /threat-model and judge-security-auditor for a deep pass.")
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description=__doc__, epilog=sl.GUIDELINE_EPILOG)
-    ap.add_argument("--root", default=".", help="consumer repo root to audit (default: cwd)")
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-
-    report = audit(Path(args.root))
-    if args.json:
-        import json
-        print(json.dumps(report, indent=2))
-    else:
-        _print(report)
-    # Audit is advisory: always exit 0 (it informs, it does not gate the consumer).
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/src/scripts/setup_eval_venv.sh

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-# Bootstrap .venv/ for the live trigger-eval runner.
-#
-# We install the anthropic SDK in an isolated virtualenv instead of the
-# system Python so:
-#   - the SDK never leaks into the user's global interpreter,
-#   - no PYTHONPATH / sys.path surprises when other scripts import
-#     parts of the repo,
-#   - removal is `rm -rf .venv/` with no state elsewhere.
-#
-# The Taskfile target `task test-triggers-live` calls
-# `.venv/bin/python3 scripts/skill_trigger_eval.py` directly, so the
-# user never needs to `source .venv/bin/activate`.
-#
-# Idempotent: safe to rerun to refresh pins.
-
-set -euo pipefail
-
-REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-cd "${REPO_ROOT}"
-
-VENV_DIR=".venv"
-REQ_FILE="scripts/requirements-evals.txt"
-
-if [[ ! -f "${REQ_FILE}" ]]; then
-    echo "❌  ${REQ_FILE} missing — aborting."
-    exit 2
-fi
-
-# Prefer python3.10+ if available; fall back to the system python3.
-PY="python3"
-for candidate in python3.12 python3.11 python3.10; do
-    if command -v "${candidate}" >/dev/null 2>&1; then
-        PY="${candidate}"
-        break
-    fi
-done
-
-if [[ ! -d "${VENV_DIR}" ]]; then
-    echo "→ Creating ${VENV_DIR} with ${PY} ..."
-    "${PY}" -m venv "${VENV_DIR}"
-else
-    echo "→ Reusing existing ${VENV_DIR}"
-fi
-
-echo "→ Upgrading pip ..."
-"${VENV_DIR}/bin/pip" install --upgrade pip -q
-
-echo "→ Installing pinned requirements from ${REQ_FILE} ..."
-"${VENV_DIR}/bin/pip" install -q -r "${REQ_FILE}"
-
-echo
-"${VENV_DIR}/bin/python3" -c "import anthropic; print(f'✅  anthropic {anthropic.__version__}')"
-"${VENV_DIR}/bin/python3" --version
-echo
-echo "Venv ready at ${REPO_ROOT}/${VENV_DIR}."
-echo "Next:  task install-anthropic-key    # will prompt for your key"
-echo "Then:  task test-triggers-live -- eloquent"

package/src/scripts/skill_collision_clusters.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-"""Skill-collision cluster analysis (Phase 2.2 of step-1-v2-feedback-followup).
-
-Walks `.agent-src.uncondensed/skills/<id>/SKILL.md`, extracts the
-`description` frontmatter, computes pairwise keyword overlap, and groups
-high-overlap skill pairs into clusters. The output drives the
-selection-accuracy fixture set defined by council file 05 (Round-3
-protocol — ≥ 3 shared significant terms → collision cluster).
-
-Output: `agents/runtime/reports/skill-collision-clusters.json`
-
-Schema:
-    {
-      "skill_count": int,
-      "cluster_count": int,
-      "clusters": [
-        {
-          "cluster_id": "C01",
-          "members": ["skill-a", "skill-b", ...],
-          "shared_keywords": [...],
-          "max_overlap": float,
-          "descriptions": {"skill-a": "...", ...}
-        },
-        ...
-      ]
-    }
-"""
-
-from __future__ import annotations
-
-import json
-import re
-import sys
-from itertools import combinations
-from pathlib import Path
-
-import yaml
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-SKILLS_DIR = REPO_ROOT / ".agent-src.uncondensed" / "skills"
-OUT_JSON = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
-
-KEYWORD_OVERLAP_THRESHOLD = 0.40
-MIN_SHARED_KEYWORDS = 3
-TOP_N_CLUSTERS = 10
-
-STOPWORDS = {
-    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a",
-    "an", "is", "in", "on", "by", "be", "at", "as", "it", "if", "are",
-    "this", "that", "from", "but", "not", "can", "any", "all", "no",
-    "after", "before", "during", "user", "agent", "code", "project",
-    "via", "into", "onto", "even", "without", "naming", "uses", "used",
-    "using", "also", "etc", "across", "between", "review", "design",
-    "writing", "create", "creating", "edit", "editing", "make", "making",
-    "set", "setting", "based", "well", "right", "left", "new",
-}
-
-
-def keyword_set(text: str) -> set[str]:
-    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
-    return {t for t in tokens if t not in STOPWORDS and not t.isdigit()}
-
-
-def overlap_fraction(a: set[str], b: set[str]) -> float:
-    if not a or not b:
-        return 0.0
-    return len(a & b) / min(len(a), len(b))
-
-
-def load_skills() -> list[dict]:
-    skills = []
-    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
-        text = skill_md.read_text()
-        if not text.startswith("---"):
-            continue
-        parts = text.split("---", 2)
-        if len(parts) < 3:
-            continue
-        try:
-            fm = yaml.safe_load(parts[1]) or {}
-        except yaml.YAMLError:
-            continue
-        name = fm.get("name") or skill_md.parent.name
-        description = (fm.get("description") or "").strip()
-        if not description:
-            continue
-        skills.append(
-            {
-                "name": name,
-                "description": description,
-                "_keywords": keyword_set(description),
-            }
-        )
-    return skills
-
-
-def build_clusters(skills: list[dict]) -> list[dict]:
-    # Pairwise edges where overlap & shared-keyword threshold is met.
-    edges: list[tuple[str, str, set[str], float]] = []
-    by_name = {s["name"]: s for s in skills}
-    for a, b in combinations(skills, 2):
-        shared = a["_keywords"] & b["_keywords"]
-        ov = overlap_fraction(a["_keywords"], b["_keywords"])
-        if len(shared) >= MIN_SHARED_KEYWORDS and ov >= KEYWORD_OVERLAP_THRESHOLD:
-            edges.append((a["name"], b["name"], shared, ov))
-
-    # Union-find over edge set → connected-component clusters.
-    parent: dict[str, str] = {}
-
-    def find(x: str) -> str:
-        parent.setdefault(x, x)
-        while parent[x] != x:
-            parent[x] = parent[parent[x]]
-            x = parent[x]
-        return x
-
-    def union(x: str, y: str) -> None:
-        rx, ry = find(x), find(y)
-        if rx != ry:
-            parent[rx] = ry
-
-    for a, b, _, _ in edges:
-        union(a, b)
-
-    components: dict[str, list[str]] = {}
-    for name in {n for edge in edges for n in edge[:2]}:
-        components.setdefault(find(name), []).append(name)
-
-    clusters: list[dict] = []
-    for idx, (_, members) in enumerate(sorted(components.items(), key=lambda kv: -len(kv[1])), start=1):
-        member_kws = [by_name[m]["_keywords"] for m in members]
-        shared_all = set.intersection(*member_kws) if member_kws else set()
-        member_edges = [(a, b, sk, ov) for a, b, sk, ov in edges if a in members and b in members]
-        max_ov = max((ov for *_, ov in member_edges), default=0.0)
-        clusters.append({
-            "cluster_id": f"C{idx:02d}",
-            "members": sorted(members),
-            "shared_keywords": sorted(shared_all),
-            "max_overlap": round(max_ov, 3),
-            "descriptions": {m: by_name[m]["description"] for m in sorted(members)},
-        })
-    return clusters[:TOP_N_CLUSTERS]
-
-
-def main() -> int:
-    if not SKILLS_DIR.exists():
-        print(f"❌  Skills dir not found: {SKILLS_DIR}", file=sys.stderr)
-        return 2
-    skills = load_skills()
-    clusters = build_clusters(skills)
-    OUT_JSON.parent.mkdir(parents=True, exist_ok=True)
-    OUT_JSON.write_text(json.dumps({
-        "skill_count": len(skills),
-        "cluster_count": len(clusters),
-        "clusters": clusters,
-    }, indent=2) + "\n")
-    print(f"✅  Wrote {OUT_JSON.relative_to(REPO_ROOT)} — {len(clusters)} clusters from {len(skills)} skills")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())