@event4u/agent-config 2.12.0 → 2.14.0

package/scripts/score_skill_selection.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Selection-accuracy scorer (council file 05, Phase 2.2).
+
+Reads `tests/fixtures/skill_selection/fixtures.yml` and a predictions
+JSON (`{fixture_id: selected_skill_name}`), then computes:
+
+- (a) intended-skill hit rate — exact `intended` match
+- (b) correct-cluster hit rate — any member of the same cluster
+
+Per-cluster pass/fail uses the Round-3 protocol:
+    pass = (a) >= 0.90  OR  (b) >= 0.95
+    fail = (a) <  0.80  AND  (b) <  0.80   →  cluster needs `routes_to`
+
+Predictions source:
+- `--predictions <path>`: external JSON file (LLM run, eval harness, manual).
+- `--baseline`: built-in TF-IDF-style description-similarity baseline. The
+  baseline does NOT speak for any specific host tool; it estimates what
+  pure description-matching would do and provides a numeric floor.
+
+Output: human-readable summary on stdout + machine JSON to
+`agents/reports/skill-selection-accuracy.json` (or `--out`).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+FIXTURES = REPO_ROOT / "tests" / "fixtures" / "skill_selection" / "fixtures.yml"
+CLUSTERS = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
+SKILLS_DIR = REPO_ROOT / ".agent-src.uncompressed" / "skills"
+DEFAULT_OUT = REPO_ROOT / "agents" / "reports" / "skill-selection-accuracy.json"
+
+PASS_A = 0.90
+PASS_B = 0.95
+FAIL_THRESHOLD = 0.80
+
+STOPWORDS = {
+    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a", "an",
+    "is", "in", "on", "by", "be", "at", "as", "it", "if", "are", "this",
+    "that", "from", "but", "not", "can", "any", "all", "no", "after",
+    "before", "during", "user", "agent", "code", "project", "via", "into",
+    "onto", "even", "without", "naming", "uses", "used", "using", "also",
+    "etc", "across", "between",
+}
+
+
+def tokenize(text: str) -> list[str]:
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
+    return [t for t in tokens if t not in STOPWORDS and not t.isdigit()]
+
+
+def load_skills() -> dict[str, str]:
+    out = {}
+    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
+        text = skill_md.read_text()
+        if not text.startswith("---"):
+            continue
+        parts = text.split("---", 2)
+        if len(parts) < 3:
+            continue
+        try:
+            fm = yaml.safe_load(parts[1]) or {}
+        except yaml.YAMLError:
+            continue
+        name = fm.get("name") or skill_md.parent.name
+        desc = (fm.get("description") or "").strip()
+        if desc:
+            out[name] = desc
+    return out
+
+
+def tfidf_vectors(docs: dict[str, str]) -> tuple[dict[str, dict[str, float]], dict[str, float]]:
+    n_docs = len(docs)
+    df: Counter[str] = Counter()
+    tokenized = {k: tokenize(v) for k, v in docs.items()}
+    for toks in tokenized.values():
+        for term in set(toks):
+            df[term] += 1
+    idf = {term: math.log((n_docs + 1) / (count + 1)) + 1 for term, count in df.items()}
+    vectors: dict[str, dict[str, float]] = {}
+    for name, toks in tokenized.items():
+        tf = Counter(toks)
+        vectors[name] = {term: tf[term] * idf.get(term, 0.0) for term in tf}
+    return vectors, idf
+
+
+def cosine(a: dict[str, float], b: dict[str, float]) -> float:
+    if not a or not b:
+        return 0.0
+    common = set(a) & set(b)
+    dot = sum(a[t] * b[t] for t in common)
+    na = math.sqrt(sum(v * v for v in a.values()))
+    nb = math.sqrt(sum(v * v for v in b.values()))
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+def baseline_predict(fixtures: list[dict], skills: dict[str, str]) -> dict[str, str]:
+    vectors, idf = tfidf_vectors(skills)
+    preds: dict[str, str] = {}
+    for fx in fixtures:
+        prompt_tokens = tokenize(fx["prompt"])
+        tf = Counter(prompt_tokens)
+        pv = {term: tf[term] * idf.get(term, 0.0) for term in tf}
+        best_name, best_score = "", -1.0
+        for name, vec in vectors.items():
+            score = cosine(pv, vec)
+            if score > best_score:
+                best_name, best_score = name, score
+        preds[fx["id"]] = best_name
+    return preds
+
+
+def score(fixtures: list[dict], clusters: list[dict], preds: dict[str, str]) -> dict:
+    # Look up cluster membership by intended-skill (robust to cluster_id renumbering).
+    by_member: dict[str, set[str]] = {}
+    for c in clusters:
+        members = set(c["members"])
+        for m in members:
+            by_member[m] = members
+    per_cluster = defaultdict(lambda: {"total": 0, "hits_a": 0, "hits_b": 0, "misses": [], "label": ""})
+    for fx in fixtures:
+        intended = fx["intended"]
+        members = by_member.get(intended, {intended})
+        # Stable label: sorted members joined — survives cluster_id renumbering.
+        cid = fx.get("cluster") or "+".join(sorted(members)[:2])
+        pred = preds.get(fx["id"], "")
+        rec = per_cluster[cid]
+        rec["total"] += 1
+        rec["label"] = ",".join(sorted(members))
+        if pred == intended:
+            rec["hits_a"] += 1
+        if pred in members:
+            rec["hits_b"] += 1
+        else:
+            rec["misses"].append({"id": fx["id"], "intended": intended, "predicted": pred})
+    results = []
+    for cid, rec in sorted(per_cluster.items()):
+        a = rec["hits_a"] / rec["total"]
+        b = rec["hits_b"] / rec["total"]
+        if a >= PASS_A or b >= PASS_B:
+            verdict = "pass"
+        elif a < FAIL_THRESHOLD and b < FAIL_THRESHOLD:
+            verdict = "fail-needs-routes_to"
+        else:
+            verdict = "mixed"
+        results.append({"cluster": cid, "n": rec["total"], "hit_a": round(a, 3),
+                        "hit_b": round(b, 3), "verdict": verdict, "misses": rec["misses"]})
+    total = sum(r["n"] for r in results)
+    overall_a = sum(r["hit_a"] * r["n"] for r in results) / total if total else 0.0
+    overall_b = sum(r["hit_b"] * r["n"] for r in results) / total if total else 0.0
+    return {"clusters": results,
+            "overall": {"n": total, "hit_a": round(overall_a, 3), "hit_b": round(overall_b, 3)}}
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--predictions", type=Path, help="JSON file: {fixture_id: skill_name}")
+    p.add_argument("--baseline", action="store_true", help="Use built-in TF-IDF baseline")
+    p.add_argument("--source", default="external", help="Label recorded in output")
+    p.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = p.parse_args()
+
+    if not args.predictions and not args.baseline:
+        print("❌  Specify --predictions <file> or --baseline", file=sys.stderr)
+        return 2
+    fixtures = yaml.safe_load(FIXTURES.read_text())["fixtures"]
+    clusters = json.loads(CLUSTERS.read_text())["clusters"]
+    skills = load_skills()
+    if args.baseline:
+        preds = baseline_predict(fixtures, skills)
+        source = "tfidf-baseline"
+    else:
+        preds = json.loads(args.predictions.read_text())
+        source = args.source
+    report = score(fixtures, clusters, preds)
+    report["source"] = source
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(report, indent=2) + "\n")
+    print(f"✅  Wrote {args.out.relative_to(REPO_ROOT)}  (source={source})")
+    print(f"   overall: hit_a={report['overall']['hit_a']:.3f}  hit_b={report['overall']['hit_b']:.3f}  n={report['overall']['n']}")
+    for c in report["clusters"]:
+        print(f"   {c['cluster']:6}  n={c['n']:2}  hit_a={c['hit_a']:.2f}  hit_b={c['hit_b']:.2f}  {c['verdict']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/skill_collision_clusters.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Skill-collision cluster analysis (Phase 2.2 of step-1-v2-feedback-followup).
+
+Walks `.agent-src.uncompressed/skills/<id>/SKILL.md`, extracts the
+`description` frontmatter, computes pairwise keyword overlap, and groups
+high-overlap skill pairs into clusters. The output drives the
+selection-accuracy fixture set defined by council file 05 (Round-3
+protocol — ≥ 3 shared significant terms → collision cluster).
+
+Output: `agents/reports/skill-collision-clusters.json`
+
+Schema:
+    {
+      "skill_count": int,
+      "cluster_count": int,
+      "clusters": [
+        {
+          "cluster_id": "C01",
+          "members": ["skill-a", "skill-b", ...],
+          "shared_keywords": [...],
+          "max_overlap": float,
+          "descriptions": {"skill-a": "...", ...}
+        },
+        ...
+      ]
+    }
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from itertools import combinations
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+SKILLS_DIR = REPO_ROOT / ".agent-src.uncompressed" / "skills"
+OUT_JSON = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
+
+KEYWORD_OVERLAP_THRESHOLD = 0.40
+MIN_SHARED_KEYWORDS = 3
+TOP_N_CLUSTERS = 10
+
+STOPWORDS = {
+    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a",
+    "an", "is", "in", "on", "by", "be", "at", "as", "it", "if", "are",
+    "this", "that", "from", "but", "not", "can", "any", "all", "no",
+    "after", "before", "during", "user", "agent", "code", "project",
+    "via", "into", "onto", "even", "without", "naming", "uses", "used",
+    "using", "also", "etc", "across", "between", "review", "design",
+    "writing", "create", "creating", "edit", "editing", "make", "making",
+    "set", "setting", "based", "well", "right", "left", "new",
+}
+
+
+def keyword_set(text: str) -> set[str]:
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
+    return {t for t in tokens if t not in STOPWORDS and not t.isdigit()}
+
+
+def overlap_fraction(a: set[str], b: set[str]) -> float:
+    if not a or not b:
+        return 0.0
+    return len(a & b) / min(len(a), len(b))
+
+
+def load_skills() -> list[dict]:
+    skills = []
+    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
+        text = skill_md.read_text()
+        if not text.startswith("---"):
+            continue
+        parts = text.split("---", 2)
+        if len(parts) < 3:
+            continue
+        try:
+            fm = yaml.safe_load(parts[1]) or {}
+        except yaml.YAMLError:
+            continue
+        name = fm.get("name") or skill_md.parent.name
+        description = (fm.get("description") or "").strip()
+        if not description:
+            continue
+        skills.append(
+            {
+                "name": name,
+                "description": description,
+                "_keywords": keyword_set(description),
+            }
+        )
+    return skills
+
+
+def build_clusters(skills: list[dict]) -> list[dict]:
+    # Pairwise edges where overlap & shared-keyword threshold is met.
+    edges: list[tuple[str, str, set[str], float]] = []
+    by_name = {s["name"]: s for s in skills}
+    for a, b in combinations(skills, 2):
+        shared = a["_keywords"] & b["_keywords"]
+        ov = overlap_fraction(a["_keywords"], b["_keywords"])
+        if len(shared) >= MIN_SHARED_KEYWORDS and ov >= KEYWORD_OVERLAP_THRESHOLD:
+            edges.append((a["name"], b["name"], shared, ov))
+
+    # Union-find over edge set → connected-component clusters.
+    parent: dict[str, str] = {}
+
+    def find(x: str) -> str:
+        parent.setdefault(x, x)
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(x: str, y: str) -> None:
+        rx, ry = find(x), find(y)
+        if rx != ry:
+            parent[rx] = ry
+
+    for a, b, _, _ in edges:
+        union(a, b)
+
+    components: dict[str, list[str]] = {}
+    for name in {n for edge in edges for n in edge[:2]}:
+        components.setdefault(find(name), []).append(name)
+
+    clusters: list[dict] = []
+    for idx, (_, members) in enumerate(sorted(components.items(), key=lambda kv: -len(kv[1])), start=1):
+        member_kws = [by_name[m]["_keywords"] for m in members]
+        shared_all = set.intersection(*member_kws) if member_kws else set()
+        member_edges = [(a, b, sk, ov) for a, b, sk, ov in edges if a in members and b in members]
+        max_ov = max((ov for *_, ov in member_edges), default=0.0)
+        clusters.append({
+            "cluster_id": f"C{idx:02d}",
+            "members": sorted(members),
+            "shared_keywords": sorted(shared_all),
+            "max_overlap": round(max_ov, 3),
+            "descriptions": {m: by_name[m]["description"] for m in sorted(members)},
+        })
+    return clusters[:TOP_N_CLUSTERS]
+
+
+def main() -> int:
+    if not SKILLS_DIR.exists():
+        print(f"❌  Skills dir not found: {SKILLS_DIR}", file=sys.stderr)
+        return 2
+    skills = load_skills()
+    clusters = build_clusters(skills)
+    OUT_JSON.parent.mkdir(parents=True, exist_ok=True)
+    OUT_JSON.write_text(json.dumps({
+        "skill_count": len(skills),
+        "cluster_count": len(clusters),
+        "clusters": clusters,
+    }, indent=2) + "\n")
+    print(f"✅  Wrote {OUT_JSON.relative_to(REPO_ROOT)} — {len(clusters)} clusters from {len(skills)} skills")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())