@event4u/agent-config 2.11.0 → 2.13.0

package/scripts/score_skill_selection.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Selection-accuracy scorer (council file 05, Phase 2.2).
+
+Reads `tests/fixtures/skill_selection/fixtures.yml` and a predictions
+JSON (`{fixture_id: selected_skill_name}`), then computes:
+
+- (a) intended-skill hit rate — exact `intended` match
+- (b) correct-cluster hit rate — any member of the same cluster
+
+Per-cluster pass/fail uses the Round-3 protocol:
+    pass = (a) >= 0.90  OR  (b) >= 0.95
+    fail = (a) <  0.80  AND  (b) <  0.80   →  cluster needs `routes_to`
+
+Predictions source:
+- `--predictions <path>`: external JSON file (LLM run, eval harness, manual).
+- `--baseline`: built-in TF-IDF-style description-similarity baseline. The
+  baseline does NOT speak for any specific host tool; it estimates what
+  pure description-matching would do and provides a numeric floor.
+
+Output: human-readable summary on stdout + machine JSON to
+`agents/reports/skill-selection-accuracy.json` (or `--out`).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+FIXTURES = REPO_ROOT / "tests" / "fixtures" / "skill_selection" / "fixtures.yml"
+CLUSTERS = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
+SKILLS_DIR = REPO_ROOT / ".agent-src.uncompressed" / "skills"
+DEFAULT_OUT = REPO_ROOT / "agents" / "reports" / "skill-selection-accuracy.json"
+
+PASS_A = 0.90
+PASS_B = 0.95
+FAIL_THRESHOLD = 0.80
+
+STOPWORDS = {
+    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a", "an",
+    "is", "in", "on", "by", "be", "at", "as", "it", "if", "are", "this",
+    "that", "from", "but", "not", "can", "any", "all", "no", "after",
+    "before", "during", "user", "agent", "code", "project", "via", "into",
+    "onto", "even", "without", "naming", "uses", "used", "using", "also",
+    "etc", "across", "between",
+}
+
+
+def tokenize(text: str) -> list[str]:
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
+    return [t for t in tokens if t not in STOPWORDS and not t.isdigit()]
+
+
+def load_skills() -> dict[str, str]:
+    out = {}
+    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
+        text = skill_md.read_text()
+        if not text.startswith("---"):
+            continue
+        parts = text.split("---", 2)
+        if len(parts) < 3:
+            continue
+        try:
+            fm = yaml.safe_load(parts[1]) or {}
+        except yaml.YAMLError:
+            continue
+        name = fm.get("name") or skill_md.parent.name
+        desc = (fm.get("description") or "").strip()
+        if desc:
+            out[name] = desc
+    return out
+
+
+def tfidf_vectors(docs: dict[str, str]) -> tuple[dict[str, dict[str, float]], dict[str, float]]:
+    n_docs = len(docs)
+    df: Counter[str] = Counter()
+    tokenized = {k: tokenize(v) for k, v in docs.items()}
+    for toks in tokenized.values():
+        for term in set(toks):
+            df[term] += 1
+    idf = {term: math.log((n_docs + 1) / (count + 1)) + 1 for term, count in df.items()}
+    vectors: dict[str, dict[str, float]] = {}
+    for name, toks in tokenized.items():
+        tf = Counter(toks)
+        vectors[name] = {term: tf[term] * idf.get(term, 0.0) for term in tf}
+    return vectors, idf
+
+
+def cosine(a: dict[str, float], b: dict[str, float]) -> float:
+    if not a or not b:
+        return 0.0
+    common = set(a) & set(b)
+    dot = sum(a[t] * b[t] for t in common)
+    na = math.sqrt(sum(v * v for v in a.values()))
+    nb = math.sqrt(sum(v * v for v in b.values()))
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+def baseline_predict(fixtures: list[dict], skills: dict[str, str]) -> dict[str, str]:
+    vectors, idf = tfidf_vectors(skills)
+    preds: dict[str, str] = {}
+    for fx in fixtures:
+        prompt_tokens = tokenize(fx["prompt"])
+        tf = Counter(prompt_tokens)
+        pv = {term: tf[term] * idf.get(term, 0.0) for term in tf}
+        best_name, best_score = "", -1.0
+        for name, vec in vectors.items():
+            score = cosine(pv, vec)
+            if score > best_score:
+                best_name, best_score = name, score
+        preds[fx["id"]] = best_name
+    return preds
+
+
+def score(fixtures: list[dict], clusters: list[dict], preds: dict[str, str]) -> dict:
+    # Look up cluster membership by intended-skill (robust to cluster_id renumbering).
+    by_member: dict[str, set[str]] = {}
+    for c in clusters:
+        members = set(c["members"])
+        for m in members:
+            by_member[m] = members
+    per_cluster = defaultdict(lambda: {"total": 0, "hits_a": 0, "hits_b": 0, "misses": [], "label": ""})
+    for fx in fixtures:
+        intended = fx["intended"]
+        members = by_member.get(intended, {intended})
+        # Stable label: sorted members joined — survives cluster_id renumbering.
+        cid = fx.get("cluster") or "+".join(sorted(members)[:2])
+        pred = preds.get(fx["id"], "")
+        rec = per_cluster[cid]
+        rec["total"] += 1
+        rec["label"] = ",".join(sorted(members))
+        if pred == intended:
+            rec["hits_a"] += 1
+        if pred in members:
+            rec["hits_b"] += 1
+        else:
+            rec["misses"].append({"id": fx["id"], "intended": intended, "predicted": pred})
+    results = []
+    for cid, rec in sorted(per_cluster.items()):
+        a = rec["hits_a"] / rec["total"]
+        b = rec["hits_b"] / rec["total"]
+        if a >= PASS_A or b >= PASS_B:
+            verdict = "pass"
+        elif a < FAIL_THRESHOLD and b < FAIL_THRESHOLD:
+            verdict = "fail-needs-routes_to"
+        else:
+            verdict = "mixed"
+        results.append({"cluster": cid, "n": rec["total"], "hit_a": round(a, 3),
+                        "hit_b": round(b, 3), "verdict": verdict, "misses": rec["misses"]})
+    total = sum(r["n"] for r in results)
+    overall_a = sum(r["hit_a"] * r["n"] for r in results) / total if total else 0.0
+    overall_b = sum(r["hit_b"] * r["n"] for r in results) / total if total else 0.0
+    return {"clusters": results,
+            "overall": {"n": total, "hit_a": round(overall_a, 3), "hit_b": round(overall_b, 3)}}
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--predictions", type=Path, help="JSON file: {fixture_id: skill_name}")
+    p.add_argument("--baseline", action="store_true", help="Use built-in TF-IDF baseline")
+    p.add_argument("--source", default="external", help="Label recorded in output")
+    p.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = p.parse_args()
+
+    if not args.predictions and not args.baseline:
+        print("❌  Specify --predictions <file> or --baseline", file=sys.stderr)
+        return 2
+    fixtures = yaml.safe_load(FIXTURES.read_text())["fixtures"]
+    clusters = json.loads(CLUSTERS.read_text())["clusters"]
+    skills = load_skills()
+    if args.baseline:
+        preds = baseline_predict(fixtures, skills)
+        source = "tfidf-baseline"
+    else:
+        preds = json.loads(args.predictions.read_text())
+        source = args.source
+    report = score(fixtures, clusters, preds)
+    report["source"] = source
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(report, indent=2) + "\n")
+    print(f"✅  Wrote {args.out.relative_to(REPO_ROOT)}  (source={source})")
+    print(f"   overall: hit_a={report['overall']['hit_a']:.3f}  hit_b={report['overall']['hit_b']:.3f}  n={report['overall']['n']}")
+    for c in report["clusters"]:
+        print(f"   {c['cluster']:6}  n={c['n']:2}  hit_a={c['hit_a']:.2f}  hit_b={c['hit_b']:.2f}  {c['verdict']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/skill_collision_clusters.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Skill-collision cluster analysis (Phase 2.2 of step-1-v2-feedback-followup).
+
+Walks `.agent-src.uncompressed/skills/<id>/SKILL.md`, extracts the
+`description` frontmatter, computes pairwise keyword overlap, and groups
+high-overlap skill pairs into clusters. The output drives the
+selection-accuracy fixture set defined by council file 05 (Round-3
+protocol — ≥ 3 shared significant terms → collision cluster).
+
+Output: `agents/reports/skill-collision-clusters.json`
+
+Schema:
+    {
+      "skill_count": int,
+      "cluster_count": int,
+      "clusters": [
+        {
+          "cluster_id": "C01",
+          "members": ["skill-a", "skill-b", ...],
+          "shared_keywords": [...],
+          "max_overlap": float,
+          "descriptions": {"skill-a": "...", ...}
+        },
+        ...
+      ]
+    }
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from itertools import combinations
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+SKILLS_DIR = REPO_ROOT / ".agent-src.uncompressed" / "skills"
+OUT_JSON = REPO_ROOT / "agents" / "reports" / "skill-collision-clusters.json"
+
+KEYWORD_OVERLAP_THRESHOLD = 0.40
+MIN_SHARED_KEYWORDS = 3
+TOP_N_CLUSTERS = 10
+
+STOPWORDS = {
+    "the", "and", "for", "with", "when", "use", "or", "of", "to", "a",
+    "an", "is", "in", "on", "by", "be", "at", "as", "it", "if", "are",
+    "this", "that", "from", "but", "not", "can", "any", "all", "no",
+    "after", "before", "during", "user", "agent", "code", "project",
+    "via", "into", "onto", "even", "without", "naming", "uses", "used",
+    "using", "also", "etc", "across", "between", "review", "design",
+    "writing", "create", "creating", "edit", "editing", "make", "making",
+    "set", "setting", "based", "well", "right", "left", "new",
+}
+
+
+def keyword_set(text: str) -> set[str]:
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())
+    return {t for t in tokens if t not in STOPWORDS and not t.isdigit()}
+
+
+def overlap_fraction(a: set[str], b: set[str]) -> float:
+    if not a or not b:
+        return 0.0
+    return len(a & b) / min(len(a), len(b))
+
+
+def load_skills() -> list[dict]:
+    skills = []
+    for skill_md in sorted(SKILLS_DIR.glob("*/SKILL.md")):
+        text = skill_md.read_text()
+        if not text.startswith("---"):
+            continue
+        parts = text.split("---", 2)
+        if len(parts) < 3:
+            continue
+        try:
+            fm = yaml.safe_load(parts[1]) or {}
+        except yaml.YAMLError:
+            continue
+        name = fm.get("name") or skill_md.parent.name
+        description = (fm.get("description") or "").strip()
+        if not description:
+            continue
+        skills.append(
+            {
+                "name": name,
+                "description": description,
+                "_keywords": keyword_set(description),
+            }
+        )
+    return skills
+
+
+def build_clusters(skills: list[dict]) -> list[dict]:
+    # Pairwise edges where overlap & shared-keyword threshold is met.
+    edges: list[tuple[str, str, set[str], float]] = []
+    by_name = {s["name"]: s for s in skills}
+    for a, b in combinations(skills, 2):
+        shared = a["_keywords"] & b["_keywords"]
+        ov = overlap_fraction(a["_keywords"], b["_keywords"])
+        if len(shared) >= MIN_SHARED_KEYWORDS and ov >= KEYWORD_OVERLAP_THRESHOLD:
+            edges.append((a["name"], b["name"], shared, ov))
+
+    # Union-find over edge set → connected-component clusters.
+    parent: dict[str, str] = {}
+
+    def find(x: str) -> str:
+        parent.setdefault(x, x)
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(x: str, y: str) -> None:
+        rx, ry = find(x), find(y)
+        if rx != ry:
+            parent[rx] = ry
+
+    for a, b, _, _ in edges:
+        union(a, b)
+
+    components: dict[str, list[str]] = {}
+    for name in {n for edge in edges for n in edge[:2]}:
+        components.setdefault(find(name), []).append(name)
+
+    clusters: list[dict] = []
+    for idx, (_, members) in enumerate(sorted(components.items(), key=lambda kv: -len(kv[1])), start=1):
+        member_kws = [by_name[m]["_keywords"] for m in members]
+        shared_all = set.intersection(*member_kws) if member_kws else set()
+        member_edges = [(a, b, sk, ov) for a, b, sk, ov in edges if a in members and b in members]
+        max_ov = max((ov for *_, ov in member_edges), default=0.0)
+        clusters.append({
+            "cluster_id": f"C{idx:02d}",
+            "members": sorted(members),
+            "shared_keywords": sorted(shared_all),
+            "max_overlap": round(max_ov, 3),
+            "descriptions": {m: by_name[m]["description"] for m in sorted(members)},
+        })
+    return clusters[:TOP_N_CLUSTERS]
+
+
+def main() -> int:
+    if not SKILLS_DIR.exists():
+        print(f"❌  Skills dir not found: {SKILLS_DIR}", file=sys.stderr)
+        return 2
+    skills = load_skills()
+    clusters = build_clusters(skills)
+    OUT_JSON.parent.mkdir(parents=True, exist_ok=True)
+    OUT_JSON.write_text(json.dumps({
+        "skill_count": len(skills),
+        "cluster_count": len(clusters),
+        "clusters": clusters,
+    }, indent=2) + "\n")
+    print(f"✅  Wrote {OUT_JSON.relative_to(REPO_ROOT)} — {len(clusters)} clusters from {len(skills)} skills")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/skill_linter.py

@@ -775,8 +775,14 @@ def lint_skill(path: Path, text: str) -> LintResult:
     # is *both* large AND prose-dominant OR ships ≥ 2 independently invocable
     # procedures. Reference catalogues (quality-tools 411 L / density 0.83)
     # pass; multi-procedure skills are flagged for split.
+    #
+    # Frontmatter opt-out: `meta_skill: true` exempts a skill from the size
+    # warn when the skill's purpose *is* breadth (skill-writing, agent-docs-
+    # writing, skill-reviewer, etc.). Meta-skills inherently bundle multiple
+    # procedures and inline examples.
     total_lines = len(text.splitlines())
-    if total_lines > 400:
+    is_meta_skill = bool(fm) and re.search(r"^meta_skill:\s*true\s*$", fm, re.MULTILINE)
+    if total_lines > 400 and not is_meta_skill:
         density = _density_score(text)
         procedures = _count_procedure_sections(text)
         if density < 0.6 or procedures >= 2:
@@ -832,6 +838,12 @@ def lint_skill(path: Path, text: str) -> LintResult:
                                f"{meaningful_steps} steps) — may lack its own executable workflow"))
             suggestions.append("Expand the skill so it remains executable without opening a guideline")
 
+    # --- evals.json schema validator ---
+    # When a skill ships sibling `evals/evals.json` (quantitative behavior
+    # eval per skill-writing § 7), validate its shape. Triggers.json is a
+    # separate concern handled elsewhere. All issues here are WARN.
+    issues.extend(validate_evals_json(path))
+
     return LintResult(
         file=str(path),
         artifact_type="skill",
@@ -841,6 +853,64 @@ def lint_skill(path: Path, text: str) -> LintResult:
     )
 
 
+def validate_evals_json(skill_path: Path) -> list[Issue]:
+    """Validate `{skill_dir}/evals/evals.json` against the schema declared
+    in `skill-writing` § 7. Returns WARN-level issues only; never blocks.
+    Skipped entirely when the file is absent."""
+    evals_path = skill_path.parent / "evals" / "evals.json"
+    if not evals_path.is_file():
+        return []
+    issues: list[Issue] = []
+    try:
+        data = json.loads(evals_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        return [Issue("warning", "evals_json_unreadable",
+                      f"evals/evals.json could not be parsed: {exc}")]
+    if not isinstance(data, dict):
+        return [Issue("warning", "evals_json_shape",
+                      "evals/evals.json root must be an object")]
+    if "skill" not in data or not isinstance(data["skill"], str):
+        issues.append(Issue("warning", "evals_json_missing_skill",
+                            "evals/evals.json must declare top-level 'skill' (string)"))
+    scenarios = data.get("scenarios")
+    if not isinstance(scenarios, list) or len(scenarios) < 1:
+        issues.append(Issue("warning", "evals_json_no_scenarios",
+                            "evals/evals.json must declare 'scenarios' (non-empty array)"))
+        return issues
+    valid_kinds = {"contains", "file_exists", "rubric"}
+    for idx, scenario in enumerate(scenarios):
+        loc = f"scenarios[{idx}]"
+        if not isinstance(scenario, dict):
+            issues.append(Issue("warning", "evals_json_scenario_shape",
+                                f"{loc} must be an object"))
+            continue
+        for key in ("id", "prompt"):
+            if key not in scenario or not isinstance(scenario[key], str) or not scenario[key].strip():
+                issues.append(Issue("warning", "evals_json_scenario_missing_field",
+                                    f"{loc} missing required string field '{key}'"))
+        assertions = scenario.get("assertions")
+        if not isinstance(assertions, list) or len(assertions) < 1:
+            issues.append(Issue("warning", "evals_json_scenario_no_assertions",
+                                f"{loc}.assertions must be a non-empty array"))
+            continue
+        for a_idx, assertion in enumerate(assertions):
+            a_loc = f"{loc}.assertions[{a_idx}]"
+            if not isinstance(assertion, dict):
+                issues.append(Issue("warning", "evals_json_assertion_shape",
+                                    f"{a_loc} must be an object"))
+                continue
+            kind = assertion.get("kind")
+            if kind not in valid_kinds:
+                issues.append(Issue("warning", "evals_json_assertion_kind",
+                                    f"{a_loc}.kind must be one of {sorted(valid_kinds)}, got {kind!r}"))
+                continue
+            required_field = {"contains": "value", "file_exists": "path", "rubric": "criterion"}[kind]
+            if required_field not in assertion or not isinstance(assertion[required_field], str):
+                issues.append(Issue("warning", "evals_json_assertion_missing_field",
+                                    f"{a_loc} (kind={kind}) missing required string field '{required_field}'"))
+    return issues
+
+
 def extract_frontmatter(text: str) -> Optional[str]:
     match = FRONTMATTER_PATTERN.search(text)
     return match.group(1) if match else None