@event4u/agent-config 5.10.0 → 6.0.0

package/scripts/skill_tools/run_block_d_eval.py

@@ -1,129 +0,0 @@
-#!/usr/bin/env python3
-"""Block D · D5 — eval gate runner.
-
-Runs D2 (`score_skill_relevance`), D3 (`audit_persona_coverage`), and
-D4 (`suggest_skill_for_task`) against the corpora in
-`agents/evidence/eval-corpora/block-d/` and emits a pass/fail summary per the
-council verdict targets:
-
-  - **D2**: ≥ 85 % of corpus tasks have an `expected_top3` skill in
-    the actual top-3 ranking.
-  - **D3**: ≥ 2 personas flagged as `under-cited`.
-  - **D4**: ≥ 3 / 5 blind tasks where suggestion #1 matches the
-    human-curated top-1.
-
-Pilot pass = ≥ 2 / 3 tools pass. Anything less → kill switch.
-
-Stdlib-only. Embedded `_SAMPLE` for self-demo.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from pathlib import Path
-from typing import Dict, List
-
-from skill_tools.audit_persona_coverage import audit  # type: ignore
-from skill_tools.score_skill_relevance import (  # type: ignore
-    DEFAULT_SKILLS_DIR,
-    rank,
-)
-from skill_tools.suggest_skill_for_task import suggest  # type: ignore
-
-ROOT = Path(__file__).resolve().parents[2]
-CORPUS_DIR = ROOT / "agents" / "eval-corpora" / "block-d"
-PERSONAS_DIR = ROOT / ".agent-src.uncondensed" / "personas"
-
-
-def _eval_d2(corpus: Path, skills_dir: Path) -> Dict[str, object]:
-    data = json.loads(corpus.read_text(encoding="utf-8"))
-    tasks = data["tasks"]
-    hits, misses = 0, []
-    for t in tasks:
-        ranked = rank(t["task"], skills_dir)[:3]
-        names = [n for n, _, _ in ranked]
-        if any(e in names for e in t["expected_top3"]):
-            hits += 1
-        else:
-            misses.append({"id": t["id"], "expected": t["expected_top3"],
-                           "got": names})
-    pct = hits / len(tasks) if tasks else 0.0
-    return {"hits": hits, "total": len(tasks), "pct": round(pct, 3),
-            "passed": pct >= 0.85, "misses": misses}
-
-
-def _eval_d3(skills_dir: Path, personas_dir: Path) -> Dict[str, object]:
-    rows = audit(skills_dir, personas_dir)
-    flagged = [r["persona"] for r in rows if r["status"] == "under-cited"]
-    return {"flagged": flagged, "count": len(flagged),
-            "passed": len(flagged) >= 2}
-
-
-def _eval_d4(corpus: Path, skills_dir: Path,
-             personas_dir: Path) -> Dict[str, object]:
-    data = json.loads(corpus.read_text(encoding="utf-8"))
-    tasks = data["tasks"]
-    hits, misses = 0, []
-    for t in tasks:
-        out = suggest(t["task"], skills_dir, personas_dir, top=1)
-        got = out[0]["skill"] if out else None
-        if got == t["expected_top1"]:
-            hits += 1
-        else:
-            misses.append({"id": t["id"], "expected": t["expected_top1"],
-                           "got": got})
-    return {"hits": hits, "total": len(tasks),
-            "passed": hits >= 3, "misses": misses}
-
-
-def run_all(skills_dir: Path, personas_dir: Path,
-            corpus_dir: Path) -> Dict[str, object]:
-    d2 = _eval_d2(corpus_dir / "d2-tasks.json", skills_dir)
-    d3 = _eval_d3(skills_dir, personas_dir)
-    d4 = _eval_d4(corpus_dir / "d4-tasks.json", skills_dir, personas_dir)
-    passes = sum(1 for r in (d2, d3, d4) if r["passed"])
-    return {"D2": d2, "D3": d3, "D4": d4,
-            "tools_passed": passes,
-            "pilot_passed": passes >= 2}
-
-
-def _print_human(report: Dict[str, object]) -> None:
-    icons = {True: "✅", False: "❌"}
-    for key in ("D2", "D3", "D4"):
-        r: Dict[str, object] = report[key]  # type: ignore[assignment]
-        print(f"  {icons[bool(r['passed'])]}  {key}: {_summary(key, r)}")
-    overall = bool(report["pilot_passed"])
-    print(f"\n  pilot: {report['tools_passed']}/3 tools passed → "
-          f"{'PASS' if overall else 'FAIL'}")
-
-
-def _summary(key: str, r: Dict[str, object]) -> str:
-    if key == "D2":
-        return f"{r['hits']}/{r['total']} ({float(r['pct']) * 100:.0f}%) ≥ 85% target"
-    if key == "D3":
-        return f"{r['count']} under-cited personas (≥ 2 target)"
-    return f"{r['hits']}/{r['total']} top-1 hits (≥ 3/5 target)"
-
-
-def main(argv: List[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
-    parser.add_argument("--skills-dir", default=str(DEFAULT_SKILLS_DIR))
-    parser.add_argument("--personas-dir", default=str(PERSONAS_DIR))
-    parser.add_argument("--corpus-dir", default=str(CORPUS_DIR))
-    parser.add_argument("--json", action="store_true")
-    args = parser.parse_args(argv)
-    report = run_all(Path(args.skills_dir), Path(args.personas_dir),
-                     Path(args.corpus_dir))
-    if args.json:
-        json.dump(report, sys.stdout, indent=2)
-        sys.stdout.write("\n")
-    else:
-        _print_human(report)
-    return 0 if report["pilot_passed"] else 1
-
-
-_SAMPLE = {"corpus_dir": str(CORPUS_DIR)}
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/scripts/skill_tools/score_skill_relevance.py

@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-"""Block D · D2 — score_skill_relevance.
-
-Rank skills by relevance to a free-form task description.
-
-Heuristic (council iter-1 D-OQ1 verdict (b) — discovery-story tool 1):
-
-  score = keyword_overlap * 70 + persona_match * 30
-
-where:
-  - keyword_overlap = |task_terms ∩ skill_terms| / |task_terms|
-    (skill_terms = tokens from `name` + `description`)
-  - persona_match  = 1.0 if any persona on the skill is named or
-    role-mentioned in the task, else 0.0
-
-Inputs:
-  --task TEXT      — task description (required)
-  --skills-dir DIR — directory holding SKILL.md files (default: package skills)
-  --top N          — emit only top-N ranked skills (default: all non-zero)
-  --json           — machine-readable ranked output
-
-Output: ranked list with integer scores 0–100, descending. Ties break on name.
-
-Stdlib-only. ≤ 180 LOC. Embedded `_SAMPLE` for self-demo (`python3 -m … --json`).
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import re
-import sys
-from pathlib import Path
-from typing import Dict, Iterable, List, Tuple
-
-ROOT = Path(__file__).resolve().parents[2]
-DEFAULT_SKILLS_DIR = ROOT / ".agent-src.uncondensed" / "skills"
-TOKEN_RE = re.compile(r"[a-z][a-z0-9]+")
-STOPWORDS = frozenset({
-    "the", "a", "an", "and", "or", "but", "of", "for", "with", "to", "in",
-    "on", "at", "by", "from", "as", "is", "are", "was", "were", "be", "been",
-    "this", "that", "these", "those", "it", "its", "use", "when", "even",
-    "via", "via:", "into", "onto", "use:", "skill", "skills", "task", "tasks",
-    "code", "file", "files", "doing", "make", "do", "go", "get", "set",
-    "not", "no", "yes", "any", "some", "all", "one", "two", "new", "old",
-    "user", "users", "our", "your", "their", "they", "we", "you", "i", "me",
-})
-
-
-def _tokenize(text: str) -> set:
-    return {t for t in TOKEN_RE.findall(text.lower()) if t not in STOPWORDS and len(t) > 2}
-
-
-def _parse_frontmatter(path: Path) -> Dict[str, object]:
-    """Minimal YAML-frontmatter reader (stdlib-only). Returns {} on parse miss."""
-    text = path.read_text(encoding="utf-8", errors="replace")
-    if not text.startswith("---"):
-        return {}
-    end = text.find("\n---", 3)
-    if end == -1:
-        return {}
-    block = text[3:end]
-    out: Dict[str, object] = {}
-    current_list_key: str | None = None
-    for raw in block.splitlines():
-        line = raw.rstrip()
-        if not line or line.startswith("#"):
-            continue
-        if current_list_key and line.startswith("  - "):
-            out.setdefault(current_list_key, []).append(line[4:].strip())  # type: ignore[union-attr]
-            continue
-        current_list_key = None
-        m = re.match(r"^([a-zA-Z_][\w-]*)\s*:\s*(.*)$", line)
-        if not m:
-            continue
-        key, val = m.group(1), m.group(2).strip()
-        if val == "":
-            current_list_key = key
-            continue
-        if val.startswith('"') and val.endswith('"'):
-            val = val[1:-1]
-        out[key] = val
-    return out
-
-
-def _load_skills(skills_dir: Path) -> List[Dict[str, object]]:
-    skills: List[Dict[str, object]] = []
-    for skill_md in sorted(skills_dir.glob("*/SKILL.md")):
-        fm = _parse_frontmatter(skill_md)
-        name = str(fm.get("name") or skill_md.parent.name)
-        desc = str(fm.get("description") or "")
-        personas = fm.get("personas") or []
-        if isinstance(personas, str):
-            personas = [personas]
-        skills.append({
-            "name": name,
-            "description": desc,
-            "personas": list(personas),
-            "terms": _tokenize(name + " " + desc),
-        })
-    return skills
-
-
-def _score(task_terms: set, skill: Dict[str, object]) -> int:
-    if not task_terms:
-        return 0
-    skill_terms = skill["terms"]  # type: ignore[index]
-    overlap = len(task_terms & skill_terms) / max(len(task_terms), 1)  # type: ignore[arg-type]
-    persona_hit = 0.0
-    task_lower = " ".join(task_terms)
-    for persona in skill["personas"]:  # type: ignore[union-attr]
-        slug = str(persona).lower()
-        if slug in task_lower or any(part in task_terms for part in slug.split("-")):  # type: ignore[operator]
-            persona_hit = 1.0
-            break
-    return round(overlap * 70 + persona_hit * 30)
-
-
-def rank(task: str, skills_dir: Path) -> List[Tuple[str, int, List[str]]]:
-    task_terms = _tokenize(task)
-    skills = _load_skills(skills_dir)
-    rows: List[Tuple[str, int, List[str]]] = []
-    for s in skills:
-        score = _score(task_terms, s)
-        if score > 0:
-            rows.append((str(s["name"]), score, list(s["personas"])))  # type: ignore[arg-type]
-    rows.sort(key=lambda r: (-r[1], r[0]))
-    return rows
-
-
-def _print_human(rows: Iterable[Tuple[str, int, List[str]]], top: int | None) -> None:
-    rows = list(rows)
-    if top:
-        rows = rows[:top]
-    if not rows:
-        print("(no relevant skills found)")
-        return
-    width = max(len(r[0]) for r in rows)
-    for name, score, personas in rows:
-        persona_str = ", ".join(personas) if personas else "—"
-        print(f"  {score:3d}  {name:<{width}}  {persona_str}")
-
-
-def main(argv: List[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
-    parser.add_argument("--task", required=False, default="",
-                        help="task description (required unless --sample is used)")
-    parser.add_argument("--skills-dir", default=str(DEFAULT_SKILLS_DIR),
-                        help="directory holding SKILL.md files")
-    parser.add_argument("--top", type=int, default=0, help="emit only top-N rows")
-    parser.add_argument("--json", action="store_true", help="emit JSON instead of text")
-    parser.add_argument("--sample", action="store_true", help="run against the embedded sample task")
-    args = parser.parse_args(argv)
-    task = _SAMPLE["task"] if args.sample else args.task
-    if not task:
-        parser.error("--task is required (or pass --sample)")
-    rows = rank(task, Path(args.skills_dir))
-    if args.json:
-        payload = [{"name": n, "score": s, "personas": p} for n, s, p in (rows[:args.top] if args.top else rows)]
-        json.dump({"task": task, "ranked": payload}, sys.stdout, indent=2)
-        sys.stdout.write("\n")
-    else:
-        _print_human(rows, args.top or None)
-    return 0
-
-
-_SAMPLE = {"task": "build a livewire component for the user dashboard with reactive state"}
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/scripts/skill_tools/suggest_skill_for_task.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-"""Block D · D4 — suggest_skill_for_task.
-
-CLI wrapper that combines D2 (`score_skill_relevance`) with the persona
-matrix from D3 (`audit_persona_coverage`) and emits the top-3 skill +
-persona combos with a one-line justification each.
-
-Inputs:
-  --task TEXT        — task description (required)
-  --skills-dir DIR   — SKILL.md directory
-  --personas-dir DIR — persona Markdown directory
-  --top N            — emit top-N combos (default: 3)
-  --json             — machine-readable output
-
-Output: ranked combos with `skill`, `score`, `personas[]`, and `why`.
-
-Stdlib-only. ≤ 100 LOC. Embedded `_SAMPLE` for self-demo via `--sample`.
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from pathlib import Path
-from typing import Dict, List
-
-from skill_tools.audit_persona_coverage import audit  # type: ignore
-from skill_tools.score_skill_relevance import (  # type: ignore
-    DEFAULT_SKILLS_DIR,
-    rank,
-)
-
-ROOT = Path(__file__).resolve().parents[2]
-DEFAULT_PERSONAS = ROOT / ".agent-src.uncondensed" / "personas"
-
-
-def _persona_status(rows: List[Dict[str, object]]) -> Dict[str, str]:
-    return {str(r["persona"]): str(r["status"]) for r in rows}
-
-
-def _justify(name: str, score: int, personas: List[str],
-             status: Dict[str, str]) -> str:
-    if score >= 70:
-        head = "high keyword + persona match"
-    elif score >= 40:
-        head = "strong keyword overlap"
-    else:
-        head = "partial overlap — confirm with reviewer"
-    if personas:
-        tier_hits = ", ".join(
-            f"{p} ({status.get(p, 'unknown')})" for p in personas
-        )
-        return f"{head}; lenses: {tier_hits}"
-    return f"{head}; no persona declared on `{name}`"
-
-
-def suggest(task: str, skills_dir: Path, personas_dir: Path,
-            top: int = 3) -> List[Dict[str, object]]:
-    ranked = rank(task, skills_dir)[:top]
-    persona_rows = audit(skills_dir, personas_dir)
-    status = _persona_status(persona_rows)
-    return [
-        {
-            "skill": name,
-            "score": score,
-            "personas": personas,
-            "why": _justify(name, score, personas, status),
-        }
-        for name, score, personas in ranked
-    ]
-
-
-def _print_human(combos: List[Dict[str, object]]) -> None:
-    if not combos:
-        print("(no skill suggestions for this task)")
-        return
-    for i, c in enumerate(combos, 1):
-        personas = ", ".join(c["personas"]) if c["personas"] else "—"  # type: ignore[arg-type]
-        print(f"  {i}. {c['skill']}  ({c['score']}/100)")
-        print(f"     personas: {personas}")
-        print(f"     why: {c['why']}")
-
-
-def main(argv: List[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
-    parser.add_argument("--task", default="",
-                        help="task description (required unless --sample)")
-    parser.add_argument("--skills-dir", default=str(DEFAULT_SKILLS_DIR))
-    parser.add_argument("--personas-dir", default=str(DEFAULT_PERSONAS))
-    parser.add_argument("--top", type=int, default=3)
-    parser.add_argument("--json", action="store_true",
-                        help="emit JSON instead of text")
-    parser.add_argument("--sample", action="store_true",
-                        help="run against the embedded sample task")
-    args = parser.parse_args(argv)
-    task = _SAMPLE["task"] if args.sample else args.task
-    if not task:
-        parser.error("--task is required (or pass --sample)")
-    combos = suggest(task, Path(args.skills_dir),
-                     Path(args.personas_dir), args.top)
-    if args.json:
-        json.dump({"task": task, "suggestions": combos},
-                  sys.stdout, indent=2)
-        sys.stdout.write("\n")
-    else:
-        _print_human(combos)
-    return 0
-
-
-_SAMPLE = {"task": "review a livewire component for accessibility and reactive state"}
-
-if __name__ == "__main__":
-    raise SystemExit(main())