@event4u/agent-config 2.19.0 → 2.20.0

package/scripts/bench_drift_check.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Drift detector for the bench corpus — step-4 Phase 3 Step 2.
+
+Compares the latest `bench/reports/<stamp>-<corpus>.json` against the
+previous N reports (default 5) for the same corpus. Drift defined as:
+
+    - selection-accuracy: latest is more than `accuracy_drop_pp` below
+      the rolling mean (default 5 pp)
+    - cost: latest USD total is more than `cost_increase_pct` above the
+      rolling mean (default 20 %); skipped when source != "captured"
+    - quality: latest quality_score is more than `quality_drop_pp`
+      below the rolling mean (default 10 pp); skipped when source ==
+      "not_collected"
+
+Exit codes:
+    0 — no drift detected (or no baseline yet — warn-only)
+    1 — argument / read error
+    2 — drift detected (CI surface; not a merge gate per roadmap)
+
+CLI:
+    python3 scripts/bench_drift_check.py --corpus dev
+    python3 scripts/bench_drift_check.py --corpus dev --window 5 --json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from _lib import script_output  # type: ignore[import-not-found]  # noqa: E402
+
+
+def _load_reports(reports_dir: Path, corpus: str) -> list[tuple[Path, dict[str, Any]]]:
+    out: list[tuple[Path, dict[str, Any]]] = []
+    for p in sorted(reports_dir.glob(f"*-{corpus}.json")):
+        try:
+            out.append((p, json.loads(p.read_text(encoding="utf-8"))))
+        except (OSError, json.JSONDecodeError) as exc:
+            script_output.warn(f"  ⚠️  skip unreadable report {p.name}: {exc}")
+    return out
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values) if values else 0.0
+
+
+def _check(latest: dict[str, Any], baseline: list[dict[str, Any]],
+           thresholds: dict[str, float]) -> list[dict[str, Any]]:
+    findings: list[dict[str, Any]] = []
+
+    sel_latest = float(latest["selection"]["selection_accuracy"])
+    sel_baseline = _mean([float(r["selection"]["selection_accuracy"]) for r in baseline])
+    sel_drop_pp = (sel_baseline - sel_latest) * 100.0
+    if sel_drop_pp > thresholds["accuracy_drop_pp"]:
+        findings.append({
+            "axis": "selection_accuracy",
+            "latest": sel_latest, "baseline_mean": sel_baseline,
+            "delta_pp": -sel_drop_pp, "threshold_pp": -thresholds["accuracy_drop_pp"],
+        })
+
+    captured = [r for r in baseline + [latest] if r["cost"].get("source") == "captured"]
+    if len(captured) >= 2 and latest["cost"].get("source") == "captured":
+        cost_latest = float(latest["cost"]["totals"]["cost_usd"])
+        baseline_costs = [float(r["cost"]["totals"]["cost_usd"])
+                          for r in baseline if r["cost"].get("source") == "captured"]
+        if baseline_costs:
+            cost_baseline = _mean(baseline_costs)
+            if cost_baseline > 0:
+                pct = (cost_latest - cost_baseline) / cost_baseline * 100.0
+                if pct > thresholds["cost_increase_pct"]:
+                    findings.append({
+                        "axis": "cost_usd",
+                        "latest": cost_latest, "baseline_mean": cost_baseline,
+                        "delta_pct": pct, "threshold_pct": thresholds["cost_increase_pct"],
+                    })
+
+    if latest["quality"].get("source") != "not_collected":
+        q_latest = float(latest["quality"]["quality_score"])
+        q_baseline = _mean([float(r["quality"]["quality_score"])
+                            for r in baseline
+                            if r["quality"].get("source") != "not_collected"])
+        if q_baseline:
+            q_drop_pp = (q_baseline - q_latest) * 100.0
+            if q_drop_pp > thresholds["quality_drop_pp"]:
+                findings.append({
+                    "axis": "quality_score",
+                    "latest": q_latest, "baseline_mean": q_baseline,
+                    "delta_pp": -q_drop_pp, "threshold_pp": -thresholds["quality_drop_pp"],
+                })
+
+    return findings
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--corpus", default="dev")
+    ap.add_argument("--reports-dir", default="bench/reports")
+    ap.add_argument("--window", type=int, default=5, help="rolling window size (default 5)")
+    ap.add_argument("--accuracy-drop-pp", type=float, default=5.0)
+    ap.add_argument("--cost-increase-pct", type=float, default=20.0)
+    ap.add_argument("--quality-drop-pp", type=float, default=10.0)
+    ap.add_argument("--json", action="store_true", help="emit JSON instead of Markdown")
+    args = ap.parse_args(argv)
+
+    reports = _load_reports(REPO_ROOT / args.reports_dir, args.corpus)
+    if len(reports) < 2:
+        msg = (f"  ℹ️  bench-drift · corpus={args.corpus} · "
+               f"{len(reports)} report(s) — need ≥ 2 to compare; no drift gate yet.")
+        if args.json:
+            print(json.dumps({"status": "warmup", "reports": len(reports)}))
+        else:
+            print(msg)
+        return 0
+
+    latest_path, latest = reports[-1]
+    baseline = [r for _, r in reports[-(args.window + 1):-1]]
+    thresholds = {
+        "accuracy_drop_pp": args.accuracy_drop_pp,
+        "cost_increase_pct": args.cost_increase_pct,
+        "quality_drop_pp": args.quality_drop_pp,
+    }
+    findings = _check(latest, baseline, thresholds)
+
+    payload = {
+        "status": "drift" if findings else "ok",
+        "corpus": args.corpus,
+        "latest_report": latest_path.name,
+        "baseline_window": len(baseline),
+        "thresholds": thresholds,
+        "findings": findings,
+    }
+    if args.json:
+        print(json.dumps(payload, indent=2))
+    else:
+        emoji = "⚠️" if findings else "✅"
+        print(f"  {emoji}  bench-drift · corpus={args.corpus} · "
+              f"latest={latest_path.name} · window={len(baseline)} · "
+              f"findings={len(findings)}")
+        for f in findings:
+            print(f"     · {f['axis']}: latest={f['latest']:.4f} "
+                  f"baseline_mean={f['baseline_mean']:.4f}")
+    return 2 if findings else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/bench_per_tool.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Per-tool projection-fidelity bench — step-4 Phase 4.
+
+Re-runs the keyword-overlap selection scorer against each projected
+tool surface and computes:
+
+    fidelity(tool) = selection_accuracy(tool) / selection_accuracy(reference)
+
+Reference = Augment projection (most complete per roadmap). Threshold
+for "fit for purpose" is >= 0.85.
+
+Surfaces:
+- `.augment/skills/`   skill projection      automated  (reference)
+- `.claude/skills/`    skill projection      automated
+- `.cursor/rules/`     rules-only            not_applicable (no skill projection)
+- `.windsurfrules`     single concatenated   not_applicable
+- `.clinerules/`       rules-only            not_applicable
+
+Usage:
+    python3 scripts/bench_per_tool.py --corpus dev
+    python3 scripts/bench_per_tool.py --corpus dev --json
+    python3 scripts/bench_per_tool.py --corpus dev --threshold 0.85
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import re
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write("error: PyYAML required (pip install pyyaml)\n")
+    sys.exit(2)
+
+# Reuse tokenization + ranking from the reference runner so the only
+# axis that changes between tools is the skill catalogue on disk.
+from bench_runner import rank_skills  # type: ignore  # noqa: E402
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+CORPUS_DIR = REPO_ROOT / "tests" / "eval"
+REPORTS_DIR = REPO_ROOT / "bench" / "reports"
+
+# tool_id -> (skills_root, kind). kind = "skills" | "rules_only" | "single_file".
+SURFACES: dict[str, tuple[Path, str]] = {
+    "augment": (REPO_ROOT / ".augment" / "skills", "skills"),
+    "claude":  (REPO_ROOT / ".claude" / "skills",  "skills"),
+    "cursor":  (REPO_ROOT / ".cursor" / "rules",   "rules_only"),
+    "cline":   (REPO_ROOT / ".clinerules",         "rules_only"),
+    "windsurf":(REPO_ROOT / ".windsurfrules",      "single_file"),
+}
+
+REFERENCE_TOOL = "augment"
+
+
+def load_descriptions(root: Path) -> dict[str, str]:
+    """Return {skill_name: 'name + description'} for SKILL.md files under root."""
+    out: dict[str, str] = {}
+    if not root.is_dir():
+        return out
+    for skill_dir in sorted(root.iterdir()):
+        skill_md = skill_dir / "SKILL.md"
+        if not skill_md.is_file():
+            continue
+        text = skill_md.read_text(encoding="utf-8")
+        m = re.search(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL)
+        if not m:
+            continue
+        try:
+            fm = yaml.safe_load(m.group(1)) or {}
+        except yaml.YAMLError:
+            continue
+        desc = fm.get("description") or ""
+        name = fm.get("name") or skill_dir.name
+        if desc:
+            out[name] = f"{name} {desc}"
+    return out
+
+
+def score_corpus(skills: dict[str, str], prompts: list[dict], top_k: int) -> dict:
+    hits = 0
+    per_prompt = []
+    for p in prompts:
+        ranked = rank_skills(p["prompt"], skills, top_k)
+        expected = set(p.get("expected_skills", []))
+        hit = bool(expected & set(ranked))
+        if hit:
+            hits += 1
+        per_prompt.append({"id": p["id"], "expected": sorted(expected),
+                           "ranked": ranked, "hit": hit})
+    n = len(prompts)
+    return {
+        "prompts_total": n,
+        "prompts_hit": hits,
+        "selection_accuracy": round(hits / n, 4) if n else 0.0,
+        "skill_count": len(skills),
+        "per_prompt": per_prompt,
+    }
+
+
+def evaluate(corpus_path: Path, top_k: int, threshold: float) -> dict:
+    corpus = yaml.safe_load(corpus_path.read_text(encoding="utf-8"))
+    prompts = corpus["prompts"]
+    results: dict[str, dict] = {}
+
+    for tool, (root, kind) in SURFACES.items():
+        if kind != "skills":
+            results[tool] = {
+                "status": "not_applicable",
+                "reason": f"surface is {kind}; no SKILL.md projection",
+                "path": str(root.relative_to(REPO_ROOT)),
+            }
+            continue
+        skills = load_descriptions(root)
+        if not skills:
+            results[tool] = {"status": "error", "reason": "no skills found",
+                             "path": str(root.relative_to(REPO_ROOT))}
+            continue
+        scored = score_corpus(skills, prompts, top_k)
+        scored["status"] = "ok"
+        scored["path"] = str(root.relative_to(REPO_ROOT))
+        results[tool] = scored
+
+    ref = results.get(REFERENCE_TOOL, {})
+    ref_acc = ref.get("selection_accuracy", 0.0) if ref.get("status") == "ok" else 0.0
+    below = []
+    for tool, r in results.items():
+        if r.get("status") != "ok":
+            continue
+        fidelity = (r["selection_accuracy"] / ref_acc) if ref_acc else 0.0
+        r["fidelity"] = round(fidelity, 4)
+        r["passed_threshold"] = fidelity >= threshold
+        if tool != REFERENCE_TOOL and not r["passed_threshold"]:
+            below.append(tool)
+
+    return {
+        "schema": "projection-fidelity-v1",
+        "generated_at": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "corpus_id": corpus["corpus_id"],
+        "top_k": top_k,
+        "threshold": threshold,
+        "reference_tool": REFERENCE_TOOL,
+        "reference_accuracy": ref_acc,
+        "tools": results,
+        "below_threshold": below,
+    }
+
+
+def render_markdown(summary: dict) -> str:
+    lines = [
+        f"# Projection fidelity — {summary['corpus_id']}",
+        "",
+        f"_Generated {summary['generated_at']} · top-K={summary['top_k']} · "
+        f"threshold={summary['threshold']:.2f} · reference=`{summary['reference_tool']}`_",
+        "",
+        "| tool | status | skills | accuracy | fidelity | pass |",
+        "|---|---|---:|---:|---:|---|",
+    ]
+    for tool, r in summary["tools"].items():
+        status = r.get("status", "?")
+        if status != "ok":
+            lines.append(f"| `{tool}` | {status} | — | — | — | — |")
+            continue
+        lines.append(
+            f"| `{tool}` | ok | {r['skill_count']} | "
+            f"{r['selection_accuracy']:.2%} | {r['fidelity']:.2f} | "
+            f"{'✅' if r['passed_threshold'] else '❌'} |"
+        )
+    if summary["below_threshold"]:
+        lines += ["", f"**Below threshold:** {', '.join(summary['below_threshold'])} "
+                  f"→ inspect `scripts/_lib/generate_tools.py` projection mapping."]
+    else:
+        lines += ["", "**All projections fit-for-purpose** (≥ threshold)."]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--corpus", default="dev")
+    ap.add_argument("--top-k", type=int, default=3)
+    ap.add_argument("--threshold", type=float, default=0.85)
+    ap.add_argument("--json", action="store_true")
+    ap.add_argument("--write-report", action="store_true",
+                    help="emit bench/reports/<ts>-<corpus>-projection.{json,md}")
+    args = ap.parse_args(argv)
+
+    corpus_path = CORPUS_DIR / f"corpus-{args.corpus}.yaml"
+    if not corpus_path.is_file():
+        sys.stderr.write(f"error: corpus not found: {corpus_path}\n")
+        return 2
+
+    summary = evaluate(corpus_path, args.top_k, args.threshold)
+
+    if args.write_report:
+        REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+        stamp = summary["generated_at"].replace(":", "-")
+        base = REPORTS_DIR / f"{stamp}-{args.corpus}-projection"
+        base.with_suffix(".json").write_text(json.dumps(summary, indent=2) + "\n")
+        base.with_suffix(".md").write_text(render_markdown(summary))
+        sys.stderr.write(f"wrote {base}.json + {base}.md\n")
+
+    if args.json:
+        print(json.dumps(summary, indent=2))
+    else:
+        print(render_markdown(summary))
+
+    return 1 if summary["below_threshold"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

package/scripts/bench_run.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""Bench orchestrator — step-4 measurement-and-benchmark Phase 2.
+
+Wraps the selection-accuracy baseline collector (`scripts/bench_runner.py`),
+captures token / cost data from `agents/cost-tracking/sessions.jsonl` if
+present (per ruflo pattern, external-findings § 2), runs structural
+quality assertions per prompt, and emits a versioned JSON + Markdown
+report under `bench/reports/` per
+`docs/contracts/benchmark-report-schema.md`.
+
+Usage:
+    python3 scripts/bench_run.py --corpus dev
+    python3 scripts/bench_run.py --corpus dev --top-k 3 --quiet
+    python3 scripts/bench_run.py --corpus dev --agent-output outputs.json
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from _lib import script_output  # type: ignore[import-not-found]  # noqa: E402
+from _lib.bench_cost import aggregate_sessions  # noqa: E402
+from _lib.bench_quality import score_corpus  # noqa: E402
+from _lib.bench_report import (  # noqa: E402
+    report_paths,
+    render_markdown,
+    utc_now_filename_stamp,
+    utc_now_iso,
+    write_json,
+    write_markdown,
+)
+from bench_runner import run_corpus  # noqa: E402
+
+try:
+    import yaml
+except ImportError:
+    script_output.error("error: PyYAML required (pip install pyyaml)")
+    sys.exit(2)
+
+BENCH_RUN_VERSION = "0.1.0"
+PRICING_PATH = REPO_ROOT / "bench" / "pricing.yaml"
+SESSIONS_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
+REPORTS_DIR = REPO_ROOT / "bench" / "reports"
+CORPUS_DIR = REPO_ROOT / "tests" / "eval"
+BASELINE_COLLECTOR = REPO_ROOT / "scripts" / "bench_runner.py"
+
+
+def _baseline_sha_or_mtime() -> str:
+    try:
+        return f"mtime:{int(BASELINE_COLLECTOR.stat().st_mtime)}"
+    except OSError:
+        return "unavailable"
+
+
+def _verdict(selection: dict, quality: dict) -> dict[str, str]:
+    sel = "pass" if selection["passed"] else "fail"
+    if quality["source"] == "not_collected":
+        qual = "not_collected"
+        overall = "partial"
+    else:
+        qual = "pass" if quality["quality_score"] >= 0.60 else "fail"
+        overall = "pass" if (sel == "pass" and qual == "pass") else "fail"
+    return {"selection": sel, "quality": qual, "overall": overall}
+
+
+def build_report(
+    corpus_path: Path,
+    top_k: int,
+    agent_output: Path | None,
+) -> dict:
+    selection = run_corpus(corpus_path, top_k)
+    corpus_yaml = yaml.safe_load(corpus_path.read_text(encoding="utf-8"))
+    prompts = corpus_yaml.get("prompts", [])
+    cost = aggregate_sessions(SESSIONS_JSONL, PRICING_PATH)
+    quality = score_corpus(prompts, agent_output)
+    verdict = _verdict(selection, quality)
+    return {
+        "schema_version": 1,
+        "generated_at": utc_now_iso(),
+        "corpus": {
+            "id": selection["corpus_id"],
+            "path": str(corpus_path.relative_to(REPO_ROOT)),
+            "prompt_count": len(prompts),
+        },
+        "runner": {
+            "bench_run_version": BENCH_RUN_VERSION,
+            "baseline_collector": str(BASELINE_COLLECTOR.relative_to(REPO_ROOT)),
+            "baseline_collector_sha": _baseline_sha_or_mtime(),
+        },
+        "selection": selection,
+        "cost": cost,
+        "quality": quality,
+        "verdict": verdict,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--corpus", default="dev", help="corpus id (default: dev)")
+    ap.add_argument("--top-k", type=int, default=3)
+    ap.add_argument("--agent-output", type=Path, default=None,
+                    help="Path to JSON {id: output_text} for quality scoring (Phase 3)")
+    ap.add_argument("--quiet", action="store_true",
+                    help="Print only the report path + headline")
+    ap.add_argument("--stamp", default=None,
+                    help="Override timestamp (test hook); defaults to UTC now")
+    ap.add_argument("--no-write", action="store_true",
+                    help="Compute the report but do not write files (dry run)")
+    args = ap.parse_args(argv)
+
+    corpus_path = CORPUS_DIR / f"corpus-{args.corpus}.yaml"
+    if not corpus_path.is_file():
+        script_output.error(f"error: corpus not found: {corpus_path}")
+        return 2
+
+    report = build_report(corpus_path, args.top_k, args.agent_output)
+    stamp = args.stamp or utc_now_filename_stamp()
+    json_path, md_path = report_paths(REPORTS_DIR, report["corpus"]["id"], stamp)
+
+    if not args.no_write:
+        write_json(json_path, report)
+        write_markdown(md_path, report)
+
+    verdict = report["verdict"]
+    sel = report["selection"]
+    qual = report["quality"]
+    cost = report["cost"]
+    headline = (
+        f"bench {report['corpus']['id']} · "
+        f"selection {sel['selection_accuracy']:.2%} ({verdict['selection']}) · "
+        f"cost ${cost['totals']['total_cost_usd']:.6f} ({cost.get('source', 'n/a')}) · "
+        f"quality {qual['quality_score']:.2%} ({verdict['quality']}) · "
+        f"overall {verdict['overall']}"
+    )
+
+    if args.quiet:
+        print(headline)
+        if not args.no_write:
+            print(f"report: {md_path.relative_to(REPO_ROOT)}")
+    else:
+        print(render_markdown(report))
+        if not args.no_write:
+            print(f"\n→ json:     {json_path.relative_to(REPO_ROOT)}")
+            print(f"→ markdown: {md_path.relative_to(REPO_ROOT)}")
+
+    # Exit zero on overall pass OR partial (partial = quality_not_collected by design).
+    return 0 if verdict["overall"] in ("pass", "partial") else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/scripts/config/__init__.py

@@ -0,0 +1,9 @@
+"""Config-layer loaders (profile, preset, pack).
+
+Phase 1 of step-15 product refinement. Single home for the audience /
+governance / workflow axes introduced by
+:mod:`docs.contracts.profile-system`,
+:mod:`docs.contracts.config-presets`, and the upcoming workflow-packs
+contract. Loaders here are pure, read-only, lazy-PyYAML; they layer on
+top of :mod:`scripts._lib.agent_settings` for project-root anchoring.
+"""