@event4u/agent-config 3.3.0 → 4.2.0

package/scripts/_lib/bench_ab_scoring.py

@@ -0,0 +1,209 @@
+"""Structural success criteria for Track B.
+
+Phase 4 Step 3 of `agents/roadmaps/road-to-package-impact-benchmark.md`.
+
+No LLM-judge. Each criterion is a syntactic or behavioural check executable
+against the post-run working tree + the captured transcript. If the structural
+signal turns out too weak, a separate follow-up roadmap adds an LLM judge —
+not this one.
+
+Per-category criteria, expressed as keys in the task's `success_criteria`
+dict (see internal/bench/corpora/ab-trackb.yaml):
+
+- `target_file_modified`: <path>            — file at <path> changed between
+  the pre-run snapshot and the post-run snapshot.
+- `regex_in_target`: <pattern>              — pattern found in the named
+  target_file (case-insensitive).
+- `regex_in_any`: <pattern>                 — pattern found in any modified file.
+- `new_test_file_exists`: <path>            — new test file present after the run.
+- `test_assertion_added`: <path>            — file contains at least one
+  `assert` / `expect(` / `test(` call.
+- `one_of_files_modified`: [<paths>]        — at least one path modified.
+- `preserves_public_api`: [<names>]         — each name still exported / present.
+- `transcript_contains_one_of`: [<strings>] — any string appears in the
+  transcript (case-insensitive).
+- `no_file_write_before_audit`: bool        — if true, transcript shows an
+  audit reference before the first write tool call (UI-audit category).
+- `no_existing_test_removed`: [<names>]     — pre-existing test names still
+  present in the file.
+- `min_test_count`: int                     — at least N `test(` /
+  `it(` / `describe(` calls.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+
+def _read(path: Path) -> str:
+    try:
+        return path.read_text(errors="replace")
+    except OSError:
+        return ""
+
+
+def _file_changed(pre: dict[str, str], post: dict[str, str], rel: str) -> bool:
+    return pre.get(rel) != post.get(rel)
+
+
+def _has_regex(text: str, pattern: str) -> bool:
+    return bool(re.search(pattern, text, re.IGNORECASE | re.MULTILINE))
+
+
+def _count_regex(text: str, pattern: str) -> int:
+    return len(re.findall(pattern, text, re.IGNORECASE | re.MULTILINE))
+
+
+def score_task(
+    task: dict[str, Any],
+    *,
+    pre_snapshot: dict[str, str],
+    post_snapshot: dict[str, str],
+    clone_root: Path,
+    transcript: str,
+) -> dict[str, Any]:
+    """Score one task. Returns {passed: bool, checks: [{name, ok, reason}]}."""
+    crit = task.get("success_criteria") or {}
+    checks: list[dict[str, Any]] = []
+
+    def add(name: str, ok: bool, reason: str = "") -> None:
+        checks.append({"name": name, "ok": bool(ok), "reason": reason})
+
+    # target_file_modified
+    target_modified_path = crit.get("target_file_modified")
+    if target_modified_path:
+        ok = _file_changed(pre_snapshot, post_snapshot, target_modified_path)
+        add("target_file_modified", ok, f"file: {target_modified_path}")
+
+    # regex_in_target — uses target_file_modified path, or the new_test_file_exists path
+    regex_target_pattern = crit.get("regex_in_target")
+    if regex_target_pattern:
+        target_rel = (
+            crit.get("target_file_modified")
+            or crit.get("new_test_file_exists")
+            or ""
+        )
+        body = _read(clone_root / target_rel) if target_rel else ""
+        ok = _has_regex(body, regex_target_pattern)
+        add(
+            "regex_in_target",
+            ok,
+            f"pattern={regex_target_pattern!r} in {target_rel!r}",
+        )
+
+    # regex_in_any
+    regex_any_pattern = crit.get("regex_in_any")
+    if regex_any_pattern:
+        modified_files = [
+            rel for rel in post_snapshot if _file_changed(pre_snapshot, post_snapshot, rel)
+        ]
+        ok = any(
+            _has_regex(_read(clone_root / rel), regex_any_pattern)
+            for rel in modified_files
+        )
+        add(
+            "regex_in_any",
+            ok,
+            f"pattern={regex_any_pattern!r} across {len(modified_files)} modified files",
+        )
+
+    # new_test_file_exists
+    new_test = crit.get("new_test_file_exists")
+    if new_test:
+        ok = (clone_root / new_test).exists() and new_test not in pre_snapshot
+        add("new_test_file_exists", ok, f"path={new_test}")
+
+    # test_assertion_added
+    test_target = crit.get("test_assertion_added")
+    if test_target:
+        body = _read(clone_root / test_target)
+        ok = _has_regex(body, r"assert|expect\(|test\(|it\(")
+        add("test_assertion_added", ok, f"in {test_target}")
+
+    # one_of_files_modified
+    one_of = crit.get("one_of_files_modified")
+    if isinstance(one_of, list) and one_of:
+        ok = any(_file_changed(pre_snapshot, post_snapshot, rel) for rel in one_of)
+        add("one_of_files_modified", ok, f"any of: {one_of}")
+
+    # preserves_public_api
+    api = crit.get("preserves_public_api")
+    if isinstance(api, list) and api and target_modified_path:
+        body = _read(clone_root / target_modified_path)
+        missing = [name for name in api if name not in body]
+        add(
+            "preserves_public_api",
+            not missing,
+            f"missing: {missing}" if missing else "all present",
+        )
+
+    # transcript_contains_one_of
+    transcript_one_of = crit.get("transcript_contains_one_of")
+    if isinstance(transcript_one_of, list) and transcript_one_of:
+        lt = (transcript or "").lower()
+        ok = any(s.lower() in lt for s in transcript_one_of)
+        add(
+            "transcript_contains_one_of",
+            ok,
+            f"any of: {transcript_one_of}",
+        )
+
+    # no_file_write_before_audit
+    audit_first = crit.get("no_file_write_before_audit")
+    if audit_first:
+        ok = _no_write_before_audit(transcript)
+        add(
+            "no_file_write_before_audit",
+            ok,
+            "audit reference precedes any write tool call",
+        )
+
+    # no_existing_test_removed
+    keep_tests = crit.get("no_existing_test_removed")
+    if isinstance(keep_tests, list) and keep_tests and target_modified_path:
+        body = _read(clone_root / target_modified_path)
+        missing = [name for name in keep_tests if name not in body]
+        add(
+            "no_existing_test_removed",
+            not missing,
+            f"missing: {missing}" if missing else "all present",
+        )
+
+    # min_test_count
+    min_tests = crit.get("min_test_count")
+    if isinstance(min_tests, int) and min_tests > 0 and (new_test or test_target):
+        path = new_test or test_target
+        body = _read(clone_root / path) if path else ""
+        count = _count_regex(body, r"\btest\s*\(|\bit\s*\(|\bdescribe\s*\(")
+        add(
+            "min_test_count",
+            count >= min_tests,
+            f"found={count}, required={min_tests}",
+        )
+
+    passed = bool(checks) and all(c["ok"] for c in checks)
+    return {"passed": passed, "checks": checks}
+
+
+def _no_write_before_audit(transcript: str) -> bool:
+    """Best-effort: scan the transcript for any string suggesting an audit
+    reference; require it to appear before any write/edit tool call.
+
+    Without a structured tool-call log this is a heuristic; the task runner
+    emits a structured `events` list (Phase 4 Step 2) that the scorer can
+    later consume directly when we want a stricter check.
+    """
+    if not transcript:
+        # Empty transcript = nothing fired = treat as not-failed-yet (will fail other checks)
+        return False
+    lt = transcript.lower()
+    audit_markers = ["existing-ui-audit", "ui_audit", "audit"]
+    write_markers = ["str-replace-editor", "save-file", "edit(", "write("]
+    audit_idx = min((lt.find(m) for m in audit_markers if m in lt), default=-1)
+    write_idx = min((lt.find(m) for m in write_markers if m in lt), default=-1)
+    if audit_idx == -1:
+        return False
+    if write_idx == -1:
+        return True
+    return audit_idx < write_idx

package/scripts/_lib/{bench_caveman.py → bench_telegraph.py}

@@ -1,16 +1,16 @@
-# Caveman compression bench — step-16 Phase 1 Step 4.
+# Telegraph condensation bench — step-16 Phase 1 Step 4.
 #
-# Three-arm live bench against internal/bench/corpora/caveman/prompts.yaml:
-#   compressed     — system prompt embeds caveman-speak rule (aggressive).
+# Three-arm live bench against internal/bench/corpora/telegraph/prompts.yaml:
+#   condensed     — system prompt embeds telegraph-speak rule (aggressive).
 #   terse_control  — system prompt = "Answer concisely. …" (carve-out-free baseline).
-#   uncompressed   — generic helpful-assistant system prompt.
+#   uncondensed   — generic helpful-assistant system prompt.
 #
 # Token counts come from Anthropic API `usage` (authoritative). Carve-out
 # share is measured via regex extraction on the reply text; chars/4 yields
 # an estimated carve-out-token figure for the carve-out-tax accounting.
 #
 # Cost-touch: 10 prompts × 3 arms × claude-sonnet-4-5 (~$3/M in, ~$15/M out).
-"""Caveman compression bench runner."""
+"""Telegraph condensation bench runner."""
 from __future__ import annotations
 
 import re
@@ -24,16 +24,16 @@ import yaml
 
 # ── system prompts per arm ──────────────────────────────────────────────
 
-SYSTEM_PROMPT_COMPRESSED = """You are speaking in CAVEMAN-SPEAK mode (speak_scope=aggressive).
+SYSTEM_PROMPT_CONDENSED = """You are speaking in TELEGRAPH-SPEAK mode (speak_scope=aggressive).
 
-Compress all body prose to caveman grammar:
+Condense all body prose to telegraph grammar:
 - Drop articles (the, a, an).
 - Drop linking auxiliaries (is, are, was, be) where unambiguous.
 - Drop pronouns when context is clear.
 - Keep nouns, verbs, key adjectives, negation, numbers.
 - Example: "I will now check the file and see if it exists" -> "Check file. Exists?"
 
-Carve-outs — preserve BYTE-FOR-BYTE (do NOT compress these):
+Carve-outs — preserve BYTE-FOR-BYTE (do NOT condense these):
 1. Triple-backtick code/literal blocks (any language, including ALL-CAPS Iron-Law fences).
 2. Numbered-options lines matching ^\\d+\\.\\s + a **Recommendation:** label.
 3. Backtick spans (file paths, command names, identifiers).
@@ -43,22 +43,22 @@ Carve-outs — preserve BYTE-FOR-BYTE (do NOT compress these):
 7. Deliverables (PR titles, commit messages, ticket summaries, articles, the prompt
    line of any single question asked to the user).
 
-Apply caveman compression aggressively to every other prose surface."""
+Apply telegraph condensation aggressively to every other prose surface."""
 
 SYSTEM_PROMPT_TERSE = (
     "Answer concisely. Skip preamble. Do not restate the question. "
     "Avoid filler phrases ('Let me', 'Here is', 'I will'). Get to the answer."
 )
 
-SYSTEM_PROMPT_UNCOMPRESSED = (
+SYSTEM_PROMPT_UNCONDENSED = (
     "You are a helpful AI assistant. Answer the user's question clearly and completely."
 )
 
-ARMS: tuple[str, ...] = ("compressed", "terse_control", "uncompressed")
+ARMS: tuple[str, ...] = ("condensed", "terse_control", "uncondensed")
 ARM_SYSTEM_PROMPT: dict[str, str] = {
-    "compressed": SYSTEM_PROMPT_COMPRESSED,
+    "condensed": SYSTEM_PROMPT_CONDENSED,
     "terse_control": SYSTEM_PROMPT_TERSE,
-    "uncompressed": SYSTEM_PROMPT_UNCOMPRESSED,
+    "uncondensed": SYSTEM_PROMPT_UNCONDENSED,
 }
 
 # ── carve-out detection ────────────────────────────────────────────────
@@ -113,15 +113,15 @@ class PromptResult:
 
     @property
     def savings_vs_raw(self) -> float | None:
-        c = self.arms.get("compressed")
-        u = self.arms.get("uncompressed")
+        c = self.arms.get("condensed")
+        u = self.arms.get("uncondensed")
         if not c or not u or u.output_tokens == 0:
             return None
         return 1.0 - (c.output_tokens / u.output_tokens)
 
     @property
     def savings_vs_terse(self) -> float | None:
-        c = self.arms.get("compressed")
+        c = self.arms.get("condensed")
         t = self.arms.get("terse_control")
         if not c or not t or t.output_tokens == 0:
             return None
@@ -131,7 +131,7 @@ class PromptResult:
 # ── corpus + runner ────────────────────────────────────────────────────
 
 def load_corpus(corpus_path: Path) -> list[dict[str, Any]]:
-    """Read internal/bench/corpora/caveman/prompts.yaml → list of prompt dicts."""
+    """Read internal/bench/corpora/telegraph/prompts.yaml → list of prompt dicts."""
     data = yaml.safe_load(corpus_path.read_text(encoding="utf-8")) or {}
     prompts = data.get("prompts") or []
     if not prompts:
@@ -191,12 +191,12 @@ def _stats(values: list[float]) -> dict[str, float]:
 
 
 def aggregate_results(results: list[PromptResult]) -> dict[str, Any]:
-    """Compute median/p10/p90 for compression metrics across the corpus."""
+    """Compute median/p10/p90 for condensation metrics across the corpus."""
     vs_raw = [r.savings_vs_raw for r in results if r.savings_vs_raw is not None]
     vs_terse = [r.savings_vs_terse for r in results if r.savings_vs_terse is not None]
     realised_carve_pct = [
-        r.arms["compressed"].realised_carve_out_pct
-        for r in results if "compressed" in r.arms and r.arms["compressed"].output_chars
+        r.arms["condensed"].realised_carve_out_pct
+        for r in results if "condensed" in r.arms and r.arms["condensed"].output_chars
     ]
     expected_carve_pct = [r.expected_carve_out_pct for r in results]
 
@@ -242,7 +242,7 @@ def compute_cost(results: list[PromptResult], pricing: dict[str, float]) -> dict
 
 # ── orchestrator ───────────────────────────────────────────────────────────
 
-def run_caveman_bench(
+def run_telegraph_bench(
     client: Any,
     corpus_path: Path,
     *,

package/scripts/_lib/{bench_caveman_report.py → bench_telegraph_report.py}

@@ -1,18 +1,18 @@
-# Caveman bench report serializer — step-16 Phase 1 Step 5.
+# Telegraph bench report serializer — step-16 Phase 1 Step 5.
 #
-# Emits the caveman-v1 JSON + Markdown shape. Distinct schema_version
-# ("caveman-v1") from the selection-accuracy bench (v1) because the
-# blocks are disjoint: caveman has no `selection`/`quality`, and the
-# selection bench has no three-arm compression metrics.
-"""Caveman bench report serializer."""
+# Emits the telegraph-v1 JSON + Markdown shape. Distinct schema_version
+# ("telegraph-v1") from the selection-accuracy bench (v1) because the
+# blocks are disjoint: telegraph has no `selection`/`quality`, and the
+# selection bench has no three-arm condensation metrics.
+"""Telegraph bench report serializer."""
 from __future__ import annotations
 
 from typing import Any
 
-from _lib.bench_caveman import ARMS, PromptResult, aggregate_results, compute_cost
+from _lib.bench_telegraph import ARMS, PromptResult, aggregate_results, compute_cost
 
 
-def build_caveman_report(
+def build_telegraph_report(
     *,
     results: list[PromptResult],
     corpus_path_rel: str,
@@ -30,10 +30,10 @@ def build_caveman_report(
     cost["pricing_sourced_on"] = pricing_sourced_on
     errors = cost["totals"]["errors"]
     return {
-        "schema_version": "caveman-v1",
+        "schema_version": "telegraph-v1",
         "generated_at": generated_at,
         "corpus": {
-            "id": "caveman",
+            "id": "telegraph",
             "path": corpus_path_rel,
             "prompt_count": len(results),
         },
@@ -42,7 +42,7 @@ def build_caveman_report(
             "transport": transport,
             "model": model,
         },
-        "caveman": {
+        "telegraph": {
             "arms": list(ARMS),
             "aggregate": aggregate,
             "per_prompt": [_prompt_block(r) for r in results],
@@ -61,8 +61,8 @@ def _prompt_block(r: PromptResult) -> dict[str, Any]:
         "category": r.category,
         "expected_carve_out_pct": r.expected_carve_out_pct,
         "realised_carve_out_pct": (
-            r.arms["compressed"].realised_carve_out_pct
-            if "compressed" in r.arms else None
+            r.arms["condensed"].realised_carve_out_pct
+            if "condensed" in r.arms else None
         ),
         "savings_vs_raw": r.savings_vs_raw,
         "savings_vs_terse": r.savings_vs_terse,
@@ -85,12 +85,12 @@ def _fmt_pct(x: float | None) -> str:
     return f"{x:.2%}" if isinstance(x, (int, float)) else "—"
 
 
-def render_caveman_markdown(report: dict[str, Any]) -> str:
-    cv = report["caveman"]
+def render_telegraph_markdown(report: dict[str, Any]) -> str:
+    cv = report["telegraph"]
     agg = cv["aggregate"]
     cost = report["cost"]
     head = [
-        f"# Caveman Bench Report — `caveman` · {report['generated_at']}",
+        f"# Telegraph Bench Report — `telegraph` · {report['generated_at']}",
         "",
         "## Headline",
         "",
@@ -102,7 +102,7 @@ def render_caveman_markdown(report: dict[str, Any]) -> str:
         f"(p10 {_fmt_pct(agg['savings_vs_raw']['p10'])} · p90 {_fmt_pct(agg['savings_vs_raw']['p90'])})",
         f"- median savings vs terse-control: **{_fmt_pct(agg['savings_vs_terse']['median'])}** "
         f"(p10 {_fmt_pct(agg['savings_vs_terse']['p10'])} · p90 {_fmt_pct(agg['savings_vs_terse']['p90'])})",
-        f"- median realised carve-out share (compressed arm): **{_fmt_pct(agg['realised_carve_out_pct']['median'])}** "
+        f"- median realised carve-out share (condensed arm): **{_fmt_pct(agg['realised_carve_out_pct']['median'])}** "
         f"(expected median {_fmt_pct(agg['expected_carve_out_pct']['median'])})",
         f"- total cost: **${cost['totals']['total_cost_usd']:.6f}** "
         f"(calls {cost['totals']['calls']} · errors {cost['totals']['errors']})",
@@ -125,14 +125,14 @@ def render_caveman_markdown(report: dict[str, Any]) -> str:
     per_prompt = [
         "## Per-prompt results",
         "",
-        "| id | category | exp.carve | real.carve | out.compressed | out.terse | out.uncompressed | vs raw | vs terse |",
+        "| id | category | exp.carve | real.carve | out.condensed | out.terse | out.uncondensed | vs raw | vs terse |",
         "|---|---|---:|---:|---:|---:|---:|---:|---:|",
     ]
     for r in cv["per_prompt"]:
         arms = r["arms"]
-        oc = arms.get("compressed", {}).get("output_tokens", "—")
+        oc = arms.get("condensed", {}).get("output_tokens", "—")
         ot = arms.get("terse_control", {}).get("output_tokens", "—")
-        ou = arms.get("uncompressed", {}).get("output_tokens", "—")
+        ou = arms.get("uncondensed", {}).get("output_tokens", "—")
         per_prompt.append(
             f"| `{r['id']}` | {r['category']} | "
             f"{_fmt_pct(r['expected_carve_out_pct'])} | {_fmt_pct(r['realised_carve_out_pct'])} | "
@@ -145,7 +145,7 @@ def render_caveman_markdown(report: dict[str, Any]) -> str:
         "",
         f"- corpus: `{report['corpus']['path']}`",
         f"- pricing: `internal/bench/pricing.yaml` (sourced {cost.get('pricing_sourced_on') or '—'})",
-        f"- schema: `caveman-v1` (see `docs/contracts/benchmark-report-schema.md`)",
+        f"- schema: `telegraph-v1` (see `docs/contracts/benchmark-report-schema.md`)",
         f"- bench_run version: `{report['runner']['bench_run_version']}`",
         "",
     ]

package/scripts/_lib/claude_desktop_bundler.py

@@ -17,7 +17,7 @@ Contract:
 - Command bundles wrap a single ``.agent-src/commands/<path>.md`` file
   as ``SKILL.md`` inside the ZIP. Nested commands flatten to
   ``<cluster>-<leaf>`` slugs (e.g. ``council/default.md`` →
-  ``council-default.zip``) to mirror ``compress.py``.
+  ``council-default.zip``) to mirror ``condense.py``.
 - Exclusions: ``.git*``, ``__pycache__``, ``*.pyc`` — matched on the
   basename of any path component.
 - A skill folder without a ``SKILL.md`` is skipped (defensive: avoids
@@ -26,7 +26,7 @@ Contract:
 - Command files named ``AGENTS.md`` are skipped (cluster authoring docs,
   not invocable commands).
 - A command slug that collides with an existing skill name is skipped —
-  the real skill bundle wins, matching ``compress.generate_claude_commands``.
+  the real skill bundle wins, matching ``condense.generate_claude_commands``.
 - Writes are atomic via tempfile → ``os.replace``.
 - Idempotent: each ZIP gets a sibling ``<slug>.sha256`` recording
   the manifest digest. If the recomputed digest matches the recorded
@@ -165,7 +165,7 @@ def build_skill_bundles(
 def _command_slug(source_file: Path, commands_root: Path) -> str:
     """Return the flat slug for a command source file.
 
-    Mirrors ``scripts/compress.py::_command_slug``: top-level commands
+    Mirrors ``scripts/condense.py::_command_slug``: top-level commands
     keep their stem (``commit.md`` → ``commit``); nested commands flatten
     the relative path with ``-`` (``council/default.md`` →
     ``council-default``).
@@ -178,7 +178,7 @@ def _iter_command_files(commands_root: Path) -> Iterable[Path]:
     """Yield every command ``.md`` file under ``commands_root`` (recursive).
 
     Skips ``AGENTS.md`` cluster authoring docs, matching
-    ``scripts/compress.py::_iter_commands``.
+    ``scripts/condense.py::_iter_commands``.
     """
     for source_file in sorted(commands_root.rglob("*.md")):
         if source_file.name == "AGENTS.md":
@@ -196,7 +196,7 @@ def build_command_bundles(
 
     Each ZIP contains a single ``SKILL.md`` whose bytes are the source
     command ``.md`` file — same wrapping pattern that
-    ``compress.generate_claude_commands`` uses for Claude Code via
+    ``condense.generate_claude_commands`` uses for Claude Code via
     ``.claude/skills/<slug>/SKILL.md`` symlinks.
 
     Slugs that collide with an existing skill folder under