@event4u/agent-config 3.3.0 → 4.1.0

package/scripts/bench_ab_clone.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Materialise the `with` and `without` clones for the package-impact A/B bench.
+
+Phase 1 Step 2 of `agents/roadmaps/road-to-package-impact-benchmark.md`.
+
+The fixture lives at `internal/bench/ab/fixture/`. Both clones are byte-identical
+copies of the fixture; the `with` clone additionally receives the agent-config
+surface (`.claude/`, `.augment/`, `AGENTS.md`, `CLAUDE.md`) so a Claude Code
+session run inside it sees the same files a consumer project would after
+running the installer.
+
+Idempotent: re-running without `--refresh` leaves an existing clone alone. With
+`--refresh`, the target clone is removed and rebuilt from scratch.
+
+The clones tree (`internal/bench/ab/clones/`) is gitignored — only this script's
+output schema is committed.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+AB_ROOT = REPO_ROOT / "internal" / "bench" / "ab"
+FIXTURE = AB_ROOT / "fixture"
+CLONES = AB_ROOT / "clones"
+
+# Surfaces the `with` clone inherits from the package root.
+WITH_SURFACES = (
+    ".claude",
+    ".augment",
+    "AGENTS.md",
+    "CLAUDE.md",
+)
+
+
+def die(msg: str) -> None:
+    sys.stderr.write(f"bench_ab_clone: {msg}\n")
+    raise SystemExit(1)
+
+
+def copytree_preserve(src: Path, dst: Path) -> None:
+    """Copy tree, dereferencing symlinks.
+
+    The package installs the `.claude/` surface as a tree of symlinks into
+    `.agent-src/`. Cloning the surface as symlinks would carry pointers that
+    resolve against the package root, not the clone — meaning a Claude Code
+    session inside the clone could not actually read the rule bodies. Deref
+    at copy time produces standalone files inside the clone.
+    """
+    if dst.exists():
+        shutil.rmtree(dst)
+    shutil.copytree(src, dst, symlinks=False)
+
+
+def materialise_clone(variant: str, target: Path) -> None:
+    """Copy the fixture into the target, then layer the variant-specific surface."""
+    target.mkdir(parents=True, exist_ok=True)
+    # Mirror the fixture
+    for entry in FIXTURE.iterdir():
+        dest = target / entry.name
+        if entry.is_dir():
+            copytree_preserve(entry, dest)
+        else:
+            shutil.copy2(entry, dest)
+    # Layer the agent-config surface onto the `with` variant
+    if variant == "with":
+        for surface in WITH_SURFACES:
+            src = REPO_ROOT / surface
+            if not src.exists():
+                # Best-effort: a missing surface is reported but does not fail
+                sys.stderr.write(
+                    f"bench_ab_clone: surface '{surface}' missing in package root; "
+                    "with-clone may not be representative\n"
+                )
+                continue
+            dest = target / surface
+            if src.is_dir():
+                copytree_preserve(src, dest)
+            else:
+                shutil.copy2(src, dest)
+
+
+def target_shape_hash() -> str:
+    """Stable hash of the fixture tree + the with-surface list.
+
+    Used by Phase 2's cache key. Recomputing this here keeps the cache code
+    and the clone code reading the same surface definition.
+    """
+    h = hashlib.sha256()
+    h.update(b"with-surfaces:" + json.dumps(WITH_SURFACES).encode() + b"\n")
+    for path in sorted(FIXTURE.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(FIXTURE).as_posix()
+        h.update(f"{rel}\n".encode())
+        h.update(path.read_bytes())
+        h.update(b"\n")
+    return h.hexdigest()[:16]
+
+
+def write_manifest(variant: str, target: Path) -> None:
+    """Drop a small manifest so other scripts can verify the clone shape."""
+    manifest = {
+        "variant": variant,
+        "target_shape_hash": target_shape_hash(),
+        "with_surfaces": list(WITH_SURFACES),
+        "fixture_relpath": FIXTURE.relative_to(REPO_ROOT).as_posix(),
+    }
+    (target / ".bench-ab-manifest.json").write_text(
+        json.dumps(manifest, indent=2) + "\n"
+    )
+
+
+def clone(variant: str, *, refresh: bool) -> Path:
+    target = CLONES / variant
+    if target.exists() and not refresh:
+        sys.stdout.write(f"bench_ab_clone: {variant} clone already present at {target} (use --refresh to rebuild)\n")
+        return target
+    if target.exists():
+        shutil.rmtree(target)
+    materialise_clone(variant, target)
+    write_manifest(variant, target)
+    sys.stdout.write(f"bench_ab_clone: built {variant} clone at {target}\n")
+    return target
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Materialise `with` and `without` clones for the A/B bench."
+    )
+    parser.add_argument(
+        "--refresh",
+        action="store_true",
+        help="Force-rebuild even if the clone already exists.",
+    )
+    parser.add_argument(
+        "--variant",
+        choices=("with", "without", "both"),
+        default="both",
+        help="Which clone to materialise (default: both).",
+    )
+    parser.add_argument(
+        "--print-shape-hash",
+        action="store_true",
+        help="Print the target-shape hash and exit without cloning.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    if not FIXTURE.exists():
+        die(f"fixture missing at {FIXTURE}")
+    if args.print_shape_hash:
+        sys.stdout.write(target_shape_hash() + "\n")
+        return 0
+    variants = ("with", "without") if args.variant == "both" else (args.variant,)
+    for v in variants:
+        clone(v, refresh=args.refresh)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/bench_ab_diff.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Diff two A/B reports (one per variant) into a comparison artefact.
+
+Phase 2 Step 4 of `agents/roadmaps/road-to-package-impact-benchmark.md`.
+
+Inputs: two report JSON paths. Output: a JSON artefact under
+`internal/bench/reports/ab/diff/{stamp}-{corpus}-diff.json` plus a matching
+`.md`. Phase 5's renderer consumes this artefact to populate `docs/benchmark.md`.
+
+The diff content depends on the corpus:
+
+- `ab-tracka` — trigger-accuracy %, false-positive count, per-rule lift.
+- `ab-trackb` — completion-rate per category, wall-time, tokens, cost,
+  ask-vs-act ratio, tool-call count.
+
+Phase 2 only writes the structural skeleton (delta object with `with`,
+`without`, `delta` keys); Phases 3 and 4 plug their real metrics into
+the `results` blocks the runners emit, and the diff is computed in
+`compute_track_a_diff` / `compute_track_b_diff` here.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+REPORTS_DIR = REPO_ROOT / "internal" / "bench" / "reports" / "ab"
+DIFF_DIR = REPORTS_DIR / "diff"
+
+
+def utc_stamp() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
+
+
+def load_report(path: Path) -> dict:
+    return json.loads(path.read_text())
+
+
+def compute_track_a_diff(with_results: dict, without_results: dict) -> dict:
+    """Track A: trigger accuracy + per-rule lift.
+
+    Phase 3 populates `triggers`, `per_rule_accuracy`, `false_positives` in the
+    `results` block. This helper computes the delta. While Phase 3 is not yet
+    landed, we surface what we have and zero what we don't — never invent
+    numbers.
+    """
+    def take(d: dict, key: str, default: float = 0.0) -> float:
+        value = d.get(key, default)
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    with_acc = take(with_results, "trigger_accuracy")
+    without_acc = take(without_results, "trigger_accuracy")
+    return {
+        "trigger_accuracy": {
+            "with": with_acc,
+            "without": without_acc,
+            "delta_pct_points": round(with_acc - without_acc, 3),
+        },
+        "false_positives": {
+            "with": with_results.get("false_positives", 0),
+            "without": without_results.get("false_positives", 0),
+        },
+        "per_rule": {
+            "with": with_results.get("per_rule_accuracy", {}),
+            "without": without_results.get("per_rule_accuracy", {}),
+        },
+    }
+
+
+def compute_track_b_diff(with_results: dict, without_results: dict) -> dict:
+    """Track B: completion rate per category + wall-time + tokens + cost + ask-vs-act."""
+    def mean(d: dict, key: str) -> float:
+        try:
+            return float(d.get(key, 0.0))
+        except (TypeError, ValueError):
+            return 0.0
+
+    with_cats = with_results.get("per_category", {})
+    without_cats = without_results.get("per_category", {})
+    categories = sorted(set(with_cats) | set(without_cats))
+    per_category = {}
+    for cat in categories:
+        per_category[cat] = {
+            "with": with_cats.get(cat, {}),
+            "without": without_cats.get(cat, {}),
+        }
+
+    return {
+        "per_category": per_category,
+        "wall_time_seconds": {
+            "with": mean(with_results, "mean_wall_time"),
+            "without": mean(without_results, "mean_wall_time"),
+            "delta": round(
+                mean(with_results, "mean_wall_time")
+                - mean(without_results, "mean_wall_time"),
+                3,
+            ),
+        },
+        "tokens": {
+            "with": mean(with_results, "mean_tokens"),
+            "without": mean(without_results, "mean_tokens"),
+            "delta": round(
+                mean(with_results, "mean_tokens")
+                - mean(without_results, "mean_tokens"),
+                3,
+            ),
+        },
+        "cost_usd": {
+            "with": mean(with_results, "mean_cost_usd"),
+            "without": mean(without_results, "mean_cost_usd"),
+            "delta": round(
+                mean(with_results, "mean_cost_usd")
+                - mean(without_results, "mean_cost_usd"),
+                4,
+            ),
+        },
+        "ask_vs_act_ratio": {
+            "with": mean(with_results, "ask_vs_act_ratio"),
+            "without": mean(without_results, "ask_vs_act_ratio"),
+        },
+        "tool_calls_per_task": {
+            "with": mean(with_results, "mean_tool_calls"),
+            "without": mean(without_results, "mean_tool_calls"),
+        },
+    }
+
+
+def render_markdown(diff: dict) -> str:
+    lines = [
+        f"# A/B Bench Diff — {diff['corpus']}",
+        "",
+        f"- Stamp: `{diff['stamp']}`",
+        f"- With:    `{diff['with_report']}`",
+        f"- Without: `{diff['without_report']}`",
+        "",
+        "## Delta",
+        "",
+        "```json",
+        json.dumps(diff.get("delta", {}), indent=2),
+        "```",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Diff two A/B bench reports (one per variant)."
+    )
+    parser.add_argument("with_report", type=Path, help="Report JSON for variant=with")
+    parser.add_argument("without_report", type=Path, help="Report JSON for variant=without")
+    parser.add_argument(
+        "--out-dir",
+        type=Path,
+        default=DIFF_DIR,
+        help="Where to write the diff artefact (default: internal/bench/reports/ab/diff/)",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    if not args.with_report.exists():
+        sys.stderr.write(f"bench_ab_diff: missing {args.with_report}\n")
+        return 1
+    if not args.without_report.exists():
+        sys.stderr.write(f"bench_ab_diff: missing {args.without_report}\n")
+        return 1
+    with_rep = load_report(args.with_report)
+    without_rep = load_report(args.without_report)
+    if with_rep.get("variant") != "with":
+        sys.stderr.write(
+            f"bench_ab_diff: {args.with_report} variant is "
+            f"{with_rep.get('variant')!r}, expected 'with'\n"
+        )
+        return 1
+    if without_rep.get("variant") != "without":
+        sys.stderr.write(
+            f"bench_ab_diff: {args.without_report} variant is "
+            f"{without_rep.get('variant')!r}, expected 'without'\n"
+        )
+        return 1
+    if with_rep.get("corpus") != without_rep.get("corpus"):
+        sys.stderr.write(
+            f"bench_ab_diff: corpus mismatch — with={with_rep.get('corpus')} "
+            f"without={without_rep.get('corpus')}\n"
+        )
+        return 1
+    corpus = with_rep.get("corpus") or "unknown"
+    with_results = with_rep.get("results", {})
+    without_results = without_rep.get("results", {})
+    if corpus == "ab-tracka":
+        delta = compute_track_a_diff(with_results, without_results)
+    elif corpus == "ab-trackb":
+        delta = compute_track_b_diff(with_results, without_results)
+    else:
+        delta = {
+            "note": f"no diff strategy registered for corpus {corpus!r}",
+            "with_results": with_results,
+            "without_results": without_results,
+        }
+    stamp = utc_stamp()
+    diff = {
+        "schema": "ab-bench-diff/0.1",
+        "stamp": stamp,
+        "corpus": corpus,
+        "with_report": str(args.with_report.resolve().relative_to(REPO_ROOT)),
+        "without_report": str(args.without_report.resolve().relative_to(REPO_ROOT)),
+        "delta": delta,
+    }
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    json_path = args.out_dir / f"{stamp}-{corpus}-diff.json"
+    md_path = json_path.with_suffix(".md")
+    json_path.write_text(json.dumps(diff, indent=2) + "\n")
+    md_path.write_text(render_markdown(diff))
+    sys.stdout.write(f"bench_ab_diff: wrote {json_path.relative_to(REPO_ROOT)}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/bench_ab_integrity.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Assert the two A/B clones differ only in the agent-config surface.
+
+Phase 1 Step 3 of `agents/roadmaps/road-to-package-impact-benchmark.md`.
+
+The bench's whole validity hinges on the two clones being identical except for
+whether the agent-config surface is present. This script enumerates the file
+trees of both clones and compares byte-by-byte, allowing differences only at
+the documented surface paths (`.claude/`, `.augment/`, `AGENTS.md`,
+`CLAUDE.md`) and the variant manifest.
+
+Exit code:
+    0  — clones are identical except at the allowed surface
+    1  — clone is missing, or a task-target file diverges between variants
+    2  — usage error
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+AB_ROOT = REPO_ROOT / "internal" / "bench" / "ab"
+CLONES = AB_ROOT / "clones"
+
+# Surfaces where divergence is expected (variant-bearing).
+ALLOWED_DELTA_PATHS = (
+    ".claude",
+    ".augment",
+    "AGENTS.md",
+    "CLAUDE.md",
+)
+# Variant-distinguishing manifest written by bench_ab_clone.
+ALLOWED_DELTA_FILES = (
+    ".bench-ab-manifest.json",
+)
+
+
+def is_under_allowed_path(rel: Path) -> bool:
+    parts = rel.parts
+    if not parts:
+        return False
+    head = parts[0]
+    if head in ALLOWED_DELTA_PATHS:
+        return True
+    return rel.as_posix() in ALLOWED_DELTA_FILES
+
+
+def file_hash(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def index_clone(root: Path) -> dict[str, str]:
+    """Return {relpath: sha256} for every regular file under `root`."""
+    out: dict[str, str] = {}
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root)
+        out[rel.as_posix()] = file_hash(path)
+    return out
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Verify the A/B clones differ only in the agent-config surface."
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Print every checked file (default: only divergences)",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+
+    with_root = CLONES / "with"
+    without_root = CLONES / "without"
+    for label, root in (("with", with_root), ("without", without_root)):
+        if not root.exists():
+            sys.stderr.write(
+                f"bench_ab_integrity: {label} clone missing at {root} — run scripts/bench_ab_clone.py first\n"
+            )
+            return 1
+
+    with_index = index_clone(with_root)
+    without_index = index_clone(without_root)
+
+    # Files only in `with` — must all sit under the allowed surface
+    only_in_with = sorted(set(with_index) - set(without_index))
+    bad_only_with = [
+        rel for rel in only_in_with if not is_under_allowed_path(Path(rel))
+    ]
+    # Files only in `without` — there should be none
+    only_in_without = sorted(set(without_index) - set(with_index))
+    bad_only_without = [
+        rel for rel in only_in_without if not is_under_allowed_path(Path(rel))
+    ]
+    # Files present in both — must match byte-for-byte unless under the surface
+    shared = sorted(set(with_index) & set(without_index))
+    bad_diff = [
+        rel
+        for rel in shared
+        if with_index[rel] != without_index[rel] and not is_under_allowed_path(Path(rel))
+    ]
+
+    if args.verbose:
+        sys.stdout.write(
+            f"bench_ab_integrity: with={len(with_index)} files, without={len(without_index)} files, shared={len(shared)}\n"
+        )
+
+    if not bad_only_with and not bad_only_without and not bad_diff:
+        sys.stdout.write(
+            "bench_ab_integrity: clones differ only at the allowed surface (.claude, .augment, AGENTS.md, CLAUDE.md, manifest).\n"
+        )
+        return 0
+
+    sys.stderr.write("bench_ab_integrity: INTEGRITY FAILURE\n")
+    if bad_only_with:
+        sys.stderr.write("  files only in `with` (NOT in allowed surface):\n")
+        for rel in bad_only_with:
+            sys.stderr.write(f"    + {rel}\n")
+    if bad_only_without:
+        sys.stderr.write("  files only in `without` (NOT in allowed surface):\n")
+        for rel in bad_only_without:
+            sys.stderr.write(f"    - {rel}\n")
+    if bad_diff:
+        sys.stderr.write("  files present in both but byte-divergent:\n")
+        for rel in bad_diff:
+            sys.stderr.write(f"    ~ {rel}\n")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())