@event4u/agent-config 2.20.1 → 2.21.0

package/docs/contracts/cost-summary-schema.md

@@ -0,0 +1,107 @@
+---
+stability: beta
+keep-beta-until: 2026-08-15
+---
+
+# cost-summary schema (`cost-summary/v1`)
+
+Stable JSON contract for inter-tool consumption of cost-tracking data
+emitted by [`scripts/cost_summary.py`](../../scripts/cost_summary.py).
+Schema-versioned so downstream consumers can pin and migrate explicitly.
+
+Design reference: Ruflo `scripts/summary.mjs` (upstream cite). Our shape
+diverges to align with the local `agents/cost-tracking/sessions.jsonl`
+fields and the caveman-suspended-multiplier contract.
+
+## Envelope
+
+```json
+{
+  "schema_version": "cost-summary/v1",
+  "generated_at": "2026-05-16T23:45:00Z",
+  "totals": { ... },
+  "by_session": [ ... ],
+  "by_conversation": [ ... ],
+  "by_model": [ ... ]
+}
+```
+
+| Field | Type | Notes |
+|---|---|---|
+| `schema_version` | string | Pinned to `cost-summary/v1`. Downstream consumers MUST refuse unknown versions. |
+| `generated_at` | string (ISO-8601 UTC, `Z` suffix) | Emit time. |
+| `totals` | object | Lifetime aggregates — see `totals` below. |
+| `by_session` | array | Per `sessionId` row; ordered by `sessionId` ascending. |
+| `by_conversation` | array | Per `conversation_id` row; ordered by `conversation_id` ascending. |
+| `by_model` | array | Per `model` row; ordered by `model` ascending. |
+
+## `totals` shape
+
+```json
+{
+  "sessions": 123,
+  "total_cost_usd": 1.2345,
+  "input_tokens": 100000,
+  "output_tokens": 50000,
+  "caveman_delta_tokens": 0,
+  "caveman_multiplier_version": "v1",
+  "caveman_multiplier_active": false
+}
+```
+
+`caveman_delta_tokens` is always `0` while
+`caveman_multiplier_active == false` — see
+[`caveman-telemetry.md`](caveman-telemetry.md) for the suspension contract.
+
+## `by_session` / `by_conversation` row shape
+
+```json
+{
+  "key": "<sessionId or conversation_id>",
+  "sessions": 12,
+  "total_cost_usd": 0.4567,
+  "input_tokens": 8000,
+  "output_tokens": 4500,
+  "caveman_delta_tokens": 0
+}
+```
+
+The `key` field is the grouping identifier; consumers identify the
+group by inspecting which array the row lives in.
+
+## `by_model` row shape
+
+```json
+{
+  "model": "claude-3-5-sonnet-20241022",
+  "sessions": 12,
+  "total_cost_usd": 0.4567,
+  "input_tokens": 8000,
+  "output_tokens": 4500
+}
+```
+
+`by_model` omits caveman fields — the multiplier is dialect-scoped, not
+model-scoped.
+
+## Stability guarantees
+
+- **Field additions** are **non-breaking**: consumers MUST ignore unknown fields.
+- **Field removals or renames** bump the `schema_version` minor (`v1` → `v2`).
+- **Type changes** bump the major (`v1.*` → `v2.0`).
+- Downstream consumers SHOULD pin to a specific `schema_version` and
+  refuse unknown ones; the pin is the migration boundary.
+
+## Downstream consumers
+
+- `agent-status` skill — surfaces lifetime / current-conversation slice.
+- Future `cost-export-to-monitoring` scripts (deferred; trigger:
+  consumer request) would wrap this JSON to push to Prometheus / OTLP.
+
+## See also
+
+- [`caveman-telemetry.md`](caveman-telemetry.md) — defines the
+  `caveman_*` fields and the suspended-multiplier contract.
+- [`scripts/cost_summary.py`](../../scripts/cost_summary.py) — implementation.
+- [`scripts/cost_by_conversation.py`](../../scripts/cost_by_conversation.py) — narrower per-conversation lens with the same JSONL source.
+- [`scripts/caveman_stats.py`](../../scripts/caveman_stats.py) — caveman-only delta lens with the same JSONL source.

package/docs/contracts/file-ownership-matrix.json

@@ -1800,6 +1800,12 @@
       "load_context": [],
       "load_context_eager": []
     },
+    ".agent-src.uncompressed/skills/compress-memory/SKILL.md": {
+      "kind": "skill",
+      "rule_type": null,
+      "load_context": [],
+      "load_context_eager": []
+    },
     ".agent-src.uncompressed/skills/content-funnel-design/SKILL.md": {
       "kind": "skill",
       "rule_type": null,
@@ -6396,6 +6402,13 @@
       "via": "self",
       "depth": 0
     },
+    {
+      "source": ".agent-src.uncompressed/rules/caveman-speak.md",
+      "target": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "type": "READ_ONLY",
+      "via": "body_link",
+      "depth": 1
+    },
     {
       "source": ".agent-src.uncompressed/rules/cli-output-handling.md",
       "target": ".agent-src.uncompressed/rules/cli-output-handling.md",
@@ -8048,6 +8061,34 @@
       "via": "self",
       "depth": 0
     },
+    {
+      "source": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "target": ".agent-src.uncompressed/rules/caveman-speak.md",
+      "type": "READ_ONLY",
+      "via": "body_link",
+      "depth": 1
+    },
+    {
+      "source": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "target": ".agent-src.uncompressed/rules/role-mode-adherence.md",
+      "type": "READ_ONLY",
+      "via": "body_link",
+      "depth": 1
+    },
+    {
+      "source": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "target": ".agent-src.uncompressed/skills/agents-md-thin-root/SKILL.md",
+      "type": "READ_ONLY",
+      "via": "body_link",
+      "depth": 1
+    },
+    {
+      "source": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "target": ".agent-src.uncompressed/skills/compress-memory/SKILL.md",
+      "type": "WRITE",
+      "via": "self",
+      "depth": 0
+    },
     {
       "source": ".agent-src.uncompressed/skills/content-funnel-design/SKILL.md",
       "target": ".agent-src.uncompressed/skills/activation-design/SKILL.md",

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@event4u/agent-config",
-    "version": "2.20.1",
+    "version": "2.21.0",
     "description": "Shared agent configuration \u2014 skills, rules, commands, guidelines, and templates for AI coding tools",
     "license": "MIT",
     "private": false,

package/scripts/_lib/bench_caveman.py

@@ -0,0 +1,273 @@
+# Caveman compression bench — step-16 Phase 1 Step 4.
+#
+# Three-arm live bench against bench/corpora/caveman/prompts.yaml:
+#   compressed     — system prompt embeds caveman-speak rule (aggressive).
+#   terse_control  — system prompt = "Answer concisely. …" (carve-out-free baseline).
+#   uncompressed   — generic helpful-assistant system prompt.
+#
+# Token counts come from Anthropic API `usage` (authoritative). Carve-out
+# share is measured via regex extraction on the reply text; chars/4 yields
+# an estimated carve-out-token figure for the carve-out-tax accounting.
+#
+# Cost-touch: 10 prompts × 3 arms × claude-sonnet-4-5 (~$3/M in, ~$15/M out).
+"""Caveman compression bench runner."""
+from __future__ import annotations
+
+import re
+import statistics
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# ── system prompts per arm ──────────────────────────────────────────────
+
+SYSTEM_PROMPT_COMPRESSED = """You are speaking in CAVEMAN-SPEAK mode (speak_scope=aggressive).
+
+Compress all body prose to caveman grammar:
+- Drop articles (the, a, an).
+- Drop linking auxiliaries (is, are, was, be) where unambiguous.
+- Drop pronouns when context is clear.
+- Keep nouns, verbs, key adjectives, negation, numbers.
+- Example: "I will now check the file and see if it exists" -> "Check file. Exists?"
+
+Carve-outs — preserve BYTE-FOR-BYTE (do NOT compress these):
+1. Triple-backtick code/literal blocks (any language, including ALL-CAPS Iron-Law fences).
+2. Numbered-options lines matching ^\\d+\\.\\s + a **Recommendation:** label.
+3. Backtick spans (file paths, command names, identifiers).
+4. Status markers: lines starting with ❌, ⚠️, or ✅.
+5. Mode markers.
+6. Markdown tables.
+7. Deliverables (PR titles, commit messages, ticket summaries, articles, the prompt
+   line of any single question asked to the user).
+
+Apply caveman compression aggressively to every other prose surface."""
+
+SYSTEM_PROMPT_TERSE = (
+    "Answer concisely. Skip preamble. Do not restate the question. "
+    "Avoid filler phrases ('Let me', 'Here is', 'I will'). Get to the answer."
+)
+
+SYSTEM_PROMPT_UNCOMPRESSED = (
+    "You are a helpful AI assistant. Answer the user's question clearly and completely."
+)
+
+ARMS: tuple[str, ...] = ("compressed", "terse_control", "uncompressed")
+ARM_SYSTEM_PROMPT: dict[str, str] = {
+    "compressed": SYSTEM_PROMPT_COMPRESSED,
+    "terse_control": SYSTEM_PROMPT_TERSE,
+    "uncompressed": SYSTEM_PROMPT_UNCOMPRESSED,
+}
+
+# ── carve-out detection ────────────────────────────────────────────────
+
+_RE_TRIPLE_BACKTICK = re.compile(r"```[\s\S]*?```")
+_RE_BACKTICK_SPAN = re.compile(r"`[^`\n]+`")
+_RE_NUMBERED_LINE = re.compile(r"^>?\s*\d+\.\s.*$", re.MULTILINE)
+_RE_STATUS_LINE = re.compile(r"^(❌|⚠️|✅).*$", re.MULTILINE)
+_RE_TABLE_LINE = re.compile(r"^\s*\|.*\|\s*$", re.MULTILINE)
+_RE_RECOMMENDATION = re.compile(r"^\*\*(Recommendation|Empfehlung):\*\*.*$", re.MULTILINE)
+
+
+def carve_out_chars(text: str) -> int:
+    """Sum byte-length of every carve-out region (union, no double-count)."""
+    if not text:
+        return 0
+    mask = bytearray(len(text))
+    for pattern in (
+        _RE_TRIPLE_BACKTICK, _RE_BACKTICK_SPAN, _RE_NUMBERED_LINE,
+        _RE_STATUS_LINE, _RE_TABLE_LINE, _RE_RECOMMENDATION,
+    ):
+        for m in pattern.finditer(text):
+            for i in range(m.start(), m.end()):
+                mask[i] = 1
+    return sum(mask)
+
+
+# ── data shapes ────────────────────────────────────────────────────────
+
+@dataclass
+class ArmResult:
+    arm: str
+    text: str
+    input_tokens: int
+    output_tokens: int
+    latency_ms: int
+    output_chars: int
+    carve_out_chars: int
+    error: str | None = None
+
+    @property
+    def realised_carve_out_pct(self) -> float:
+        return self.carve_out_chars / self.output_chars if self.output_chars else 0.0
+
+
+@dataclass
+class PromptResult:
+    id: str
+    category: str
+    expected_carve_out_pct: float
+    arms: dict[str, ArmResult] = field(default_factory=dict)
+
+    @property
+    def savings_vs_raw(self) -> float | None:
+        c = self.arms.get("compressed")
+        u = self.arms.get("uncompressed")
+        if not c or not u or u.output_tokens == 0:
+            return None
+        return 1.0 - (c.output_tokens / u.output_tokens)
+
+    @property
+    def savings_vs_terse(self) -> float | None:
+        c = self.arms.get("compressed")
+        t = self.arms.get("terse_control")
+        if not c or not t or t.output_tokens == 0:
+            return None
+        return 1.0 - (c.output_tokens / t.output_tokens)
+
+
+# ── corpus + runner ────────────────────────────────────────────────────
+
+def load_corpus(corpus_path: Path) -> list[dict[str, Any]]:
+    """Read bench/corpora/caveman/prompts.yaml → list of prompt dicts."""
+    data = yaml.safe_load(corpus_path.read_text(encoding="utf-8")) or {}
+    prompts = data.get("prompts") or []
+    if not prompts:
+        raise ValueError(f"empty corpus: {corpus_path}")
+    return prompts
+
+
+def run_arm(
+    client: Any,
+    arm: str,
+    user_prompt: str,
+    *,
+    max_tokens: int = 1024,
+) -> ArmResult:
+    """Invoke one arm against the live API. Returns ArmResult including text."""
+    t0 = time.monotonic()
+    system = ARM_SYSTEM_PROMPT[arm]
+    try:
+        resp = client.ask(system, user_prompt, max_tokens=max_tokens)
+    except Exception as exc:  # noqa: BLE001
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        return ArmResult(arm=arm, text="", input_tokens=0, output_tokens=0,
+                         latency_ms=latency_ms, output_chars=0, carve_out_chars=0,
+                         error=str(exc))
+    return ArmResult(
+        arm=arm, text=resp.text or "",
+        input_tokens=int(resp.input_tokens or 0),
+        output_tokens=int(resp.output_tokens or 0),
+        latency_ms=int(resp.latency_ms or (time.monotonic() - t0) * 1000),
+        output_chars=len(resp.text or ""),
+        carve_out_chars=carve_out_chars(resp.text or ""),
+        error=resp.error,
+    )
+
+
+# ── aggregation ────────────────────────────────────────────────────────────
+
+def _stats(values: list[float]) -> dict[str, float]:
+    """Median / p10 / p90 / stdev / n on a list of floats. Empty → zeros."""
+    if not values:
+        return {"n": 0, "median": 0.0, "p10": 0.0, "p90": 0.0, "stdev": 0.0}
+    s = sorted(values)
+    n = len(s)
+    def _pct(p: float) -> float:
+        if n == 1:
+            return s[0]
+        k = (n - 1) * p
+        lo, hi = int(k), min(int(k) + 1, n - 1)
+        return s[lo] + (s[hi] - s[lo]) * (k - lo)
+    return {
+        "n": n,
+        "median": statistics.median(s),
+        "p10": _pct(0.10),
+        "p90": _pct(0.90),
+        "stdev": statistics.pstdev(s) if n > 1 else 0.0,
+    }
+
+
+def aggregate_results(results: list[PromptResult]) -> dict[str, Any]:
+    """Compute median/p10/p90 for compression metrics across the corpus."""
+    vs_raw = [r.savings_vs_raw for r in results if r.savings_vs_raw is not None]
+    vs_terse = [r.savings_vs_terse for r in results if r.savings_vs_terse is not None]
+    realised_carve_pct = [
+        r.arms["compressed"].realised_carve_out_pct
+        for r in results if "compressed" in r.arms and r.arms["compressed"].output_chars
+    ]
+    expected_carve_pct = [r.expected_carve_out_pct for r in results]
+
+    per_arm_tokens: dict[str, list[int]] = {a: [] for a in ARMS}
+    for r in results:
+        for arm in ARMS:
+            ar = r.arms.get(arm)
+            if ar:
+                per_arm_tokens[arm].append(ar.output_tokens)
+
+    return {
+        "savings_vs_raw": _stats(vs_raw),
+        "savings_vs_terse": _stats(vs_terse),
+        "realised_carve_out_pct": _stats(realised_carve_pct),
+        "expected_carve_out_pct": _stats(expected_carve_pct),
+        "output_tokens": {
+            arm: _stats([float(v) for v in per_arm_tokens[arm]]) for arm in ARMS
+        },
+    }
+
+
+def compute_cost(results: list[PromptResult], pricing: dict[str, float]) -> dict[str, Any]:
+    """Sum input/output tokens across all arms; cost from per-1M pricing dict."""
+    totals = {"input_tokens": 0, "output_tokens": 0, "calls": 0, "errors": 0}
+    per_arm: dict[str, dict[str, int]] = {a: {"input_tokens": 0, "output_tokens": 0, "calls": 0} for a in ARMS}
+    for r in results:
+        for arm, ar in r.arms.items():
+            totals["input_tokens"] += ar.input_tokens
+            totals["output_tokens"] += ar.output_tokens
+            totals["calls"] += 1
+            if ar.error:
+                totals["errors"] += 1
+            per_arm[arm]["input_tokens"] += ar.input_tokens
+            per_arm[arm]["output_tokens"] += ar.output_tokens
+            per_arm[arm]["calls"] += 1
+    cost_usd = (
+        totals["input_tokens"] / 1e6 * pricing.get("input", 0.0)
+        + totals["output_tokens"] / 1e6 * pricing.get("output", 0.0)
+    )
+    totals["total_cost_usd"] = round(cost_usd, 6)
+    return {"totals": totals, "per_arm": per_arm}
+
+
+# ── orchestrator ───────────────────────────────────────────────────────────
+
+def run_caveman_bench(
+    client: Any,
+    corpus_path: Path,
+    *,
+    max_prompts: int | None = None,
+    max_tokens: int = 1024,
+    on_progress: Any = None,
+) -> list[PromptResult]:
+    """Run all three arms over the corpus. Returns per-prompt results."""
+    prompts = load_corpus(corpus_path)
+    if max_prompts:
+        prompts = prompts[:max_prompts]
+    results: list[PromptResult] = []
+    total = len(prompts) * len(ARMS)
+    done = 0
+    for p in prompts:
+        pr = PromptResult(
+            id=str(p["id"]),
+            category=str(p.get("category", "unknown")),
+            expected_carve_out_pct=float(p.get("expected_carve_out_pct", 0.0)),
+        )
+        for arm in ARMS:
+            ar = run_arm(client, arm, str(p["prompt"]), max_tokens=max_tokens)
+            pr.arms[arm] = ar
+            done += 1
+            if on_progress:
+                on_progress(done, total, pr.id, arm, ar)
+        results.append(pr)
+    return results

package/scripts/_lib/bench_caveman_report.py

@@ -0,0 +1,152 @@
+# Caveman bench report serializer — step-16 Phase 1 Step 5.
+#
+# Emits the caveman-v1 JSON + Markdown shape. Distinct schema_version
+# ("caveman-v1") from the selection-accuracy bench (v1) because the
+# blocks are disjoint: caveman has no `selection`/`quality`, and the
+# selection bench has no three-arm compression metrics.
+"""Caveman bench report serializer."""
+from __future__ import annotations
+
+from typing import Any
+
+from _lib.bench_caveman import ARMS, PromptResult, aggregate_results, compute_cost
+
+
+def build_caveman_report(
+    *,
+    results: list[PromptResult],
+    corpus_path_rel: str,
+    generated_at: str,
+    bench_run_version: str,
+    model: str,
+    transport: str,
+    pricing_rates: dict[str, float],
+    pricing_sourced_on: str | None,
+) -> dict[str, Any]:
+    aggregate = aggregate_results(results)
+    cost = compute_cost(results, pricing_rates)
+    cost["source"] = "live-api"
+    cost["model"] = model
+    cost["pricing_sourced_on"] = pricing_sourced_on
+    errors = cost["totals"]["errors"]
+    return {
+        "schema_version": "caveman-v1",
+        "generated_at": generated_at,
+        "corpus": {
+            "id": "caveman",
+            "path": corpus_path_rel,
+            "prompt_count": len(results),
+        },
+        "runner": {
+            "bench_run_version": bench_run_version,
+            "transport": transport,
+            "model": model,
+        },
+        "caveman": {
+            "arms": list(ARMS),
+            "aggregate": aggregate,
+            "per_prompt": [_prompt_block(r) for r in results],
+        },
+        "cost": cost,
+        "verdict": {
+            "overall": "measured" if errors == 0 else "partial",
+            "errors": errors,
+        },
+    }
+
+
+def _prompt_block(r: PromptResult) -> dict[str, Any]:
+    return {
+        "id": r.id,
+        "category": r.category,
+        "expected_carve_out_pct": r.expected_carve_out_pct,
+        "realised_carve_out_pct": (
+            r.arms["compressed"].realised_carve_out_pct
+            if "compressed" in r.arms else None
+        ),
+        "savings_vs_raw": r.savings_vs_raw,
+        "savings_vs_terse": r.savings_vs_terse,
+        "arms": {
+            arm: {
+                "input_tokens": ar.input_tokens,
+                "output_tokens": ar.output_tokens,
+                "latency_ms": ar.latency_ms,
+                "output_chars": ar.output_chars,
+                "carve_out_chars": ar.carve_out_chars,
+                "error": ar.error,
+                "text": ar.text,
+            }
+            for arm, ar in r.arms.items()
+        },
+    }
+
+
+def _fmt_pct(x: float | None) -> str:
+    return f"{x:.2%}" if isinstance(x, (int, float)) else "—"
+
+
+def render_caveman_markdown(report: dict[str, Any]) -> str:
+    cv = report["caveman"]
+    agg = cv["aggregate"]
+    cost = report["cost"]
+    head = [
+        f"# Caveman Bench Report — `caveman` · {report['generated_at']}",
+        "",
+        "## Headline",
+        "",
+        f"- prompts: **{report['corpus']['prompt_count']}** · "
+        f"arms: **{', '.join(cv['arms'])}** · "
+        f"model: **{report['runner']['model']}** · "
+        f"transport: **{report['runner']['transport']}**",
+        f"- median savings vs raw: **{_fmt_pct(agg['savings_vs_raw']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_raw']['p10'])} · p90 {_fmt_pct(agg['savings_vs_raw']['p90'])})",
+        f"- median savings vs terse-control: **{_fmt_pct(agg['savings_vs_terse']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_terse']['p10'])} · p90 {_fmt_pct(agg['savings_vs_terse']['p90'])})",
+        f"- median realised carve-out share (compressed arm): **{_fmt_pct(agg['realised_carve_out_pct']['median'])}** "
+        f"(expected median {_fmt_pct(agg['expected_carve_out_pct']['median'])})",
+        f"- total cost: **${cost['totals']['total_cost_usd']:.6f}** "
+        f"(calls {cost['totals']['calls']} · errors {cost['totals']['errors']})",
+        f"- verdict: **{report['verdict']['overall']}**",
+        "",
+    ]
+    per_arm = [
+        "## Per-arm token totals",
+        "",
+        "| arm | calls | input_tokens | output_tokens | median out/prompt |",
+        "|---|---:|---:|---:|---:|",
+    ]
+    for arm in cv["arms"]:
+        a = cost["per_arm"][arm]
+        m = agg["output_tokens"][arm]["median"]
+        per_arm.append(
+            f"| `{arm}` | {a['calls']} | {a['input_tokens']} | {a['output_tokens']} | {m:.0f} |"
+        )
+    per_arm.append("")
+    per_prompt = [
+        "## Per-prompt results",
+        "",
+        "| id | category | exp.carve | real.carve | out.compressed | out.terse | out.uncompressed | vs raw | vs terse |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for r in cv["per_prompt"]:
+        arms = r["arms"]
+        oc = arms.get("compressed", {}).get("output_tokens", "—")
+        ot = arms.get("terse_control", {}).get("output_tokens", "—")
+        ou = arms.get("uncompressed", {}).get("output_tokens", "—")
+        per_prompt.append(
+            f"| `{r['id']}` | {r['category']} | "
+            f"{_fmt_pct(r['expected_carve_out_pct'])} | {_fmt_pct(r['realised_carve_out_pct'])} | "
+            f"{oc} | {ot} | {ou} | "
+            f"{_fmt_pct(r['savings_vs_raw'])} | {_fmt_pct(r['savings_vs_terse'])} |"
+        )
+    per_prompt.append("")
+    notes = [
+        "## Notes",
+        "",
+        f"- corpus: `{report['corpus']['path']}`",
+        f"- pricing: `bench/pricing.yaml` (sourced {cost.get('pricing_sourced_on') or '—'})",
+        f"- schema: `caveman-v1` (see `docs/contracts/benchmark-report-schema.md`)",
+        f"- bench_run version: `{report['runner']['bench_run_version']}`",
+        "",
+    ]
+    return "\n".join(head + per_arm + per_prompt + notes)