@event4u/agent-config 4.9.0 → 5.0.0

package/scripts/lint_value_dashboard.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Lint `docs/value.md` for structural invariants.
+
+Phase 5 Step 3 of `agents/roadmaps/road-to-readable-value-dashboard.md`.
+
+Invariants enforced (any violation → exit 1):
+
+1. Required sections present (intro / Reference scale / Panel A / Panel B
+   / Glossar / NETTO line).
+2. Every cost-ladder rung row cites a `source_report` (or `n/a` for the
+   baseline rung) — no rung sneaks in without traceability.
+3. No `measured` rung renders a `pending` source — internal consistency
+   of confidence ↔ source state.
+4. No negative-saving label: the literal string "Ersparnis" must not
+   appear in a row where the displayed Δ-token value is positive (the
+   load + terse rungs are *costs*, not savings; mislabelling either is
+   a credibility failure the page explicitly forbids).
+5. The `latest.json` exists and its `cost_ladder` rung ids match the
+   five canonical rungs — the renderer cannot silently drop a rung.
+
+The linter loads `internal/bench/reports/value/latest.json` directly
+(not just the rendered `.md`) for items (3) and (5) — the rendered
+text alone is too lossy.
+
+Output: one violation per line in non-quiet mode; one-line summary in
+quiet mode. Exit 0 on clean, 1 on any violation.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any, Dict, List
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DASHBOARD = REPO_ROOT / "docs" / "value.md"
+LATEST = REPO_ROOT / "internal" / "bench" / "reports" / "value" / "latest.json"
+
+REQUIRED_SECTIONS = (
+    "# Value Dashboard",
+    "## Reference scale",
+    "## Panel A",
+    "## Panel B",
+    "## Glossar",
+    "**NETTO",
+)
+
+CANONICAL_RUNG_IDS = ("baseline", "load", "condense", "rtk", "terse")
+
+
+def _log(msg: str, quiet: bool, *, err: bool = False) -> None:
+    if err:
+        print(msg, file=sys.stderr)
+    elif not quiet:
+        print(msg)
+
+
+def check_required_sections(text: str) -> List[str]:
+    return [
+        f"missing required section: '{section}'"
+        for section in REQUIRED_SECTIONS
+        if section not in text
+    ]
+
+
+def check_source_citations(report: Dict[str, Any]) -> List[str]:
+    violations = []
+    for rung in report.get("cost_ladder", []) or []:
+        source = rung.get("source_report")
+        if not source:
+            violations.append(
+                f"rung '{rung.get('id')}' has no source_report field"
+            )
+            continue
+        if not isinstance(source, str) or not source.strip():
+            violations.append(
+                f"rung '{rung.get('id')}' has empty source_report"
+            )
+    return violations
+
+
+def check_confidence_vs_source(report: Dict[str, Any]) -> List[str]:
+    """A `measured` rung's source_report must exist on disk."""
+    violations = []
+    for rung in report.get("cost_ladder", []) or []:
+        if rung.get("confidence") != "measured":
+            continue
+        source = rung.get("source_report") or ""
+        if source in ("", "n/a"):
+            continue  # baseline rung
+        path = REPO_ROOT / source
+        if not path.exists():
+            violations.append(
+                f"rung '{rung.get('id')}' is 'measured' but its "
+                f"source_report does not exist: {source}"
+            )
+    return violations
+
+
+def check_no_negative_savings(text: str) -> List[str]:
+    """A rung whose Δ-token value is positive must not be labelled a saving.
+
+    Heuristic: scan Panel A's rows; flag any row that contains the
+    German word "Ersparnis" with a positive token-delta in the same row.
+    """
+    violations = []
+    # Panel A rows are pipe-delimited; we read every line starting with "|"
+    # inside the cost ladder section.
+    in_panel_a = False
+    for line in text.splitlines():
+        if line.startswith("## Panel A"):
+            in_panel_a = True
+            continue
+        if in_panel_a and line.startswith("## "):
+            break
+        if not in_panel_a or not line.startswith("|"):
+            continue
+        if "Ersparnis" not in line:
+            continue
+        # Look for a "+" sign at the start of an integer-shaped delta.
+        # The format renders deltas as "+4 843" / "-186".
+        m = re.search(r"\|\s*([+-][0-9 ]+)\s*\|", line)
+        if m and m.group(1).strip().startswith("+"):
+            token_value = m.group(1).strip()
+            violations.append(
+                "row labelled 'Ersparnis' has a positive Δ-token value: "
+                f"{token_value!r} — positive deltas are costs, not savings."
+            )
+    return violations
+
+
+def check_canonical_rung_set(report: Dict[str, Any]) -> List[str]:
+    rungs = report.get("cost_ladder", []) or []
+    ids = [r.get("id") for r in rungs]
+    if list(ids) != list(CANONICAL_RUNG_IDS):
+        return [
+            f"cost_ladder rung ids must be {CANONICAL_RUNG_IDS}, "
+            f"got {tuple(ids)}"
+        ]
+    return []
+
+
+def lint(quiet: bool = False) -> int:
+    violations: List[str] = []
+
+    if not DASHBOARD.exists():
+        _log(
+            f"FAIL: dashboard not found: {DASHBOARD.relative_to(REPO_ROOT)}",
+            quiet,
+            err=True,
+        )
+        return 1
+    text = DASHBOARD.read_text()
+    violations.extend(check_required_sections(text))
+    violations.extend(check_no_negative_savings(text))
+
+    if not LATEST.exists():
+        # No JSON to deep-check — that's a placeholder dashboard.
+        # Required-sections check still applies; we degrade gracefully.
+        if violations:
+            for v in violations:
+                _log(f"FAIL: {v}", quiet, err=True)
+            return 1
+        _log(
+            "lint_value_dashboard: dashboard is a placeholder "
+            "(no value-v1.json yet) — structural checks pass.",
+            quiet=False,
+        )
+        return 0
+
+    try:
+        report = json.loads(LATEST.read_text())
+    except json.JSONDecodeError as exc:
+        _log(f"FAIL: {LATEST.name} is not valid JSON: {exc}", quiet, err=True)
+        return 1
+
+    violations.extend(check_source_citations(report))
+    violations.extend(check_confidence_vs_source(report))
+    violations.extend(check_canonical_rung_set(report))
+
+    if violations:
+        for v in violations:
+            _log(f"FAIL: {v}", quiet, err=True)
+        return 1
+    _log(
+        (
+            "lint_value_dashboard: OK — "
+            f"{len(report.get('cost_ladder', []))} rungs, "
+            f"{len(report.get('behaviour', []))} behaviour metrics, all "
+            "sections present, all sources cited."
+        ),
+        quiet=False,
+    )
+    return 0
+
+
+def parse_args(argv: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Lint docs/value.md for structural invariants."
+    )
+    parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Suppress non-error output.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: List[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    return lint(quiet=args.quiet)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/render_benchmark_md.py

@@ -103,10 +103,14 @@ def render_headline(track_a: dict, track_b: dict) -> str:
     lines = [
         "## Headline",
         "",
+        "> **Track A confirms surface availability** — a precondition, not an impact metric. "
+        "For the impact view (cost-ladder + behaviour with vs. without), see "
+        "[`docs/value.md`](value.md).",
+        "",
         "| Metric | with | without | delta |",
         "|---|---|---|---|",
-        f"| Track A trigger-accuracy | {fmt_pct(a_with_acc)} | {fmt_pct(a_wo_acc)} | "
-        f"{fmt_pct((a_with_acc or 0) - (a_wo_acc or 0))} |",
+        f"| Track A surface-availability | {fmt_pct(a_with_acc)} | {fmt_pct(a_wo_acc)} | "
+        f"{fmt_pct((a_with_acc or 0) - (a_wo_acc or 0))} _(structural — files present)_ |",
         f"| Track B completion-rate  | {fmt_pct(b_with_comp)} | {fmt_pct(b_wo_comp)} | "
         f"{fmt_pct((b_with_comp or 0) - (b_wo_comp or 0))} |",
         f"| Track B mean wall-time   | {fmt_num(b_results.get('mean_wall_time'))}s "

package/scripts/render_value_md.py

@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+"""Render `docs/value.md` from the latest `value-v1` JSON report.
+
+Phase 4 Step 1 of `agents/roadmaps/road-to-readable-value-dashboard.md`.
+
+This renderer is **deterministic** — it does not run any bench, only
+formats existing reports. Mirrors `render_benchmark_md.py`'s placeholder
+discipline: when the report is missing, write a placeholder document
+explaining how to produce one. Never errors.
+
+The dashboard has two panels:
+  - Panel A — cost ladder (cumulative, min → max)
+  - Panel B — behaviour (with vs. without)
+
+Each panel uses plain language, prints `confidence` markers inline,
+and ends with a bold NETTO line that lifts the totals out of the
+table.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+VALUE_REPORTS_DIR = REPO_ROOT / "internal" / "bench" / "reports" / "value"
+LATEST = VALUE_REPORTS_DIR / "latest.json"
+OUT_PATH = REPO_ROOT / "docs" / "value.md"
+
+REQUIRED_SECTIONS = (
+    "## Reference scale",
+    "## Panel A",
+    "## Panel B",
+    "## Glossar",
+    "**NETTO",
+)
+
+
+def utc_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def safe_load(path: Path) -> Optional[Dict[str, Any]]:
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except json.JSONDecodeError:
+        return None
+
+
+def fmt_signed_int(value: int) -> str:
+    return f"{value:+,}".replace(",", " ")
+
+
+def fmt_eur(value: float) -> str:
+    return f"{value:+.2f} €"
+
+
+def fmt_pct(value: float) -> str:
+    return f"{value:+.2f}%"
+
+
+def confidence_badge(level: str) -> str:
+    badges = {
+        "measured": "✅ gemessen",
+        "estimated": "≈ geschätzt",
+        "vendor-claim": "⚠️ vendor-claim",
+        "pending": "⏳ pending",
+    }
+    return badges.get(level, level)
+
+
+def mode_badge(mode: str) -> str:
+    if mode == "live":
+        return "✅ live"
+    if mode == "dry-run":
+        return "⚠️ dry-run"
+    return mode
+
+
+def render_intro(report: Dict[str, Any]) -> str:
+    ref = report.get("reference_scale", {})
+    requests = ref.get("requests", 1000)
+    avg_in = ref.get("avg_input_tokens", 8000)
+    avg_out = ref.get("avg_output_tokens", 600)
+    tier = ref.get("model_tier", "sonnet")
+    sourced = ref.get("pricing_sourced_on", "—")
+    return (
+        f"# Value Dashboard — was kostet das Paket, was bringt es?\n"
+        "\n"
+        "> Diese Seite beantwortet **eine** Frage in echten Zahlen: "
+        "*Wie viel mehr Tokens kostet mich das Paket, und wie viel "
+        "spart es danach wieder ein?* Generiert von "
+        "`scripts/render_value_md.py` aus dem letzten `value-v1` Report; "
+        "Quelle: `internal/bench/reports/value/latest.json`.\n"
+        "\n"
+        "## Wie diese Seite zu lesen ist\n"
+        "\n"
+        "**Panel A (Kostenleiter)** — von oben nach unten lesen. Jede "
+        "Stufe sagt: *was sie macht*, *wie viele Input-Tokens sie pro "
+        "Request hinzufügt oder spart*, *was das in € auf "
+        f"{requests:,} Requests kostet*, und *wo wir kumulativ stehen*. "
+        "Die fett gedruckte **NETTO**-Zeile am Ende ist die Antwort.\n"
+        "\n"
+        "**Panel B (Verhalten)** — vier reale Vergleiche, *mit* vs. "
+        "*ohne* Paket. Hier liegt der nicht-Token-Wert: passende Skill-"
+        "Auswahl, Stopps bei riskanten Aktionen, weniger Rückfragen, "
+        "mehr abgeschlossene Aufgaben.\n"
+        "\n"
+        "**Confidence-Marker** an jeder Stufe: `✅ gemessen` = echter "
+        "Wert aus einem Report im Repo · `⏳ pending` = noch nicht "
+        "gemessen, Stufe trägt 0 zur Summe bei · `⚠️ vendor-claim` = "
+        "Behauptung eines Herstellers, nicht selbst gemessen.\n"
+        "\n"
+        "## Reference scale\n"
+        "\n"
+        f"- **{requests:,}** Requests, durchschnittlich "
+        f"**{avg_in:,}** Input-Tokens und **{avg_out:,}** Output-Tokens "
+        "pro Request\n"
+        f"- Modell-Tier: `{tier}` · "
+        f"Preisstand `{sourced}` (Quelle: `internal/bench/pricing.yaml`)\n"
+        "- Wer einen anderen Workload fährt, rechnet selbst nach — die "
+        "Methodik ist offengelegt; nichts ist hardcodiert versteckt.\n"
+    )
+
+
+def render_panel_a(report: Dict[str, Any]) -> str:
+    lines = [
+        "## Panel A — Kostenleiter (kumulativ, min → max)\n",
+        "Liest sich von oben nach unten. Positive Δ-Werte = das Paket "
+        "*kostet* Tokens (Regel-Load ist die ehrliche Up-Front-Steuer); "
+        "negative Δ-Werte = das Paket *spart* Tokens.\n",
+        "| Stufe | Was sie tut | Δ Tokens | Δ € (1k Req) | Kumulativ | Quelle |",
+        "|---|---|---:|---:|---:|---|",
+    ]
+    for rung in report.get("cost_ladder", []):
+        if rung["id"] == "baseline":
+            label_cell = f"**{rung['label']}**"
+        else:
+            label_cell = rung["label"]
+        what = rung.get("what_it_does", "")
+        token_delta = int(rung.get("token_delta", 0))
+        eur_delta = float(rung.get("eur_delta", 0.0))
+        cum = float(rung.get("cumulative_pct", 0.0))
+        conf = confidence_badge(rung.get("confidence", "pending"))
+        source = rung.get("source_report", "")
+        # Honesty stamp: an `up-front-cost` note on the load rung.
+        if rung["id"] == "load" and token_delta > 0:
+            what = f"{what} ⚠️ erst teurer"
+        lines.append(
+            f"| {label_cell} | {what} | "
+            f"{fmt_signed_int(token_delta)} | {fmt_eur(eur_delta)} | "
+            f"{fmt_pct(cum)} | `{source}` · {conf} |"
+        )
+        if rung.get("footnote"):
+            lines.append(
+                f"| | _Fußnote:_ {rung['footnote']} | | | | |"
+            )
+
+    totals = report.get("totals", {})
+    cum_tokens = int(totals.get("cumulative_token_delta", 0))
+    cum_eur = float(totals.get("cumulative_eur_delta", 0.0))
+    cum_pct = float(totals.get("cumulative_pct", 0.0))
+    verdict = totals.get("net_verdict", "—")
+    verdict_label = {
+        "net-saving": "**NETTO: Ersparnis** ✅",
+        "net-cost": "**NETTO: Mehrkosten** ⚠️",
+        "break-even": "**NETTO: Break-Even** ⚖️",
+    }.get(verdict, f"**NETTO: {verdict}**")
+    lines.extend(
+        [
+            "",
+            f"{verdict_label} — "
+            f"**{fmt_signed_int(cum_tokens)} Tokens / Request**, "
+            f"**{fmt_eur(cum_eur)}** auf "
+            f"{report.get('reference_scale', {}).get('requests', 1000):,} Requests, "
+            f"kumulativ **{fmt_pct(cum_pct)}** vs. Baseline.\n",
+        ]
+    )
+    return "\n".join(lines)
+
+
+def render_panel_b(report: Dict[str, Any]) -> str:
+    lines = [
+        "## Panel B — Verhalten (mit vs. ohne)\n",
+        "Vier reale Vergleiche aus echten Bench-Runs. Hier liegt der "
+        "Wert, den Tokens allein nicht messen: ob der Agent das "
+        "richtige Skill wählt, bei riskanten Aktionen stoppt, weniger "
+        "rückfragt und mehr Aufgaben abschließt.\n",
+        "| Metrik | Was es bedeutet | Mit Paket | Ohne Paket | Δ | Mode |",
+        "|---|---|---:|---:|---:|---|",
+    ]
+    for metric in report.get("behaviour", []):
+        label = metric["label"]
+        what = metric.get("what_this_means", "")
+        unit = metric.get("unit", "")
+        mode = mode_badge(metric.get("mode", "dry-run"))
+
+        def _fmt(v: Any) -> str:
+            if v is None:
+                return "—"
+            if unit == "pct" and isinstance(v, (int, float)):
+                return f"{float(v) * 100:.1f}%"
+            if unit == "count":
+                return str(int(v))
+            if unit == "ratio" and isinstance(v, (int, float)):
+                return f"{float(v):.3f}"
+            if unit == "seconds" and isinstance(v, (int, float)):
+                return f"{float(v):.1f}s"
+            return str(v)
+
+        with_v = _fmt(metric.get("with"))
+        without_v = _fmt(metric.get("without"))
+        delta_v = _fmt(metric.get("delta"))
+        lines.append(
+            f"| {label} | {what} | {with_v} | {without_v} | {delta_v} | {mode} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def render_glossary() -> str:
+    return (
+        "## Glossar\n"
+        "\n"
+        "Plain-language Definitionen für den nicht-Entwickler-Reader.\n"
+        "\n"
+        "- **Token** — die Einheit, in der ein Sprachmodell abrechnet. "
+        "Faustregel: ein Token ≈ 4 Zeichen deutsch/englischer Prosa. "
+        "1.000 Tokens ≈ 750 Wörter.\n"
+        "- **Input-Tokens** — alles, was das Modell pro Turn liest "
+        "(System-Prompt, immer-aktive Regeln, deine Nachricht, frühere "
+        "Konversation). Das Paket fügt hier Regeln hinzu — Installation "
+        "kostet Input-Tokens.\n"
+        "- **Output-Tokens** — was das Modell zurückschreibt. Meist "
+        "weniger als Input. Pro Token teurer als Input.\n"
+        "- **condense** — ein Build-Schritt, der die Regel-Dateien "
+        "vor dem Ausliefern schrumpft (`.agent-src.uncondensed` → "
+        "`.agent-src`). Spart Input-Tokens bei jedem Request.\n"
+        "- **rtk** — der *Rust Token Killer*, ein CLI-Wrapper, der "
+        "verbose Output (`git status`, lint-Output, test-Runner) "
+        "filtert, bevor das Modell ihn liest. Spart Input-Tokens auf "
+        "Tool-Calls.\n"
+        "- **terse / telegraph** — ein Stil (kurze Phrasen, "
+        "weggelassene Artikel), den der Agent für knappere Antworten "
+        "nutzt. Spart Output-Tokens — wenn der Korpus es belohnt.\n"
+        "- **Ohne Paket / Mit Paket** — *without the package* / *with "
+        "the package* — die zwei Arme des A/B-Vergleichs.\n"
+        "- **€-per-1k-requests** — Token-Kosten auf der "
+        "Referenz-Skala (1.000 Requests durchschnittlicher Größe, "
+        "gepreist mit den aktuellen Sonnet-Raten aus "
+        "`internal/bench/pricing.yaml`).\n"
+    )
+
+
+def render_methodology(report: Dict[str, Any]) -> str:
+    notes = report.get("notes", [])
+    lines = [
+        "## Methodik & Quellen\n",
+        "Diese Seite ist eine **abgeleitete** Sicht — keine eigene "
+        "Messung. Sie fasst drei bestehende Bench-Surfaces zusammen "
+        "(siehe Spalte 'Quelle' in Panel A). Die maschinen-lesbaren "
+        "Roh-Reports bleiben die Source-of-Truth:\n",
+        "- `internal/bench/reports/telegraph-v1.json` / `telegraph-v2.json` "
+        "— Telegraph/Condense-Messungen.\n",
+        "- `agents/runtime/frugality/baseline.jsonl` — der Paket-Load "
+        "(Metric A footprint).\n",
+        "- `internal/bench/reports/rtk/latest.json` — die rtk-Messung "
+        "(neu, Phase 2).\n",
+        "- `internal/bench/reports/ab/*-ab-trackb-{with,without}.json` "
+        "— A/B Track B (Verhalten).\n",
+        "- `internal/bench/reports/*-dev.json` — Dev-Korpus Selection-"
+        "Accuracy.\n",
+        "",
+        "**A/B-technischer Anhang:** [`docs/benchmark.md`](benchmark.md) "
+        "trägt die Cache-Key-, Integrity- und Methodik-Details des "
+        "A/B-Benches — wer den Variant-Axis-Beweis sehen will, liest "
+        "dort weiter.\n",
+        "",
+    ]
+    if notes:
+        lines.append("**Hinweise aus dem Report:**\n")
+        for note in notes:
+            lines.append(f"- {note}")
+        lines.append("")
+    lines.append(f"_Last rendered: `{utc_iso()}`_\n")
+    return "\n".join(lines)
+
+
+def render_placeholder() -> str:
+    return (
+        "# Value Dashboard — Platzhalter\n"
+        "\n"
+        "_Es liegt noch kein `value-v1` Report unter "
+        "`internal/bench/reports/value/latest.json` vor._\n"
+        "\n"
+        "Einen erzeugen mit:\n"
+        "\n"
+        "```sh\n"
+        "task value\n"
+        "```\n"
+        "\n"
+        "Die Methodik dieses Dashboards ist beschrieben in "
+        "`docs/contracts/value-dashboard-spec.md` und der zugehörigen "
+        "Roadmap `agents/roadmaps/road-to-readable-value-dashboard.md`.\n"
+        "\n"
+        f"_Last rendered: {utc_iso()}_\n"
+    )
+
+
+def render(quiet: bool = False) -> int:
+    report = safe_load(LATEST)
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    if not report:
+        OUT_PATH.write_text(render_placeholder())
+        if not quiet:
+            sys.stdout.write(
+                f"render_value_md: no report — wrote placeholder to "
+                f"{OUT_PATH.relative_to(REPO_ROOT)}\n"
+            )
+        return 0
+    parts = [
+        render_intro(report),
+        render_panel_a(report),
+        render_panel_b(report),
+        render_glossary(),
+        render_methodology(report),
+    ]
+    OUT_PATH.write_text("\n".join(parts))
+    if not quiet:
+        sys.stdout.write(
+            f"render_value_md: wrote {OUT_PATH.relative_to(REPO_ROOT)}\n"
+        )
+    return 0
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Render docs/value.md from the latest value-v1 report."
+    )
+    parser.add_argument("--quiet", action="store_true", help="Suppress stdout.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv if argv is not None else sys.argv[1:])
+    return render(quiet=args.quiet)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())