ma-agents 3.13.2 → 3.14.1

package/lib/bmad-cache/bmb/skills/bmad-agent-builder/scripts/render_report.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# ///
+"""Render the analysis report deterministically from findings JSON.
+
+Injects a validated findings JSON object into the report shell's
+report-data island and writes the self-contained HTML atomically.
+With --md, also writes a markdown rendering of the same data as the
+archival artifact.
+
+Refuses (non-zero exit, message on stderr) when the JSON does not
+parse, fails shape validation, or still carries the shell's
+placeholder subject — a refused render means fix the findings file
+and re-run, never hand-edit the HTML.
+
+Usage:
+  uv run render_report.py <findings.json> --shell <report-shell.html> \
+      -o <out.html> [--md <out.md>]
+
+On success prints one JSON line: output paths, grade, and severity
+counts derived from the findings array.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import tempfile
+from pathlib import Path
+
+SEVERITIES = ("critical", "high", "medium", "low")
+GRADES = ("excellent", "good", "fair", "poor")
+PLACEHOLDER_SUBJECT = "__PLACEHOLDER__"
+ISLAND_RE = re.compile(
+    r'(<script[^>]*\bid="report-data"[^>]*>)(.*?)(</script>)', re.DOTALL
+)
+
+
+def fail(message: str) -> None:
+    print(f"render_report: {message}", file=sys.stderr)
+    sys.exit(1)
+
+
+def validate(data: object) -> list[str]:
+    """Return a list of shape errors; empty list means valid."""
+    if not isinstance(data, dict):
+        return ["top level must be a JSON object"]
+    errors: list[str] = []
+
+    subject = data.get("subject")
+    if not isinstance(subject, str) or not subject.strip():
+        errors.append('"subject" must be a non-empty string')
+    elif PLACEHOLDER_SUBJECT in subject:
+        errors.append(
+            f'"subject" still carries the placeholder {PLACEHOLDER_SUBJECT}; '
+            "this is the unfilled shell sample, not real findings"
+        )
+
+    findings = data.get("findings")
+    if not isinstance(findings, list):
+        errors.append('"findings" must be an array (use [] for a clean pass)')
+    else:
+        for i, finding in enumerate(findings):
+            if not isinstance(finding, dict):
+                errors.append(f"findings[{i}] must be an object")
+
+    grade = data.get("grade")
+    if grade is not None and grade not in GRADES:
+        errors.append(f'"grade" must be one of: {", ".join(GRADES)}')
+
+    for key in ("themes", "recommendations"):
+        value = data.get(key)
+        if value is not None and (
+            not isinstance(value, list)
+            or any(not isinstance(item, dict) for item in value)
+        ):
+            errors.append(f'"{key}" must be an array of objects')
+
+    strengths = data.get("strengths")
+    if strengths is not None and (
+        not isinstance(strengths, list)
+        or any(not isinstance(item, str) for item in strengths)
+    ):
+        errors.append('"strengths" must be an array of strings')
+
+    return errors
+
+
+def severity_counts(findings: list[dict]) -> dict[str, int]:
+    counts = {sev: 0 for sev in SEVERITIES}
+    for finding in findings:
+        sev = finding.get("severity")
+        counts[sev if sev in counts else "low"] += 1
+    return counts
+
+
+def inject(shell_html: str, data: dict) -> str:
+    payload = json.dumps(data, ensure_ascii=False, indent=2)
+    # A "</" sequence inside a JSON string would close the script tag
+    # early in the browser; "<\/" is the same string to JSON.parse.
+    payload = payload.replace("</", "<\\/")
+
+    def replace(match: re.Match) -> str:
+        return match.group(1) + "\n" + payload + "\n" + match.group(3)
+
+    new_html, count = ISLAND_RE.subn(replace, shell_html, count=1)
+    if count != 1:
+        fail('shell has no <script id="report-data"> island to fill')
+    return new_html
+
+
+def atomic_write(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(
+        dir=path.parent, prefix=path.name + ".", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(text)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(tmp, path)
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+
+
+def _finding_lines(finding: dict, heading_level: str) -> list[str]:
+    fid = str(finding.get("id", ""))
+    title = str(finding.get("title", "(untitled finding)"))
+    lines = [f"{heading_level} {fid} — {title}" if fid else f"{heading_level} {title}", ""]
+    for key, label in (
+        ("lens", "Lens"),
+        ("location", "Location"),
+        ("evidence", "Evidence"),
+        ("recommendation", "Recommendation"),
+        ("proposed_smallest", "Proposed smallest"),
+        ("predicted_delta", "Predicted delta"),
+    ):
+        value = finding.get(key)
+        if value:
+            value = f"`{value}`" if key == "location" else str(value)
+            lines.append(f"- {label}: {value}")
+    lines.append("")
+    return lines
+
+
+def render_md(data: dict) -> str:
+    findings = [f for f in data.get("findings", []) if isinstance(f, dict)]
+    by_id = {str(f.get("id")): f for f in findings if f.get("id") is not None}
+    counts = severity_counts(findings)
+    lines: list[str] = []
+
+    lines.append(f"# Analysis Report: {data.get('subject', '')}")
+    lines.append("")
+    meta = []
+    if data.get("generated"):
+        meta.append(f"Generated: {data['generated']}")
+    if data.get("schema_version") is not None:
+        meta.append(f"Schema: {data['schema_version']}")
+    if meta:
+        lines.append(" · ".join(meta))
+        lines.append("")
+
+    if data.get("grade"):
+        lines.append(f"**Grade: {str(data['grade']).capitalize()}**")
+        lines.append("")
+    if data.get("verdict"):
+        lines.append(f"> {data['verdict']}")
+        lines.append("")
+    summary = data.get("summary")
+    if isinstance(summary, str) and summary:
+        lines.append(summary)
+        lines.append("")
+
+    lines.append("| Severity | Count |")
+    lines.append("| --- | --- |")
+    for sev in SEVERITIES:
+        lines.append(f"| {sev.capitalize()} | {counts[sev]} |")
+    lines.append("")
+
+    themes = data.get("themes") or []
+    if themes:
+        lines.append("## Themes")
+        lines.append("")
+        for i, theme in enumerate(themes, 1):
+            lines.append(f"### {i}. {theme.get('title', '(untitled theme)')}")
+            lines.append("")
+            if theme.get("root_cause"):
+                lines.append(f"- Root cause: {theme['root_cause']}")
+            if theme.get("action"):
+                lines.append(f"- Fix: {theme['action']}")
+            ids = theme.get("finding_ids") or []
+            if ids:
+                lines.append("- Findings:")
+                for fid in ids:
+                    finding = by_id.get(str(fid))
+                    if finding:
+                        loc = finding.get("location")
+                        suffix = f" — `{loc}`" if loc else ""
+                        lines.append(
+                            f"  - `{fid}` {finding.get('title', '')}{suffix}"
+                        )
+                    else:
+                        lines.append(f"  - `{fid}`")
+            lines.append("")
+
+    strengths = data.get("strengths") or []
+    if strengths:
+        lines.append("## Strengths")
+        lines.append("")
+        for strength in strengths:
+            lines.append(f"- {strength}")
+        lines.append("")
+
+    recommendations = data.get("recommendations") or []
+    if recommendations:
+        lines.append("## Recommendations")
+        lines.append("")
+        for i, rec in enumerate(recommendations, 1):
+            rank = rec.get("rank", i)
+            resolves = rec.get("resolves")
+            if isinstance(resolves, list) and resolves:
+                suffix = " (resolves: " + ", ".join(map(str, resolves)) + ")"
+            elif isinstance(resolves, (int, float)):
+                suffix = f" (resolves {int(resolves)} findings)"
+            else:
+                suffix = ""
+            lines.append(f"{rank}. {rec.get('action', '')}{suffix}")
+        lines.append("")
+
+    # Optional agent blocks: rendered only when present so the same
+    # renderer serves both the workflow and agent schemas.
+    profile = data.get("agent_profile")
+    if isinstance(profile, dict) and any(profile.values()):
+        lines.append("## Agent Profile")
+        lines.append("")
+        for key, label in (
+            ("name", "Name"),
+            ("title", "Title"),
+            ("agent_type", "Type"),
+            ("mission", "Mission"),
+        ):
+            if profile.get(key):
+                lines.append(f"- {label}: {profile[key]}")
+        lines.append("")
+
+    capabilities = data.get("capabilities")
+    if isinstance(capabilities, list) and capabilities:
+        lines.append("## Capabilities")
+        lines.append("")
+        for cap in capabilities:
+            if not isinstance(cap, dict) or not cap.get("name"):
+                continue
+            kind = f" ({cap['kind']})" if cap.get("kind") else ""
+            note = f" — {cap['note']}" if cap.get("note") else ""
+            lines.append(f"- **{cap['name']}**{kind}{note}")
+        lines.append("")
+
+    detailed = data.get("detailed_analysis")
+    if isinstance(detailed, dict) and detailed:
+        lines.append("## Per-Lens Verdicts")
+        lines.append("")
+        for lens, verdict in detailed.items():
+            if verdict:
+                lines.append(f"- **{lens}**: {verdict}")
+        lines.append("")
+
+    sanctum = data.get("sanctum")
+    if isinstance(sanctum, dict) and sanctum.get("present") is not False:
+        rows = []
+        if sanctum.get("location"):
+            rows.append(f"- Location: `{sanctum['location']}`")
+        files = sanctum.get("files") or []
+        if files:
+            rows.append("- Files: " + ", ".join(f"`{f}`" for f in files))
+        if sanctum.get("note"):
+            rows.append(f"- Note: {sanctum['note']}")
+        if rows:
+            lines.append("## Sanctum (runtime memory)")
+            lines.append("")
+            lines.extend(rows)
+            lines.append("")
+
+    experience = data.get("experience")
+    if isinstance(experience, dict):
+        journeys = [
+            j for j in experience.get("journeys") or [] if isinstance(j, dict)
+        ]
+        headless = experience.get("headless")
+        if journeys or headless:
+            lines.append("## Experience")
+            lines.append("")
+            for journey in journeys:
+                steps = f" — {journey['steps']}" if journey.get("steps") else ""
+                lines.append(f"- **{journey.get('name', '(unnamed journey)')}**{steps}")
+            if headless:
+                lines.append(f"- Headless: {headless}")
+            lines.append("")
+
+    lines.append("## Findings")
+    lines.append("")
+    if not findings:
+        lines.append("No findings: the scanners returned a clean pass.")
+        lines.append("")
+    else:
+        for sev in SEVERITIES:
+            group = [
+                f
+                for f in findings
+                if (f.get("severity") if f.get("severity") in SEVERITIES else "low")
+                == sev
+            ]
+            if not group:
+                continue
+            lines.append(f"### {sev.capitalize()} ({len(group)})")
+            lines.append("")
+            for finding in group:
+                lines.extend(_finding_lines(finding, "####"))
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Inject findings JSON into the report shell and render HTML (+ optional markdown)."
+    )
+    parser.add_argument("findings", type=Path, help="path to findings.json")
+    parser.add_argument(
+        "--shell", type=Path, required=True, help="path to report-shell.html"
+    )
+    parser.add_argument(
+        "-o", "--output", type=Path, required=True, help="output HTML path"
+    )
+    parser.add_argument(
+        "--md", type=Path, help="also write a markdown rendering to this path"
+    )
+    args = parser.parse_args()
+
+    try:
+        raw = args.findings.read_text(encoding="utf-8")
+    except OSError as err:
+        fail(f"cannot read {args.findings}: {err}")
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as err:
+        fail(f"{args.findings} is not valid JSON: {err}")
+
+    errors = validate(data)
+    if errors:
+        fail(
+            f"{args.findings} failed shape validation:\n  - "
+            + "\n  - ".join(errors)
+        )
+
+    try:
+        shell_html = args.shell.read_text(encoding="utf-8")
+    except OSError as err:
+        fail(f"cannot read shell {args.shell}: {err}")
+
+    atomic_write(args.output, inject(shell_html, data))
+    if args.md:
+        atomic_write(args.md, render_md(data))
+
+    findings = [f for f in data.get("findings", []) if isinstance(f, dict)]
+    print(
+        json.dumps(
+            {
+                "html_report": str(args.output),
+                "md_report": str(args.md) if args.md else None,
+                "grade": data.get("grade"),
+                "counts": severity_counts(findings),
+                "findings": len(findings),
+            }
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/lib/bmad-cache/bmb/skills/bmad-eval-runner/assets/adapter-claude-code.json

@@ -0,0 +1,9 @@
+{
+  "name": "claude-code",
+  "invocation": ["claude", "-p", "{prompt}", "--output-format", "stream-json", "--verbose", "--dangerously-skip-permissions"],
+  "auth_env": "ANTHROPIC_API_KEY",
+  "transcript": { "format": "stdout-jsonl" },
+  "skill_dir": ".claude/skills",
+  "load_signal": { "skill_tool": "Skill", "read_tool": "Read" },
+  "env_passthrough": []
+}

package/lib/bmad-cache/bmb/skills/bmad-eval-runner/scripts/aggregate_benchmark.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""Variance benchmark: summarize a metric across N runs, and compare two configs.
+
+A single skill run is noisy. Running the same case N times and summarizing the
+spread tells you whether a difference between two versions is real or just noise.
+This script computes, per numeric metric, the mean, the sample standard deviation
+(n-1, the unbiased estimator for a sample), the min, and the max across N runs.
+Given two such config summaries it reports the delta on each shared metric so a
+"did the change help" question gets a number instead of a guess.
+
+Input shapes accepted for a single config:
+  - a list of run records, each a flat dict of metric -> number
+      [{"elapsed_s": 12.1, "total_tokens": 800}, {"elapsed_s": 11.4, ...}]
+  - {"runs": [ ...records... ]}
+  - a directory of run folders, each holding timing.json files written by
+    run_evals.py (the script reads every timing.json under the directory and
+    treats each as one run record)
+
+Usage:
+  Summarize one config across its runs:
+    python3 aggregate_benchmark.py --runs CONFIG_A.json
+    python3 aggregate_benchmark.py --runs RUN_DIR/        (reads timing.json files)
+
+  Compare two configs (each summarized, then delta = B - A):
+    python3 aggregate_benchmark.py --baseline A.json --variant B.json
+
+  Self-test on a known fixture (no external input needed):
+    python3 aggregate_benchmark.py --self-test
+
+Output is one JSON object on stdout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+
+NUMERIC = (int, float)
+
+
+# --- statistics -------------------------------------------------------------
+
+def sample_stddev(values: list[float]) -> float:
+    """Sample standard deviation using n-1 (Bessel's correction).
+
+    Returns 0.0 for fewer than two values, where the sample variance is
+    undefined and reporting zero spread is the least surprising choice.
+    """
+    n = len(values)
+    if n < 2:
+        return 0.0
+    mean = sum(values) / n
+    var = sum((x - mean) ** 2 for x in values) / (n - 1)
+    return math.sqrt(var)
+
+
+def summarize_metric(values: list[float]) -> dict:
+    return {
+        "n": len(values),
+        "mean": (sum(values) / len(values)) if values else 0.0,
+        "stddev": sample_stddev(values),
+        "min": min(values) if values else 0.0,
+        "max": max(values) if values else 0.0,
+    }
+
+
+def collect_numeric_metrics(records: list[dict]) -> dict[str, list[float]]:
+    """Group every numeric field across records by metric name."""
+    by_metric: dict[str, list[float]] = {}
+    for rec in records:
+        if not isinstance(rec, dict):
+            continue
+        for key, val in rec.items():
+            if isinstance(val, bool):
+                continue  # bools are ints in Python; not a metric
+            if isinstance(val, NUMERIC):
+                by_metric.setdefault(key, []).append(float(val))
+    return by_metric
+
+
+def summarize_config(records: list[dict]) -> dict:
+    by_metric = collect_numeric_metrics(records)
+    return {
+        "runs": len(records),
+        "metrics": {name: summarize_metric(vals)
+                    for name, vals in sorted(by_metric.items())},
+    }
+
+
+def delta_configs(baseline: dict, variant: dict) -> dict:
+    """Per shared metric, delta = variant.mean - baseline.mean, plus context."""
+    b_metrics = baseline.get("metrics", {})
+    v_metrics = variant.get("metrics", {})
+    shared = sorted(set(b_metrics) & set(v_metrics))
+    out: dict[str, dict] = {}
+    for name in shared:
+        b = b_metrics[name]
+        v = v_metrics[name]
+        diff = v["mean"] - b["mean"]
+        pct = (diff / b["mean"] * 100.0) if b["mean"] != 0 else None
+        out[name] = {
+            "baseline_mean": b["mean"],
+            "variant_mean": v["mean"],
+            "delta": diff,
+            "delta_pct": pct,
+            "baseline_stddev": b["stddev"],
+            "variant_stddev": v["stddev"],
+        }
+    return out
+
+
+# --- input loading ----------------------------------------------------------
+
+def load_records(path: Path) -> list[dict]:
+    """Load run records from a JSON file, a {'runs': [...]} file, or a dir of
+    timing.json files."""
+    if path.is_dir():
+        records: list[dict] = []
+        for f in sorted(path.rglob("timing.json")):
+            try:
+                data = json.loads(f.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError):
+                continue
+            if isinstance(data, dict):
+                records.append(data)
+        return records
+
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict) and "runs" in data:
+        data = data["runs"]
+    if not isinstance(data, list):
+        raise ValueError(f"expected a list of run records in {path}")
+    return [r for r in data if isinstance(r, dict)]
+
+
+# --- self-test --------------------------------------------------------------
+
+def run_self_test() -> int:
+    """Verify mean/stddev/min/max/delta on a known fixture."""
+    config_a = [
+        {"elapsed_s": 10.0, "total_tokens": 100},
+        {"elapsed_s": 12.0, "total_tokens": 200},
+        {"elapsed_s": 14.0, "total_tokens": 300},
+    ]
+    summary_a = summarize_config(config_a)
+    el = summary_a["metrics"]["elapsed_s"]
+    # mean of 10,12,14 = 12; n-1 stddev = sqrt(((-2)^2+0+2^2)/2)=sqrt(4)=2
+    assert el["n"] == 3, el
+    assert abs(el["mean"] - 12.0) < 1e-9, el
+    assert abs(el["stddev"] - 2.0) < 1e-9, el
+    assert el["min"] == 10.0 and el["max"] == 14.0, el
+    tok = summary_a["metrics"]["total_tokens"]
+    # mean of 100,200,300 = 200; n-1 stddev = sqrt((10000+0+10000)/2)=100
+    assert abs(tok["mean"] - 200.0) < 1e-9, tok
+    assert abs(tok["stddev"] - 100.0) < 1e-9, tok
+
+    # single value -> stddev 0
+    one = summarize_config([{"x": 5}])
+    assert one["metrics"]["x"]["stddev"] == 0.0, one
+
+    # bools are not treated as metrics
+    with_bool = summarize_config([{"ok": True, "x": 1}, {"ok": False, "x": 3}])
+    assert "ok" not in with_bool["metrics"], with_bool
+    assert abs(with_bool["metrics"]["x"]["mean"] - 2.0) < 1e-9, with_bool
+
+    # delta: variant slower by 3s on mean, faster question answered by sign
+    config_b = [
+        {"elapsed_s": 13.0, "total_tokens": 90},
+        {"elapsed_s": 15.0, "total_tokens": 110},
+        {"elapsed_s": 17.0, "total_tokens": 100},
+    ]
+    summary_b = summarize_config(config_b)
+    d = delta_configs(summary_a, summary_b)
+    # elapsed mean: A=12, B=15 -> delta +3, pct +25%
+    assert abs(d["elapsed_s"]["delta"] - 3.0) < 1e-9, d
+    assert abs(d["elapsed_s"]["delta_pct"] - 25.0) < 1e-9, d
+    # tokens mean: A=200, B=100 -> delta -100, pct -50%
+    assert abs(d["total_tokens"]["delta"] + 100.0) < 1e-9, d
+    assert abs(d["total_tokens"]["delta_pct"] + 50.0) < 1e-9, d
+
+    print(json.dumps({"self_test": "passed",
+                      "checked": ["mean", "stddev_n_minus_1", "min", "max",
+                                  "single_value_stddev", "bool_excluded",
+                                  "delta", "delta_pct"]}))
+    return 0
+
+
+# --- main -------------------------------------------------------------------
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--runs", type=Path,
+                   help="summarize one config (JSON file or dir of timing.json)")
+    p.add_argument("--baseline", type=Path,
+                   help="baseline config for a two-config comparison")
+    p.add_argument("--variant", type=Path,
+                   help="variant config for a two-config comparison")
+    p.add_argument("--self-test", action="store_true",
+                   help="run the built-in fixture self-test and exit")
+    args = p.parse_args(argv)
+
+    if args.self_test:
+        return run_self_test()
+
+    if args.baseline and args.variant:
+        b = summarize_config(load_records(args.baseline))
+        v = summarize_config(load_records(args.variant))
+        out = {
+            "baseline": b,
+            "variant": v,
+            "delta": delta_configs(b, v),
+        }
+        print(json.dumps(out, indent=2))
+        return 0
+
+    if args.runs:
+        out = summarize_config(load_records(args.runs))
+        print(json.dumps(out, indent=2))
+        return 0
+
+    p.error("provide --runs, or both --baseline and --variant, or --self-test")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())