ops-tune-workbench-plugin 0.0.0-stage → 0.0.1-beta.2

package/skills/tool-ops-tune/ops-verdict/scripts/delta_compare.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------------------------------------
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+# ----------------------------------------------------------------------------------------------------------
+"""delta_compare.py — 前后对比硬校验（口径一致性 + 精度门禁 + 噪声带三段结论）
+
+设计对齐（docs/算子调优AGENT设计文档.md §6 数据契约）：
+  - HARD_KEYS 口径不一致 → exit 2 拒绝比较（性能数字在口径漂移下无意义）。
+  - MECHANISM_KEYS（tiling_key/block_dim）默认 WARN 写入报告；--strict 恢复禁比
+    （分核/tiling 类优化本身就会改变这两个字段——这是对优化本质的建模）。
+  - 精度门禁（第三道，脚本化）：任一侧 precision != pass → exit 2。
+  - 噪声带三段结论（实测纪律）：±5% 内不判收益也不判劣化 → 加采样轮次。
+
+用法：
+  python3 delta_compare.py --baseline baseline/result.json --after round-01/result.json \
+      --output comparison.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import sys
+from pathlib import Path
+
+HARD_KEYS = (
+    "op",
+    "soc",
+    "mode",
+    "timing_method",
+    "baseline_kind",
+    "cann_version",
+    "device_id",
+    "shape",
+    "dtype",
+    "format",
+    "warmup",
+    "repeat",
+)
+MECHANISM_KEYS = ("tiling_key", "block_dim")
+NOISE_BAND = 0.05  # ±5%：噪声带内不归因（run 间噪声 >5% 时小于噪声的收益不得宣称）
+
+
+def load(p: str) -> dict:
+    return json.loads(Path(p).read_text(encoding="utf-8"))
+
+
+def fmt_speedup(b, a):
+    if b and a and a > 0:
+        return b / a
+    return None
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="前后对比硬校验")
+    ap.add_argument(
+        "--baseline", required=True, help="snapshot.py 产出的基线 result.json"
+    )
+    ap.add_argument("--after", required=True, help="优化后 result.json（同口径）")
+    ap.add_argument("--output", required=True, help="comparison.md 输出")
+    ap.add_argument(
+        "--strict",
+        action="store_true",
+        help="机制变量（tiling_key/block_dim）差异也算口径不一致",
+    )
+    ap.add_argument("--noise-band", type=float, default=NOISE_BAND)
+    args = ap.parse_args()
+
+    base, after = load(args.baseline), load(args.after)
+
+    # ---- 门禁 1：口径硬校验 ----
+    hard = tuple(HARD_KEYS) + (MECHANISM_KEYS if args.strict else ())
+    diffs = [
+        f"{k}: {base.get(k)!r} != {after.get(k)!r}"
+        for k in hard
+        if base.get(k) is not None
+        and after.get(k) is not None
+        and base.get(k) != after.get(k)
+    ]
+    if diffs:
+        print(
+            "ERROR: 基线与优化后口径不一致，禁止比较：\n  " + "\n  ".join(diffs),
+            file=sys.stderr,
+        )
+        print("任一口径字段变化都应重建基线（跨口径比较禁令）。", file=sys.stderr)
+        return 2
+
+    # ---- 门禁 2：精度门禁（第三道）----
+    if base.get("precision") != "pass" or after.get("precision") != "pass":
+        print(
+            "ERROR: 只有精度均为 pass 的结果才能生成性能结论 "
+            f"(baseline={base.get('precision')}, after={after.get('precision')})",
+            file=sys.stderr,
+        )
+        return 2
+
+    # 机制变量 WARN ----
+    mech_warns = [
+        f"{k}: {base.get(k)!r} → {after.get(k)!r}"
+        for k in MECHANISM_KEYS
+        if base.get(k) is not None
+        and after.get(k) is not None
+        and base.get(k) != after.get(k)
+    ]
+
+    b_us, a_us = base.get("kernel_avg_us"), after.get("kernel_avg_us")
+    speedup = fmt_speedup(b_us, a_us)
+
+    # ---- 三段结论 ----
+    if speedup is None:
+        conclusion = "UNRESOLVED：缺少可比的 kernel_avg_us（或标记为 partial/dry_run）"
+        action = "补齐性能数据或更换计时口径后重比"
+    elif speedup > 1 + args.noise_band:
+        conclusion = (
+            f"提升：speedup={speedup:.3f}x（>{1 + args.noise_band:.2f} 超出噪声带）"
+        )
+        action = "KEEP：该轮保留，进入下一轮单变量迭代"
+    elif speedup < 1 - args.noise_band:
+        conclusion = f"劣化：speedup={speedup:.3f}x（<{1 - args.noise_band:.2f}）"
+        action = "REVERT：回滚本轮改动；回滚后须复跑确认回到基线水平再进入下一轮"
+    else:
+        conclusion = (
+            f"噪声带内（{speedup:.3f}x，±{args.noise_band * 100:.0f}% 内）："
+            "不判收益也不判劣化"
+        )
+        # F9：repeat 是 HARD key——翻倍复测必须「以新 repeat 同步重采基线与优化侧、
+        # 重建快照（新目录）后再比」，直接翻倍 after 会被口径硬校验 exit 2 拒绝
+        action = (
+            "增加采样轮次（repeat 翻倍）复测：以新 repeat **同步重采基线与优化侧**、"
+            "在新目录重建两侧快照后再比（repeat 属口径指纹，单侧翻倍会被 HARD 校验拒绝）；"
+            "仍落带内按无稳定收益处置"
+        )
+
+    # 误差指纹对照（精度数字必须同量级）
+    err_lines = []
+    for k in ("mismatch", "max_abs_err", "max_rel_err"):
+        bv, av = base.get(k), after.get(k)
+        if bv is not None or av is not None:
+            err_lines.append(f"| {k} | {bv} | {av} |")
+
+    # ---- 抖动诊断段（F20：分布字段存在时输出；缺失时向后兼容不输出）----
+    jitter_lines = []
+    for k in ("kernel_p50_us", "kernel_p95_us", "kernel_max_us"):
+        bv, av = base.get(k), after.get(k)
+        if bv is not None or av is not None:
+            jitter_lines.append(f"| {k} | {bv} | {av} |")
+    jitter_notes = []
+    for side_name, snap in (("baseline", base), ("after", after)):
+        p50, p95, avg = (
+            snap.get("kernel_p50_us"),
+            snap.get("kernel_p95_us"),
+            snap.get("kernel_avg_us"),
+        )
+        if p50 is not None and p95 is not None and avg:
+            if p50 * 2 < p95:
+                jitter_notes.append(
+                    f"{side_name}: p95（{p95}）> 2×p50（{p50}）——疑似双峰分布，"
+                    "先查是否有两种稳态（如 DVFS 档位 / 缓存对齐边界）再下结论"
+                )
+            elif p95 / avg > 1.5:
+                jitter_notes.append(
+                    f"{side_name}: p95 偏离均值 {p95 / avg:.2f}x——长尾显著，"
+                    "单均值口径下噪声带结论的置信度不足"
+                )
+    b_p50, b_p95 = base.get("kernel_p50_us"), base.get("kernel_p95_us")
+    a_p50, a_p95 = after.get("kernel_p50_us"), after.get("kernel_p95_us")
+    if b_p50 and b_p95 and a_p50 and a_p95:
+        if (b_p95 / b_p50) < (a_p95 / a_p50) * 0.5:
+            jitter_notes.append(
+                "优化侧 p95/p50 比值显著大于基线（≥2x）——收益集中在快轮次、"
+                "长尾反而恶化时，geomean/avg 口径可能掩盖劣化"
+            )
+
+    md = [
+        f"# 前后对比：{base.get('op')} — {after.get('variant', 'optimized')}",
+        "",
+        f"- 生成时间：{datetime.datetime.now().isoformat(timespec='seconds')}",
+        f"- 口径：{base.get('soc')} / CANN {base.get('cann_version')} / device {base.get('device_id')}"
+        f" / timing={base.get('timing_method')} / warmup={base.get('warmup')} repeat={base.get('repeat')}",
+        f"- 精度：baseline={base.get('precision')}  after={after.get('precision')}（工程现有测试口径）",
+        "",
+        "## 结论",
+        "",
+        f"**{conclusion}**",
+        "",
+        f"处置：{action}",
+        "",
+        "## 数据",
+        "",
+        "| 项 | baseline | after |",
+        "|---|---|---|",
+        f"| kernel_avg_us | {b_us} | {a_us} |",
+        f"| speedup | — | {speedup if speedup is None else round(speedup, 3)} |",
+        f"| status | {base.get('status')} | {after.get('status')} |",
+        *err_lines,
+    ]
+    if mech_warns:
+        md += [
+            "",
+            "## 机制变量（预期内变化，WARN 记录）",
+            "",
+            *[f"- {w}" for w in mech_warns],
+        ]
+    if jitter_lines:
+        md += [
+            "",
+            "## 抖动诊断（分布字段）",
+            "",
+            "| 项 | baseline | after |",
+            "|---|---|---|",
+            *jitter_lines,
+        ]
+        if jitter_notes:
+            md += ["", *[f"- ⚠ {n}" for n in jitter_notes]]
+    if base.get("verdict_file") or after.get("verdict_file"):
+        md += [
+            "",
+            "## 关联判定书",
+            "",
+            f"- baseline verdict: {base.get('verdict_file')}",
+            f"- after verdict: {after.get('verdict_file')}",
+        ]
+
+    out = Path(args.output)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text("\n".join(md) + "\n", encoding="utf-8")
+    print(f"结论: {conclusion}")
+    print(f"处置: {action}")
+    if mech_warns:
+        print("WARN 机制变量变化（预期内）: " + "; ".join(mech_warns))
+    print(f"→ {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/tool-ops-tune/ops-verdict/scripts/render_html.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------------------------------------
+# Copyright (c) 2026 Huawei Technologies Co., Ltd.
+# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
+# CANN Open Software License Agreement Version 2.0 (the "License").
+# Please refer to the License for details. You may not use this file except in compliance with the License.
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE in the root of the software repository for the full text of the License.
+# ----------------------------------------------------------------------------------------------------------
+"""render_html.py — comparison_matrix.md → 静态 HTML 报告（REVIEW2 G16 / 反馈 P2-3）
+
+设计对齐：
+  - 20 case × 多轮的对比矩阵纯文本人读不动（REVIEW2 G16）；同类工具已有
+    8 页交互式 HTML——本仓定位**静态 HTML**（数据已在 JSON/矩阵，不追交互，
+    反馈 P2-3 措施原文：「数据已在 JSON，不追交互式」）。
+  - 输入两种形态任选：
+    ① batch_pipeline 的矩阵目录（逐 case 的 comparison_md + snapshot json——
+       矩阵结论从 snapshot json 重读，保数值真源）
+    ② 显式 --snapshots（若干 result.json + comparison.md 对）——手工形态。
+  - 输出单文件 HTML（内联 CSS，零外部依赖，可离线打开 / 邮件附件 / 归档）。
+
+包含区块：
+  - 汇总卡片：geomean / KEEP / REVERT / 噪声带 / MISSING 计数
+  - 逐 variant 热力表：case × speedup 着色（绿=KEEP、红=REVERT、灰=带内/缺）
+  - 逐 case 明细折叠块（comparison.md 原文内嵌，<details> 展开查看）
+
+用法：
+  python3 render_html.py --matrix-dir tuning_artifacts/gelu/matrix -o report.html
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import html
+import json
+import math
+import re
+from pathlib import Path
+
+
+def load_matrix(matrix_dir: Path) -> dict:
+    """从矩阵目录重建逐 case 数据：{case: {variant: snapshot dict}} + 对比 md。"""
+    cases: dict[str, dict] = {}
+    comps: dict[tuple[str, str], str] = {}
+    for case_dir in sorted(p for p in matrix_dir.iterdir() if p.is_dir()):
+        cid = case_dir.name
+        for snap in case_dir.glob("*.json"):
+            data = json.loads(snap.read_text(encoding="utf-8"))
+            variant = data.get("variant", snap.stem)
+            cases.setdefault(cid, {})[variant] = data
+        for comp in case_dir.glob("*_comparison.md"):
+            variant = comp.name.replace("_comparison.md", "")
+            comps[(cid, variant)] = comp.read_text(encoding="utf-8")
+    return {"cases": cases, "comparisons": comps}
+
+
+def conclusion_of(comp_md: str | None, speedup: float | None) -> str:
+    if comp_md:
+        if "提升" in comp_md and speedup and speedup > 1.05:
+            return "KEEP"
+        if "劣化" in comp_md:
+            return "REVERT"
+        if "噪声带内" in comp_md:
+            return "NOISE"
+        if "拒绝" in comp_md or "口径" in comp_md.split("## 结论")[0][:50]:
+            pass
+    if speedup is None:
+        return "MISSING"
+    return "NOISE"
+
+
+def speedup_color(sp: float | None) -> str:
+    if sp is None:
+        return "#9e9e9e"  # 灰：无数据
+    if sp >= 1.05:
+        # 绿色渐变：越深收益越大（1.05 → 浅，≥1.5 → 深）
+        t = min((sp - 1.05) / 0.45, 1.0)
+        return f"rgba(46, 125, 50, {0.25 + 0.55 * t})"
+    if sp <= 0.95:
+        # 红色渐变：越深劣化越大
+        t = min((0.95 - sp) / 0.25, 1.0)
+        return f"rgba(198, 40, 40, {0.25 + 0.55 * t})"
+    return "#e0e0e0"  # 带内：浅灰
+
+
+def md_to_html(md_text: str) -> str:
+    """comparison.md 的极简渲染：表原样 + 代码块 + 段落（不引 markdown 库，
+    保持零依赖；已 html 转义后套结构）。"""
+    out = []
+    in_code = False
+    for line in md_text.splitlines():
+        e = html.escape(line)
+        if line.strip().startswith("```"):
+            out.append("</pre>" if in_code else "<pre>")
+            in_code = not in_code
+        elif in_code:
+            out.append(e)
+        elif line.startswith("|"):
+            cells = [c.strip() for c in line.strip("|").split("|")]
+            if set("".join(cells)) <= set("-: "):
+                continue  # 分隔行
+            out.append(
+                "<tr>" + "".join(f"<td>{html.escape(c)}</td>" for c in cells) + "</tr>"
+            )
+        elif line.startswith("#"):
+            lvl = len(line) - len(line.lstrip("#"))
+            out.append(f"<h{min(lvl + 2, 6)}>{e.lstrip('# ')}</h{min(lvl + 2, 6)}>")
+        else:
+            out.append(f"<p>{e}</p>" if line.strip() else "")
+    body = "\n".join(out)
+    # 把首个 <tr> 前插表头结构（简化：全部 td）
+    return re.sub(r"((?:<tr>.*?</tr>\n?)+)", r"<table>\1</table>", body)
+
+
+def render(data: dict, title: str) -> str:
+    cases = data["cases"]
+    comps = data["comparisons"]
+    variants = sorted(
+        {v for c in cases.values() for v in c}, key=lambda s: (s != "baseline", s)
+    )
+    case_ids = sorted(cases)
+    rows_speedup, statuses = {}, {}
+    for cid in case_ids:
+        base = cases[cid].get("baseline")
+        for v, snap in cases[cid].items():
+            if v == "baseline" or not base:
+                continue
+            b, a = base.get("kernel_avg_us"), snap.get("kernel_avg_us")
+            sp = (b / a) if (b and a) else None
+            rows_speedup[(cid, v)] = sp
+            statuses[(cid, v)] = conclusion_of(comps.get((cid, v)), sp)
+
+    # 汇总
+    per_variant: dict[str, dict] = {}
+    for v in variants:
+        if v == "baseline":
+            continue
+        sps = [sp for (cid, vv), sp in rows_speedup.items() if vv == v and sp]
+        sts = [st for (cid, vv), st in statuses.items() if vv == v]
+        per_variant[v] = {
+            "geomean": math.exp(sum(math.log(s) for s in sps) / len(sps))
+            if sps
+            else None,
+            "keep": sts.count("KEEP"),
+            "revert": sts.count("REVERT"),
+            "noise": sts.count("NOISE"),
+            "missing": sts.count("MISSING"),
+            "n": len(sts),
+        }
+
+    css = """
+    body { font-family: -apple-system, 'Segoe UI', 'PingFang SC', sans-serif; margin: 2rem auto;
+           max-width: 1100px; color: #212121; background: #fafafa; }
+    h1 { border-bottom: 2px solid #2e7d32; padding-bottom: .4rem; }
+    .cards { display: flex; gap: 1rem; flex-wrap: wrap; margin: 1rem 0 2rem; }
+    .card { background: #fff; border: 1px solid #e0e0e0; border-radius: 8px;
+            padding: .8rem 1.2rem; min-width: 130px; box-shadow: 0 1px 2px rgba(0,0,0,.06); }
+    .card .num { font-size: 1.6rem; font-weight: 600; }
+    .card .lbl { color: #757575; font-size: .8rem; }
+    table { border-collapse: collapse; background: #fff; margin: 1rem 0; width: 100%; }
+    th, td { border: 1px solid #e0e0e0; padding: .35rem .6rem; font-size: .85rem; text-align: center; }
+    th { background: #f5f5f5; }
+    td.sp { font-weight: 600; }
+    details { margin: .4rem 0; }
+    summary { cursor: pointer; color: #1565c0; font-size: .9rem; }
+    pre { background: #f5f5f5; padding: .6rem; border-radius: 4px; overflow-x: auto;
+          font-size: .8rem; }
+    .meta { color: #757575; font-size: .85rem; }
+    """
+
+    parts = [
+        "<!DOCTYPE html><html><head><meta charset='utf-8'>",
+        f"<title>{html.escape(title)}</title><style>{css}</style></head><body>",
+        f"<h1>{html.escape(title)}</h1>",
+        f"<p class='meta'>render_html.py 生成于 {datetime.datetime.now().isoformat(timespec='seconds')}"
+        f"（静态报告，数据真源为矩阵目录 snapshot JSON）</p>",
+    ]
+
+    for v, s in per_variant.items():
+        gm = f"{s['geomean']:.3f}x" if s["geomean"] else "—"
+        parts.append(
+            f"<div class='card'><div class='num'>{gm}</div><div class='lbl'>{html.escape(v)} geomean"
+            f"（{s['n']} case）</div></div>"
+            f"<div class='card'><div class='num' style='color:#2e7d32'>{s['keep']}</div>"
+            f"<div class='lbl'>KEEP</div></div>"
+            f"<div class='card'><div class='num' style='color:#c62828'>{s['revert']}</div>"
+            f"<div class='lbl'>REVERT</div></div>"
+            f"<div class='card'><div class='num' style='color:#9e9e9e'>{s['noise']}</div>"
+            f"<div class='lbl'>噪声带内</div></div>"
+            f"<div class='card'><div class='num'>{s['missing']}</div>"
+            f"<div class='lbl'>MISSING</div></div>"
+        )
+
+    # 热力表
+    parts.append(
+        "<h2>逐 case 热力表</h2><table><tr><th>Case</th><th>Shape</th><th>dtype</th>"
+        + "".join(f"<th>{html.escape(v)}</th>" for v in variants if v != "baseline")
+        + "</tr>"
+    )
+    for cid in case_ids:
+        base = cases[cid].get("baseline", {})
+        parts.append(
+            f"<tr><td>{html.escape(cid)}</td>"
+            f"<td>{html.escape(str(base.get('shape', '')))}</td>"
+            f"<td>{html.escape(str(base.get('dtype', '')))}</td>"
+        )
+        for v in variants:
+            if v == "baseline":
+                continue
+            sp = rows_speedup.get((cid, v))
+            st = statuses.get((cid, v), "MISSING")
+            color = speedup_color(sp)
+            txt = f"{sp:.3f}x" if sp else "—"
+            parts.append(
+                f"<td class='sp' style='background:{color}' title='{html.escape(st)}'>"
+                f"{txt}</td>"
+            )
+        parts.append("</tr>")
+    parts.append("</table>")
+    parts.append(
+        "<p class='meta'>着色：绿=KEEP（越深收益越大）；红=REVERT；"
+        "浅灰=噪声带内；深灰=无数据。悬停看结论。</p>"
+    )
+
+    # 逐 case 明细折叠
+    parts.append("<h2>逐 case 对比明细</h2>")
+    for (cid, v), md in sorted(comps.items()):
+        parts.append(
+            f"<details><summary>{html.escape(cid)} × {html.escape(v)}</summary>"
+            + md_to_html(md)
+            + "</details>"
+        )
+
+    parts.append("</body></html>")
+    return "\n".join(parts)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(
+        description="对比矩阵 → 静态 HTML 报告（零依赖单文件）"
+    )
+    ap.add_argument("--matrix-dir", required=True, help="batch_pipeline 的矩阵输出目录")
+    ap.add_argument("--title", default=None, help="报告标题（缺省取矩阵目录名）")
+    ap.add_argument("-o", "--output", required=True, help="HTML 输出路径")
+    args = ap.parse_args()
+
+    matrix_dir = Path(args.matrix_dir)
+    if not matrix_dir.is_dir():
+        print(f"ERROR: {matrix_dir} 不是目录", file=sys.stderr)
+        return 1
+    data = load_matrix(matrix_dir)
+    if not data["cases"]:
+        print("ERROR: 矩阵目录无 case 子目录（先跑 batch_pipeline）", file=sys.stderr)
+        return 1
+
+    title = (
+        args.title
+        or f"OpsTune 调优对比报告 — {matrix_dir.parent.name}/{matrix_dir.name}"
+    )
+    out = Path(args.output)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(render(data, title), encoding="utf-8")
+    n = sum(len(v) - (1 if "baseline" in v else 0) for v in data["cases"].values())
+    print(f"html → {out}（{len(data['cases'])} case × {n} 对比，静态单文件零依赖）")
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())