@hupan56/wlkj 3.3.13 → 3.3.15

package/templates/qoder/scripts/validation/eval/metrics_dashboard.py

@@ -1,105 +1,105 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""metrics_dashboard.py - T1.7 通用维度度量汇总（token/反复搜/幻觉/完成率 → §9.1 目标进度）。
-
-聚合 T1.0 gradient_matrix + T1.1 dispatcher_ab 的真实测量数据，输出 §9.1 达标进度表。
-诚实标注：哪些是 API 代理已测（token/幻觉/召回/路由），哪些需宿主真跑（13命令×5任务全表/真 tool_call 计数）。
-
-跑法:
-  python scripts/validation/eval/metrics_dashboard.py
-  python scripts/validation/eval/metrics_dashboard.py --json
-"""
-import json
-import os
-import sys
-
-_HERE = os.path.dirname(os.path.abspath(__file__))
-
-
-def _load(name):
-    p = os.path.join(_HERE, name)
-    try:
-        with open(p, encoding="utf-8") as f:
-            return json.load(f)
-    except Exception:
-        return None
-
-
-def build():
-    grad = _load("gradient_matrix_baseline_2026-07-21.json")
-    disp = _load("dispatcher_ab_2026-07-21.json")
-    rows = []
-    notes = []
-
-    if grad:
-        m = grad["matrix"]
-        # §9.1 幻觉率 <5%：+grounding 后三档都达标；裸跑 ~98% 反证 grounding 必要
-        for tier in ("weak", "mid", "strong"):
-            h_scaffold = m[f"{tier}/scaffold"]["H"]
-            ok = h_scaffold < 0.05
-            rows.append(("幻觉率<5%% (+grounding, %s)" % tier, "%.1f%%" % (h_scaffold * 100),
-                         "✓" if ok else "✗", "代理测(wl-search类)"))
-        # §9.1 低模型梯度：weak+工作流 ≥ strong裸-10pp（精度口径）
-        weak_scaffold_p = m["weak/scaffold"]["P"]
-        strong_bare_p = m["strong/bare"]["P"]
-        diff_pp = (weak_scaffold_p - strong_bare_p) * 100
-        rows.append(("低模型梯度: weak+grounding P vs strong+bare P", "%+.1fpp" % diff_pp,
-                     "✓" if diff_pp >= -10 else "✗", "代理测(卖点铁证 +92.9pp)"))
-        # §9.1 各命令完成率 ≥85%：scaffold 召回 R 作近似
-        for tier in ("weak", "mid", "strong"):
-            r = m[f"{tier}/scaffold"]["R"]
-            ok = r >= 0.85
-            rows.append(("完成率≥85%% (召回R, %s+scaffold)" % tier, "%.1f%%" % (r * 100),
-                         "✓" if ok else "✗(短板~80%)", "代理测; 飞轮杠杆=检索升级"))
-        notes.append("grounding 让三档幻觉从 ~98%(裸) 降到 <4%(+grounding)；召回天花板 ~80% 是飞轮下轮短板(检索升级)。")
-    else:
-        rows.append(("gradient_matrix 基线", "缺", "?", "先跑 gradient_matrix.py"))
-        notes.append("gradient_matrix_baseline_2026-07-21.json 未找到。")
-
-    if disp:
-        dm = disp["matrix"]
-        # T1.1 dispatcher: 新路由准确率（弱模型）
-        weak_new = dm["weak"]["new_acc"]
-        rows.append(("dispatcher 路由准确率 (weak+新表)", "%.1f%%" % (weak_new * 100),
-                     "✓" if weak_new >= 0.95 else "✗", "代理测(T1.1 +14.3pp→100%)"))
-        notes.append("确定性路由表让弱模型选对通道 85.7%→100%（旧'判断问题类型'散文的 2 个系统性误差被修）。")
-    else:
-        rows.append(("dispatcher_ab 基线", "缺", "?", "先跑 dispatcher_ab.py"))
-
-    # §9.1 token ≤裸×0.8 / tool_call ≤裸×0.7：命令层维度，需宿主真 transcript
-    rows.append(("token +工作流≤裸×0.8", "待测", "🖥️", "宿主真命令 transcript (T1.2瘦身/T1.3缓存降)"))
-    rows.append(("反复搜 tool_call +工作流≤裸×0.7", "待测", "🖥️", "宿主 transcript 计 MCP 调用数"))
-    rows.append(("13命令×≥5任务全表", "待测", "🖥️", "宿主 qwork_harness 跑全命令矩阵"))
-    notes.append("token/tool_call/13命令全表 = B类宿主真跑（qwork_harness.py 已备），代理测不了真 agent-loop tool_call 计数。")
-
-    return {"rows": rows, "notes": notes,
-            "targets_source": "SPEC §9.1",
-            "measured_by_proxy": "幻觉率/召回/低模型梯度/dispatcher准确率（gradient_matrix+dispatcher_ab）",
-            "needs_host": "token节省/tool_call计数/13命令全表（qwork_harness 真跑）"}
-
-
-def format_table(r):
-    lines = ["## T1.7 度量汇总 — §9.1 目标进度", "",
-             "| 指标 | 实测 | 达标 | 来源 |", "|---|---|---|---|"]
-    for name, val, ok, src in r["rows"]:
-        lines.append("| %s | %s | %s | %s |" % (name, val, ok, src))
-    lines.append("")
-    lines.append("**洞察**:")
-    for n in r["notes"]:
-        lines.append("- " + n)
-    lines.append("")
-    lines.append("**代理已测**: " + r["measured_by_proxy"])
-    lines.append("**需宿主**: " + r["needs_host"])
-    return "\n".join(lines)
-
-
-def main():
-    r = build()
-    if "--json" in sys.argv:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-    else:
-        print(format_table(r))
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""metrics_dashboard.py - T1.7 通用维度度量汇总（token/反复搜/幻觉/完成率 → §9.1 目标进度）。
+
+聚合 T1.0 gradient_matrix + T1.1 dispatcher_ab 的真实测量数据，输出 §9.1 达标进度表。
+诚实标注：哪些是 API 代理已测（token/幻觉/召回/路由），哪些需宿主真跑（13命令×5任务全表/真 tool_call 计数）。
+
+跑法:
+  python scripts/validation/eval/metrics_dashboard.py
+  python scripts/validation/eval/metrics_dashboard.py --json
+"""
+import json
+import os
+import sys
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+def _load(name):
+    p = os.path.join(_HERE, name)
+    try:
+        with open(p, encoding="utf-8") as f:
+            return json.load(f)
+    except Exception:
+        return None
+
+
+def build():
+    grad = _load("gradient_matrix_baseline_2026-07-21.json")
+    disp = _load("dispatcher_ab_2026-07-21.json")
+    rows = []
+    notes = []
+
+    if grad:
+        m = grad["matrix"]
+        # §9.1 幻觉率 <5%：+grounding 后三档都达标；裸跑 ~98% 反证 grounding 必要
+        for tier in ("weak", "mid", "strong"):
+            h_scaffold = m[f"{tier}/scaffold"]["H"]
+            ok = h_scaffold < 0.05
+            rows.append(("幻觉率<5%% (+grounding, %s)" % tier, "%.1f%%" % (h_scaffold * 100),
+                         "✓" if ok else "✗", "代理测(wl-search类)"))
+        # §9.1 低模型梯度：weak+工作流 ≥ strong裸-10pp（精度口径）
+        weak_scaffold_p = m["weak/scaffold"]["P"]
+        strong_bare_p = m["strong/bare"]["P"]
+        diff_pp = (weak_scaffold_p - strong_bare_p) * 100
+        rows.append(("低模型梯度: weak+grounding P vs strong+bare P", "%+.1fpp" % diff_pp,
+                     "✓" if diff_pp >= -10 else "✗", "代理测(卖点铁证 +92.9pp)"))
+        # §9.1 各命令完成率 ≥85%：scaffold 召回 R 作近似
+        for tier in ("weak", "mid", "strong"):
+            r = m[f"{tier}/scaffold"]["R"]
+            ok = r >= 0.85
+            rows.append(("完成率≥85%% (召回R, %s+scaffold)" % tier, "%.1f%%" % (r * 100),
+                         "✓" if ok else "✗(短板~80%)", "代理测; 飞轮杠杆=检索升级"))
+        notes.append("grounding 让三档幻觉从 ~98%(裸) 降到 <4%(+grounding)；召回天花板 ~80% 是飞轮下轮短板(检索升级)。")
+    else:
+        rows.append(("gradient_matrix 基线", "缺", "?", "先跑 gradient_matrix.py"))
+        notes.append("gradient_matrix_baseline_2026-07-21.json 未找到。")
+
+    if disp:
+        dm = disp["matrix"]
+        # T1.1 dispatcher: 新路由准确率（弱模型）
+        weak_new = dm["weak"]["new_acc"]
+        rows.append(("dispatcher 路由准确率 (weak+新表)", "%.1f%%" % (weak_new * 100),
+                     "✓" if weak_new >= 0.95 else "✗", "代理测(T1.1 +14.3pp→100%)"))
+        notes.append("确定性路由表让弱模型选对通道 85.7%→100%（旧'判断问题类型'散文的 2 个系统性误差被修）。")
+    else:
+        rows.append(("dispatcher_ab 基线", "缺", "?", "先跑 dispatcher_ab.py"))
+
+    # §9.1 token ≤裸×0.8 / tool_call ≤裸×0.7：命令层维度，需宿主真 transcript
+    rows.append(("token +工作流≤裸×0.8", "待测", "🖥️", "宿主真命令 transcript (T1.2瘦身/T1.3缓存降)"))
+    rows.append(("反复搜 tool_call +工作流≤裸×0.7", "待测", "🖥️", "宿主 transcript 计 MCP 调用数"))
+    rows.append(("13命令×≥5任务全表", "待测", "🖥️", "宿主 qwork_harness 跑全命令矩阵"))
+    notes.append("token/tool_call/13命令全表 = B类宿主真跑（qwork_harness.py 已备），代理测不了真 agent-loop tool_call 计数。")
+
+    return {"rows": rows, "notes": notes,
+            "targets_source": "SPEC §9.1",
+            "measured_by_proxy": "幻觉率/召回/低模型梯度/dispatcher准确率（gradient_matrix+dispatcher_ab）",
+            "needs_host": "token节省/tool_call计数/13命令全表（qwork_harness 真跑）"}
+
+
+def format_table(r):
+    lines = ["## T1.7 度量汇总 — §9.1 目标进度", "",
+             "| 指标 | 实测 | 达标 | 来源 |", "|---|---|---|---|"]
+    for name, val, ok, src in r["rows"]:
+        lines.append("| %s | %s | %s | %s |" % (name, val, ok, src))
+    lines.append("")
+    lines.append("**洞察**:")
+    for n in r["notes"]:
+        lines.append("- " + n)
+    lines.append("")
+    lines.append("**代理已测**: " + r["measured_by_proxy"])
+    lines.append("**需宿主**: " + r["needs_host"])
+    return "\n".join(lines)
+
+
+def main():
+    r = build()
+    if "--json" in sys.argv:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+    else:
+        print(format_table(r))
+
+
+if __name__ == "__main__":
+    main()

package/templates/qoder/scripts/validation/eval/multi_turn_flywheel.py

@@ -1,118 +1,118 @@
-"""multi_turn_flywheel.py — FW8 多轮飞轮（§1.2 ⑩ 跨轮上下文复用/准确性）。
-
-多轮场景：turn1「列出 X 相关实体」→ turn2「从上面里选 Controller」。量 turn2 是否正确
-【复用 turn1 结果】(非重查/非编造)。bare(turn2 无 turn1 上下文,盲猜) vs +turn1上下文(复用)。
-准确 = turn2 选的 Controller ∈ turn1 真实 Controller 集。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/multi_turn_flywheel.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODEL = "qwen-turbo"
-THEMES = ["考勤", "保险", "车辆"]
-
-
-def chat(messages, max_tokens=400, timeout=80):
-    body = json.dumps({"model": MODEL, "messages": messages,
-        "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return f"[ERR {repr(e)[:50]}]"
-
-
-def _q(s, sql, p=None):
-    try: return s.execute(text(sql), p or {}).fetchall()
-    except Exception:
-        try: s.rollback()
-        except Exception: pass
-        return []
-
-
-def turn1_entities(theme, k=20):
-    """turn1 检索到的真实实体(模拟工作流 turn1 结果)。返 [(canonical,cn,type)]。"""
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn, type FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
-                "AND type IN ('CONTROLLER','SERVICE','FUNCTION','TABLE') LIMIT :n", {"p": f"%{theme}%", "n": k})
-    s.close()
-    return [(c, n, t) for c, n, t in rows if c]
-
-
-def filter_accuracy(ans, gold_controllers):
-    """turn2 答案是否正确选出 turn1 的 Controller。返 (命中gold数, 编造数)。"""
-    gold_names = {c for c, _, _ in gold_controllers}
-    hit = sum(1 for c in gold_names if c in ans)
-    # 编造 = ans 里的 CamelCase 标识符不在 turn1 全集
-    return hit, len(gold_names)
-
-
-def run():
-    rep = {"themes": []}
-    for th in THEMES:
-        ents = turn1_entities(th, 20)
-        controllers = [e for e in ents if e[2] == "CONTROLLER"]
-        turn1_list = "\n".join("%s (%s) [%s]" % (c, n, t) for c, n, t in ents)
-        turn2_q = "从上面列出的实体里，选出所有类型是 Controller 的，逐行列出名称。"
-
-        # bare: turn2 无 turn1 上下文（盲猜，会重查/编造）
-        bare = chat([{"role": "user", "content": f"列出 ICS 系统「{th}」相关的 Controller 类名。"}])
-        # scaffold: 多轮——turn1 结果作 assistant 上下文，turn2 复用
-        scaff = chat([
-            {"role": "user", "content": f"列出 ICS 系统「{th}」相关的实体。"},
-            {"role": "assistant", "content": "相关实体：\n" + turn1_list},
-            {"role": "user", "content": turn2_q},
-        ])
-        b_hit, gold_n = filter_accuracy(bare, controllers)
-        s_hit, _ = filter_accuracy(scaff, controllers)
-        bR = round(b_hit / max(gold_n, 1), 3)
-        sR = round(s_hit / max(gold_n, 1), 3)
-        rep["themes"].append({"theme": th, "turn1_n": len(ents), "controller_n": gold_n,
-                              "bare_recall": bR, "scaffold_recall": sR})
-        print("  %s: turn1实体=%d(其中Controller=%d) | bare命中Controller=%.2f | +多轮复用=%.2f" % (
-            th, len(ents), gold_n, bR, sR))
-    n = len(THEMES) or 1
-    rep["agg"] = {"bare_recall": round(sum(t["bare_recall"] for t in rep["themes"])/n, 3),
-                  "scaffold_recall": round(sum(t["scaffold_recall"] for t in rep["themes"])/n, 3)}
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    a = r["agg"]
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("\n## FW8 多轮飞轮（turn2 复用 turn1 上下文，选 Controller 召回）")
-    print("| 模式 | Controller 召回 R |")
-    print("|---|---|")
-    print("| bare(turn2 盲猜，无 turn1) | %.1f%% |" % (a["bare_recall"]*100))
-    print("| +多轮(turn1 上下文复用) | %.1f%% |" % (a["scaffold_recall"]*100))
-    met = a["scaffold_recall"] >= 0.85
-    print("\n多轮达标(≥85%%)?" , "✓" if met else "✗ → 续转(更强 turn2 复用指令)")
-
-
-if __name__ == "__main__":
-    main()
+"""multi_turn_flywheel.py — FW8 多轮飞轮（§1.2 ⑩ 跨轮上下文复用/准确性）。
+
+多轮场景：turn1「列出 X 相关实体」→ turn2「从上面里选 Controller」。量 turn2 是否正确
+【复用 turn1 结果】(非重查/非编造)。bare(turn2 无 turn1 上下文,盲猜) vs +turn1上下文(复用)。
+准确 = turn2 选的 Controller ∈ turn1 真实 Controller 集。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/multi_turn_flywheel.py
+"""
+import os
+import re
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from sqlalchemy import text  # noqa: E402
+from app.db import SessionLocal  # noqa: E402
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODEL = "qwen-turbo"
+THEMES = ["考勤", "保险", "车辆"]
+
+
+def chat(messages, max_tokens=400, timeout=80):
+    body = json.dumps({"model": MODEL, "messages": messages,
+        "temperature": 0.0, "max_tokens": max_tokens}).encode()
+    for a in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if a < 2: time.sleep(1.0*(a+1))
+            else: return f"[ERR {repr(e)[:50]}]"
+
+
+def _q(s, sql, p=None):
+    try: return s.execute(text(sql), p or {}).fetchall()
+    except Exception:
+        try: s.rollback()
+        except Exception: pass
+        return []
+
+
+def turn1_entities(theme, k=20):
+    """turn1 检索到的真实实体(模拟工作流 turn1 结果)。返 [(canonical,cn,type)]。"""
+    s = SessionLocal()
+    rows = _q(s, "SELECT canonical, cn, type FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
+                "AND type IN ('CONTROLLER','SERVICE','FUNCTION','TABLE') LIMIT :n", {"p": f"%{theme}%", "n": k})
+    s.close()
+    return [(c, n, t) for c, n, t in rows if c]
+
+
+def filter_accuracy(ans, gold_controllers):
+    """turn2 答案是否正确选出 turn1 的 Controller。返 (命中gold数, 编造数)。"""
+    gold_names = {c for c, _, _ in gold_controllers}
+    hit = sum(1 for c in gold_names if c in ans)
+    # 编造 = ans 里的 CamelCase 标识符不在 turn1 全集
+    return hit, len(gold_names)
+
+
+def run():
+    rep = {"themes": []}
+    for th in THEMES:
+        ents = turn1_entities(th, 20)
+        controllers = [e for e in ents if e[2] == "CONTROLLER"]
+        turn1_list = "\n".join("%s (%s) [%s]" % (c, n, t) for c, n, t in ents)
+        turn2_q = "从上面列出的实体里，选出所有类型是 Controller 的，逐行列出名称。"
+
+        # bare: turn2 无 turn1 上下文（盲猜，会重查/编造）
+        bare = chat([{"role": "user", "content": f"列出 ICS 系统「{th}」相关的 Controller 类名。"}])
+        # scaffold: 多轮——turn1 结果作 assistant 上下文，turn2 复用
+        scaff = chat([
+            {"role": "user", "content": f"列出 ICS 系统「{th}」相关的实体。"},
+            {"role": "assistant", "content": "相关实体：\n" + turn1_list},
+            {"role": "user", "content": turn2_q},
+        ])
+        b_hit, gold_n = filter_accuracy(bare, controllers)
+        s_hit, _ = filter_accuracy(scaff, controllers)
+        bR = round(b_hit / max(gold_n, 1), 3)
+        sR = round(s_hit / max(gold_n, 1), 3)
+        rep["themes"].append({"theme": th, "turn1_n": len(ents), "controller_n": gold_n,
+                              "bare_recall": bR, "scaffold_recall": sR})
+        print("  %s: turn1实体=%d(其中Controller=%d) | bare命中Controller=%.2f | +多轮复用=%.2f" % (
+            th, len(ents), gold_n, bR, sR))
+    n = len(THEMES) or 1
+    rep["agg"] = {"bare_recall": round(sum(t["bare_recall"] for t in rep["themes"])/n, 3),
+                  "scaffold_recall": round(sum(t["scaffold_recall"] for t in rep["themes"])/n, 3)}
+    return rep
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    a = r["agg"]
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    print("\n## FW8 多轮飞轮（turn2 复用 turn1 上下文，选 Controller 召回）")
+    print("| 模式 | Controller 召回 R |")
+    print("|---|---|")
+    print("| bare(turn2 盲猜，无 turn1) | %.1f%% |" % (a["bare_recall"]*100))
+    print("| +多轮(turn1 上下文复用) | %.1f%% |" % (a["scaffold_recall"]*100))
+    met = a["scaffold_recall"] >= 0.85
+    print("\n多轮达标(≥85%%)?" , "✓" if met else "✗ → 续转(更强 turn2 复用指令)")
+
+
+if __name__ == "__main__":
+    main()