@hupan56/wlkj 3.3.12 → 3.3.14

package/templates/qoder/scripts/validation/eval/dispatcher_ab.py

@@ -1,156 +0,0 @@
-"""dispatcher_ab.py — T1.1 mode dispatcher A/B（飞轮：量→改→重测）。
-
-证"确定性路由表"比"判断问题类型"散文更能让【弱模型】选对检索通道（零猜测）。
-给 {弱/中/强} 模型喂【旧路由 / 新路由】+ 14 条 gold 查询，量选通道准确率。
-
-跑法（容器内）：
-  docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/dispatcher_ab.py
-"""
-import os
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
-CHANNELS = ["get_impact", "db", "coverage_matrix", "ask_corpus", "context_pack", "search_code", "rag_search"]
-
-# 14 条 gold 查询（7 通道 × 2），覆盖确定性路由表的每个信号
-GOLD = [
-    ("改 handleExport 影响谁", "get_impact"),
-    ("波及范围分析 vehicleNo 字段", "get_impact"),
-    ("t_asset 表结构", "db"),
-    ("status 字段有哪些枚举值", "db"),
-    ("哪些功能没测试", "coverage_matrix"),
-    ("测试覆盖盲区在哪", "coverage_matrix"),
-    ("资产管理的业务流程是什么", "ask_corpus"),
-    ("这个功能整体怎么运转", "ask_corpus"),
-    ("盘点考勤有哪些代码和字段", "context_pack"),
-    ("一次取全车辆管理的代码和API", "context_pack"),
-    ("handleExport 在哪定义", "search_code"),
-    ("AppWorkCalendarController 在哪", "search_code"),
-    ("车辆保养", "rag_search"),
-    ("保险异常筛选", "rag_search"),
-]
-
-# 旧路由（"判断问题类型" 散文，原 wl-search 第一步）
-SPEC_OLD = """本命令有 3 路检索，按问题类型选最合适的一路（不是无脑 search_code）：
-- "车辆保养"相关的代码/字段/API（语义找相关）→ rag_search 语义召回
-- "业务流程是什么/整体怎么运转"（全局问答）→ ask_corpus GraphRAG 问答
-- "handleExport 这个符号在哪定义/调用"（精确符号）→ search_code 关键词
-- "XX怎么实现的"（一次取全代码+API+字段）→ context_pack
-- "改 XX 影响谁" → get_impact
-顺序铁律：理解/盘点类先 rag_search；整体流程再补 ask_corpus；精确符号才 search_code。
-（注：数据库表结构/字段/枚举 → 先 list_envs 问环境；哪些没测/覆盖盲区 → coverage_matrix）"""
-
-# 新路由（确定性首匹配表，T1.1 改后）
-SPEC_NEW = """按顺序匹配第一个命中的信号，第一个命中决定通道，不要"判断问题类型"，按表查：
-1. 含"影响谁/改XX影响/波及" → get_impact
-2. 含"表结构/字段/枚举/真实数据/查库" → db（先 list_envs 问环境）
-3. 含"哪些没测/覆盖/盲区/测试矩阵" → coverage_matrix
-4. 含"业务流程/整体怎么运转/是干嘛的" → ask_corpus（再叠 rag_search）
-5. 含"盘点/有哪些代码/一次取全" → context_pack
-6. 精确符号（camelCase/下划线/路径，如 handleExport、/asset） → search_code
-7. 以上都没命中（默认） → rag_search（语义召回最全）"""
-
-
-def chat(model, prompt, max_tokens=40, timeout=60):
-    body = json.dumps({
-        "model": model,
-        "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.0,
-        "max_tokens": max_tokens,
-    }).encode()
-    for attempt in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            resp = urllib.request.urlopen(req, timeout=timeout)
-            return json.loads(resp.read())["choices"][0]["message"]["content"].strip()
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if attempt < 2:
-                time.sleep(1.0 * (attempt + 1))
-            else:
-                return f"[ERR {repr(e)[:60]}]"
-
-
-def _norm(ans):
-    """从短答里抽出通道名（模型可能带解释/引号/中文）。"""
-    a = ans.lower()
-    for ch in CHANNELS:
-        if ch in a:
-            return ch
-    # 容错：db 写成 list_envs / 数据库
-    if "list_envs" in a or "数据库" in ans or "查库" in ans:
-        return "db"
-    return "__none__"
-
-
-def run_spec(model, spec, spec_name, verbose=True):
-    correct = 0
-    details = []
-    for query, gold in GOLD:
-        prompt = (f"你是搜索命令 /wl-search 的路由器。严格根据下面的【路由规则】判断这条查询该用哪个检索通道。\n\n"
-                  f"【路由规则】\n{spec}\n\n"
-                  f"【查询】{query}\n\n"
-                  f"只输出一个通道名，从这些里选: {', '.join(CHANNELS)}。不要解释。")
-        ans = chat(model, prompt)
-        got = _norm(ans)
-        ok = (got == gold)
-        correct += int(ok)
-        details.append({"q": query, "gold": gold, "got": got, "ok": ok, "raw": ans[:30]})
-        if verbose:
-            print(f"    [{spec_name}] {query[:18]:<20} gold={gold:<16} got={got:<16} {'✓' if ok else '✗ '+ans[:25]}")
-    acc = correct / len(GOLD)
-    return {"acc": round(acc, 3), "correct": correct, "n": len(GOLD), "details": details}
-
-
-def run(verbose=True):
-    res = {"models": MODELS, "n": len(GOLD), "matrix": {}}
-    for tier, model in MODELS.items():
-        old = run_spec(model, SPEC_OLD, "old", verbose)
-        new = run_spec(model, SPEC_NEW, "new", verbose)
-        res["matrix"][tier] = {"old_acc": old["acc"], "new_acc": new["acc"],
-                               "gain_pp": round((new["acc"] - old["acc"]) * 100, 1)}
-        print(f"  >> {tier}({model}): old={old['acc']:.2f} new={new['acc']:.2f} "
-              f"gain={res['matrix'][tier]['gain_pp']:+.1f}pp")
-    return res
-
-
-def format_table(res):
-    m = res["matrix"]
-    lines = ["## T1.1 dispatcher A/B（选对通道准确率：旧'判断问题类型' vs 新'确定性表'）", "",
-             "| 模型档 | 旧路由 acc | 新路由 acc | 增益 |",
-             "|---|---|---|---|"]
-    for tier in MODELS:
-        c = m[tier]
-        lines.append(f"| {tier}({MODELS[tier]}) | {c['old_acc']:.1%} | {c['new_acc']:.1%} | {c['gain_pp']:+.1f}pp |")
-    lines.append("")
-    weak_gain = m["weak"]["gain_pp"]
-    weak_new_vs_strong_old = (m["weak"]["new_acc"] - m["strong"]["old_acc"]) * 100
-    lines.append(f"- 弱模型增益: **{weak_gain:+.1f}pp**（确定性表让弱模型选对更多）")
-    lines.append(f"- 弱模型+新路由 vs 强模型+旧路由: **{weak_new_vs_strong_old:+.1f}pp**（>0=弱模型借确定性表追上/反超）")
-    return "\n".join(lines)
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    res = run()
-    if args.json:
-        print(json.dumps(res, ensure_ascii=False, indent=2))
-    else:
-        print("\n" + format_table(res))
-
-
-if __name__ == "__main__":
-    main()

package/templates/qoder/scripts/validation/eval/dispatcher_ab_2026-07-21.json

@@ -1,23 +0,0 @@
-{
-  "task": "T1.1 mode dispatcher 确定性路由 (wl-search)",
-  "date": "2026-07-21",
-  "ruler": "scripts/validation/eval/dispatcher_ab.py",
-  "env": "云 10.89.7.120, DashScope turbo/plus/max, 14 gold 查询(7通道x2)",
-  "change": "wl-search '第一步:先判断问题类型'(纯LLM猜测) -> '确定性路由(按顺序首匹配信号,零猜测)' 7行表+默认rag_search; sync packages->.qoder diff空",
-  "matrix": {
-    "weak":   {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
-    "mid":    {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
-    "strong": {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3}
-  },
-  "old_failures": [
-    "status 字段有哪些枚举值 -> gold=db, got=rag_search (旧表 db 只在括号注)",
-    "盘点考勤有哪些代码和字段 -> gold=context_pack, got=rag_search (旧表'盘点'非明确信号)"
-  ],
-  "insight": "三档同错同对(2错全修)再次印证 T1.0: 结构好时弱模型=强模型; dispatcher 价值在表结构非模型档位",
-  "honest_caveats": [
-    "gold 仅14条 + 作者同时写 gold 和新表, 有小样本/作者重叠风险(新表可能贴 gold)",
-    "2 修复项(枚举->db/盘点->context_pack)确为旧表缺新表补的显式信号, 改进真实",
-    "🖥️ 宿主端到端(弱模型跑真/wl-search 是否真按表走) 待用户验"
-  ],
-  "status": "代码改+sync+自验A/B完成; 🖥️宿主端到端列入 §12 V1.2"
-}

package/templates/qoder/scripts/validation/eval/feature_fidelity_flywheel.py

@@ -1,143 +0,0 @@
-"""feature_fidelity_flywheel.py — FW10 BF5/BF6 feature-scoped 业务保真飞轮。
-
-关键：precision 改成 vs【该 feature 真业务实体集】(非任意真实体)。
-之前 code/search 量"列出的是不是真实体"(可列真但不相关的实体也得分)→ 业务保真虚高。
-BF5/BF6 量"列出的是不是【这个 feature 的】真实体"——真业务相关，非泛实体。
-  - BF6 search feature相关率 = 返回实体 ∈ feature_gold / 返回总数
-  - BF5 code feature精准 = 改实体 ∈ feature_gold / 改总数
-bare(无接地→泛/编造) vs +该feature gold接地。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/feature_fidelity_flywheel.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODEL = "qwen-turbo"
-FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
-_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
-_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
-
-
-def chat(prompt, max_tokens=400, timeout=80):
-    body = json.dumps({"model": MODEL,
-        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)助手，只答真实存在的对象。"},
-                     {"role": "user", "content": prompt}], "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return ""
-
-
-def _q(s, sql, p=None):
-    try: return s.execute(text(sql), p or {}).fetchall()
-    except Exception:
-        try: s.rollback()
-        except Exception: pass
-        return []
-
-
-def feature_gold(theme, k=20):
-    """该 feature 的真业务实体集（BF5/BF6 的 gold）。"""
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
-                "AND type IN ('CONTROLLER','SERVICE','TABLE','COLUMN','FUNCTION') LIMIT :n",
-              {"p": f"%{theme}%", "n": k})
-    s.close()
-    return [(c, n) for c, n in rows if c]
-
-
-def extract_names(ans):
-    out = set()
-    for line in ans.splitlines():
-        line = _LINE_CLEAN.sub("", line).strip()
-        if line and len(line) <= 60:
-            m = _TOK.findall(line)
-            if m: out.add(m[0])
-    return out
-
-
-def feature_precision(ans, gold):
-    """feature 保真：列出的 ∈ 该feature gold 比例（非任意真实体）。"""
-    cands = extract_names(ans)
-    if not cands: return 0.0, 0
-    gold_canon = {c for c, _ in gold}
-    hit = sum(1 for c in cands if c in gold_canon)
-    # 也算 cn 命中（ans 文本含 gold cn）
-    # precision = 命中gold的候选 / 总候选
-    feat_hit = 0
-    for c in cands:
-        if c in gold_canon:
-            feat_hit += 1
-    return round(feat_hit / len(cands), 3), len(cands)
-
-
-def run():
-    rep = {"features": []}
-    for feat, theme in FEATURES:
-        gold = feature_gold(theme, 20)
-        gold_ctx = "\n".join("%s (%s)" % (c, n) for c, n in gold) or "(无)"
-        # BF6 search
-        b_search = chat(f"找 ICS 系统「{feat}」相关的具体对象（类/表/字段/接口），逐行列出，勿编造。")
-        s_search = chat(f"找 ICS 系统「{feat}」相关对象。知识图谱检索到（均真实）：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
-        bP_s, bN_s = feature_precision(b_search, gold)
-        sP_s, sN_s = feature_precision(s_search, gold)
-        # BF5 code
-        b_code = chat(f"给「{feat}」加功能，逐行列出需修改的代码类/方法名，勿编造。")
-        s_code = chat(f"给「{feat}」加功能。get_impact 检索到真实实体：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
-        bP_c, bN_c = feature_precision(b_code, gold)
-        sP_c, sN_c = feature_precision(s_code, gold)
-        rep["features"].append({"feature": feat, "gold_n": len(gold),
-                                "BF6_bare": bP_s, "BF6_scaffold": sP_s,
-                                "BF5_bare": bP_c, "BF5_scaffold": sP_c})
-        print("  %s(gold=%d): BF6 search 相关率 bare=%.0f%%→scaff=%.0f%% | BF5 code 精准 bare=%.0f%%→scaff=%.0f%%" % (
-            feat, len(gold), bP_s*100, sP_s*100, bP_c*100, sP_c*100))
-    n = len(FEATURES) or 1
-    rep["agg"] = {
-        "BF6_bare": round(sum(f["BF6_bare"] for f in rep["features"])/n, 3),
-        "BF6_scaffold": round(sum(f["BF6_scaffold"] for f in rep["features"])/n, 3),
-        "BF5_bare": round(sum(f["BF5_bare"] for f in rep["features"])/n, 3),
-        "BF5_scaffold": round(sum(f["BF5_scaffold"] for f in rep["features"])/n, 3),
-    }
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    a = r["agg"]
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("\n## FW10 BF5/BF6 feature-scoped 业务保真飞轮（列的是【该feature】真实体，非任意真实体）")
-    print("| 指标 | bare | +该feature gold接地 | 目标 |")
-    print("|---|---|---|---|")
-    print("| BF6 search feature相关率 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF6_bare"]*100, a["BF6_scaffold"]*100))
-    print("| BF5 code feature精准 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF5_bare"]*100, a["BF5_scaffold"]*100))
-    b6 = a["BF6_scaffold"] >= 0.85; b5 = a["BF5_scaffold"] >= 0.85
-    print("\nBF6达标?" , "✓" if b6 else "✗", "| BF5达标?", "✓" if b5 else "✗",
-          "| 不达标续转(更强feature接地/拒非feature实体)")
-
-
-if __name__ == "__main__":
-    main()

package/templates/qoder/scripts/validation/eval/gradient_matrix.py

@@ -1,261 +0,0 @@
-"""gradient_matrix.py — T1.0 低模型梯度评测尺子（飞轮 ruler）。
-
-证卖点：工作流的 KG grounding 让【弱模型】不编造、召回真实体，追上强模型裸跑。
-
-矩阵 = {弱/中/强模型} × {裸, +grounding}，每格跨 10 个业务主题算：
-  - grounding 精度(P)  : 模型列出的"具体名称"里有多少是真 KG 实体（反幻觉）
-  - 召回(R)            : gold 实体被答出多少
-  - 幻觉率(H=1-P)      : 编造的占比
-  - token              : 单次总 token
-
-跑法（容器内）：
-  docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/gradient_matrix.py
-  ... --themes 考勤,排班 --json   # 子集/机器可读
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-# 容器内 /app 在 sys.path（from app.db import）
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-# ── 配置 ─────────────────────────────────────────────────────────────
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
-DEFAULT_THEMES = ["考勤", "排班", "加班", "车辆", "报警", "保险", "能耗", "预算", "导出", "审批"]
-RETRIEVE_K = 45  # +grounding 喂的检索条数（FW1: 30→45 覆盖 gold-40，解 R 封顶 30/40=0.75）
-SYS = "你是车辆物联网系统(ICS)的代码/业务知识助手。只回答你确信真实存在的系统对象，绝不编造名称。"
-
-
-def chat(model, prompt, max_tokens=600, timeout=90):
-    """同步 chat，返 (text, total_tokens)。失败重试 3 次。"""
-    body = json.dumps({
-        "model": model,
-        "messages": [{"role": "system", "content": SYS},
-                     {"role": "user", "content": prompt}],
-        "temperature": 0.0,
-        "max_tokens": max_tokens,
-    }).encode()
-    for attempt in range(3):
-        try:
-            req = urllib.request.Request(
-                CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            resp = urllib.request.urlopen(req, timeout=timeout)
-            d = json.loads(resp.read())
-            return d["choices"][0]["message"]["content"], int(d.get("usage", {}).get("total_tokens", 0))
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if attempt < 2:
-                time.sleep(1.0 * (attempt + 1))
-            else:
-                return f"[ERR {repr(e)[:80]}]", 0
-
-
-def load_universe_and_gold(themes):
-    """载入实体名称全集(精度校验) + 每主题 gold 实体(召回校验)。"""
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn FROM entities WHERE canonical IS NOT NULL")
-    UNI = set()
-    for canon, cn in rows:
-        if canon:
-            UNI.add(canon.strip())
-        if cn:
-            UNI.add(cn.strip())
-    GOLD = {}
-    for th in themes:
-        rs = _q(s, "SELECT canonical, cn, type FROM entities "
-                   "WHERE (cn ILIKE :p OR canonical ILIKE :p) LIMIT 40", {"p": f"%{th}%"})
-        GOLD[th] = [{"canonical": c, "cn": n, "type": t} for c, n, t in rs]
-    s.close()
-    return UNI, GOLD
-
-
-def retrieve_entities(theme, k=RETRIEVE_K):
-    """模拟工作流 KG 检索（wl-search 工具会做的事）：返回真实体 dict 列表。"""
-    s = SessionLocal()
-    rs = _q(s, "SELECT canonical, cn, type FROM entities "
-               "WHERE (cn ILIKE :p OR canonical ILIKE :p) LIMIT :k",
-            {"p": f"%{theme}%", "k": k})
-    s.close()
-    return [{"canonical": c, "cn": n, "type": t} for c, n, t in rs]
-
-
-def ctx_from_entities(ents):
-    """把检索实体格式化成喂给 LLM 的上下文文本。"""
-    out = []
-    for e in ents:
-        c, n, t = e["canonical"], e["cn"], e["type"]
-        out.append(f"{t}: {c}" + (f" ({n})" if n else ""))
-    return "\n".join(out)
-
-
-def _q(s, sql, params=None):
-    try:
-        return s.execute(text(sql), params or {}).fetchall()
-    except Exception:
-        try:
-            s.rollback()
-        except Exception:
-            pass
-        return []
-
-
-# ── prompt ───────────────────────────────────────────────────────────
-def prompt_bare(theme):
-    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象"
-            f"（数据库表/Java类/接口/按钮/页面等）。\n"
-            f"请逐行列出你【确信真实存在】的具体名称，每行一个，不要解释，不要编造。"
-            f"不确定就少列。")
-
-
-def prompt_scaffold(theme, ctx):
-    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象"
-            f"（数据库表/Java类/接口/按钮/页面等）。\n"
-            f"知识图谱已检索到以下相关实体（均为真实存在）：\n{ctx}\n\n"
-            f"请把以上检索结果里的实体【尽量全部逐行列出】，每行一个，"
-            f"直接用检索结果里的名称，不要自行筛选/合并/省略，不要解释。检索到几个就列几个。")
-
-
-# ── 评分 ─────────────────────────────────────────────────────────────
-_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
-_CODE_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
-
-
-def _clean_lines(ans):
-    """'每行一个' 答案 → 清洗后的短行列表（模型声称存在的对象，每行一个）。"""
-    out = []
-    for l in ans.splitlines():
-        l = _LINE_CLEAN.sub("", l).strip()
-        if l and len(l) <= 60:
-            out.append(l)
-    return out
-
-
-def score(ans, gold, UNI, ground_cn):
-    """grounding 精度：每行算'真实'当其含某真代码标识符(∈UNI) 或 命中本格 ground_cn(中文)。
-    ground_cn = 该格可作'真实'依据的中文名集合(bare=gold, scaffold=retrieved)，bare/scaffold 同口径公平。
-    """
-    lines = _clean_lines(ans)
-    real_n = 0
-    for l in lines:
-        if set(_CODE_TOK.findall(l)) & UNI:
-            real_n += 1
-        elif any(g and g in l for g in ground_cn):
-            real_n += 1
-    precision = (real_n / len(lines)) if lines else 0.0
-    hit = 0
-    for g in gold:
-        c = g.get("canonical") or ""
-        n = g.get("cn") or ""
-        if (c and c in ans) or (n and n in ans):
-            hit += 1
-    recall = (hit / len(gold)) if gold else 0.0
-    return {
-        "precision": round(precision, 3),
-        "recall": round(recall, 3),
-        "halluc": round(1 - precision, 3),
-        "n_lines": len(lines),
-        "n_real": real_n,
-        "gold_hit": hit,
-        "gold_n": len(gold),
-    }
-
-
-def aggregate(rows):
-    n = len(rows) or 1
-    return {
-        "P": round(sum(r["precision"] for r in rows) / n, 3),
-        "R": round(sum(r["recall"] for r in rows) / n, 3),
-        "H": round(sum(r["halluc"] for r in rows) / n, 3),
-        "tokens": round(sum(r["tokens"] for r in rows) / n, 0),
-        "n": len(rows),
-    }
-
-
-def run(themes, verbose=True):
-    def log(*a):
-        if verbose:
-            print(*a)
-
-    if not CHAT_KEY:
-        raise RuntimeError("缺 DASHSCOPE_API_KEY")
-    log("[ruler] 载入 KG 全集 + gold…")
-    UNI, GOLD = load_universe_and_gold(themes)
-    log(f"[ruler] 实体名称全集 {len(UNI)} 条")
-    for th in themes:
-        log(f"  theme {th}: gold={len(GOLD[th])}")
-
-    matrix = {}
-    t0 = time.time()
-    for tier, model in MODELS.items():
-        for mode in ("bare", "scaffold"):
-            rows = []
-            for th in themes:
-                gold = GOLD[th]
-                if mode == "bare":
-                    ground_cn = {g["canonical"] for g in gold} | {g["cn"] for g in gold if g["cn"]}
-                    p = prompt_bare(th)
-                else:
-                    ents = retrieve_entities(th)
-                    ground_cn = {e["canonical"] for e in ents} | {e["cn"] for e in ents if e["cn"]}
-                    p = prompt_scaffold(th, ctx_from_entities(ents))
-                ans, tok = chat(model, p)
-                sc = score(ans, gold, UNI, ground_cn)
-                sc["tokens"] = tok
-                sc["theme"] = th
-                rows.append(sc)
-                log(f"  [{tier}/{mode}] {th}: P={sc['precision']:.2f} R={sc['recall']:.2f} "
-                    f"H={sc['halluc']:.2f} tok={tok} lines={sc['n_lines']}")
-            matrix[f"{tier}/{mode}"] = aggregate(rows)
-            log(f"[{tier}/{mode}] agg P={matrix[f'{tier}/{mode}']['P']:.3f} "
-                f"R={matrix[f'{tier}/{mode}']['R']:.3f} H={matrix[f'{tier}/{mode}']['H']:.3f} "
-                f"tok={matrix[f'{tier}/{mode}']['tokens']:.0f}")
-    log(f"[ruler] 总耗时 {int(time.time()-t0)}s")
-    return {"matrix": matrix, "themes": themes, "models": MODELS}
-
-
-def format_table(res):
-    m = res["matrix"]
-    lines = ["## 低模型梯度矩阵（grounding 精度P / 召回R / 幻觉H / token）",
-             "", "| 模型档 | 裸 P | 裸 R | 裸 H | 裸tok | +grounding P | +grounding R | +grounding H | +ground tok |",
-             "|---|---|---|---|---|---|---|---|---|"]
-    for tier in MODELS:
-        b = m[f"{tier}/bare"]
-        f = m[f"{tier}/scaffold"]
-        lines.append(f"| {tier}({MODELS[tier]}) | {b['P']:.2f} | {b['R']:.2f} | {b['H']:.2f} | {b['tokens']:.0f} | "
-                     f"{f['P']:.2f} | {f['R']:.2f} | {f['H']:.2f} | {f['tokens']:.0f} |")
-    # 卖点提炼
-    lines.append("")
-    weak_gain_p = m["weak/scaffold"]["P"] - m["weak/bare"]["P"]
-    weak_catchup = m["weak/scaffold"]["P"] - m["strong/bare"]["P"]
-    lines.append(f"- 弱模型 +grounding 精度提升: **{weak_gain_p*100:+.1f}pp**（反幻觉增益）")
-    lines.append(f"- 弱模型+grounding vs 强模型裸 精度差: **{weak_catchup*100:+.1f}pp**（>0=追上/反超=卖点成立）")
-    return "\n".join(lines)
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--themes", default=",".join(DEFAULT_THEMES))
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    themes = [t.strip() for t in args.themes.split(",") if t.strip()]
-    res = run(themes)
-    if args.json:
-        print(json.dumps(res, ensure_ascii=False, indent=2))
-    else:
-        print("\n" + format_table(res))
-
-
-if __name__ == "__main__":
-    main()

package/templates/qoder/scripts/validation/eval/gradient_matrix_baseline_2026-07-21.json

@@ -1,33 +0,0 @@
-{
-  "task": "T1.0 低模型梯度评测基线",
-  "date": "2026-07-21",
-  "ruler": "scripts/validation/eval/gradient_matrix.py",
-  "env": "云 10.89.7.120 容器 wlinkj-workspace-backend-1, DashScope compatible-mode, KG entities=81255 edges=96965",
-  "models": {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"},
-  "themes": ["考勤", "排班", "加班", "车辆", "报警", "保险", "能耗", "预算", "导出", "审批"],
-  "wall_seconds": 138,
-  "chat_calls": 60,
-  "matrix": {
-    "weak/bare":     {"P": 0.02,  "R": 0.00, "H": 0.98, "tokens": 210},
-    "weak/scaffold": {"P": 0.961, "R": 0.815, "H": 0.039, "tokens": 929},
-    "mid/bare":      {"P": 0.00,  "R": 0.00, "H": 1.00, "tokens": 134},
-    "mid/scaffold":  {"P": 0.986, "R": 0.823, "H": 0.014, "tokens": 859},
-    "strong/bare":   {"P": 0.032, "R": 0.00, "H": 0.968, "tokens": 136},
-    "strong/scaffold":{"P": 1.000,"R": 0.796, "H": 0.000, "tokens": 844}
-  },
-  "selling_point_proof": {
-    "all_bare_hallucinate": "三档裸跑 P 0-0.032 / R 0 / H 0.97-1.0 — 连 qwen-max 无 grounding 召回 0",
-    "weak_plus_workflow_vs_strong_bare_pp": 92.9,
-    "tier_converges_with_grounding": "+grounding 后三档 P 0.961/0.986/1.000, R 0.815/0.823/0.796 — 模型档位几乎不影响质量"
-  },
-  "flywheel_round1_gap": {
-    "shortboard": "召回天花板 R≈0.80 (ILIKE top-30 未覆盖全 gold + 模型偶过滤)",
-    "lever": "检索升级: 真实 rag_search 语义 + 1跳图扩展 (映射 T1.4)",
-    "target_R": 0.90
-  },
-  "honest_caveats": [
-    "token 维度本尺子不适用: +grounding 必然加 token(喂上下文换 P 0->0.96); §9.1 的 token<=裸×0.8 指命令层去重/缓存(T1.2/T1.3/T1.7 真命令transcript量)",
-    "🖥️ 宿主真梯度(QoderWork Haiku/Sonnet/Opus × 真命令) 待用户跑 (SPEC §12 V1.1); 本代理用 DashScope turbo/plus/max 同构证卖点方向"
-  ],
-  "status": "尺子就绪 + API 代理基线完成; 🖥️ 宿主梯度列入 §12"
-}