@hupan56/wlkj 3.3.13 → 3.3.15

package/templates/qoder/scripts/validation/eval/dispatcher_ab_2026-07-21.json

@@ -1,23 +1,23 @@
-{
-  "task": "T1.1 mode dispatcher 确定性路由 (wl-search)",
-  "date": "2026-07-21",
-  "ruler": "scripts/validation/eval/dispatcher_ab.py",
-  "env": "云 10.89.7.120, DashScope turbo/plus/max, 14 gold 查询(7通道x2)",
-  "change": "wl-search '第一步:先判断问题类型'(纯LLM猜测) -> '确定性路由(按顺序首匹配信号,零猜测)' 7行表+默认rag_search; sync packages->.qoder diff空",
-  "matrix": {
-    "weak":   {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
-    "mid":    {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
-    "strong": {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3}
-  },
-  "old_failures": [
-    "status 字段有哪些枚举值 -> gold=db, got=rag_search (旧表 db 只在括号注)",
-    "盘点考勤有哪些代码和字段 -> gold=context_pack, got=rag_search (旧表'盘点'非明确信号)"
-  ],
-  "insight": "三档同错同对(2错全修)再次印证 T1.0: 结构好时弱模型=强模型; dispatcher 价值在表结构非模型档位",
-  "honest_caveats": [
-    "gold 仅14条 + 作者同时写 gold 和新表, 有小样本/作者重叠风险(新表可能贴 gold)",
-    "2 修复项(枚举->db/盘点->context_pack)确为旧表缺新表补的显式信号, 改进真实",
-    "🖥️ 宿主端到端(弱模型跑真/wl-search 是否真按表走) 待用户验"
-  ],
-  "status": "代码改+sync+自验A/B完成; 🖥️宿主端到端列入 §12 V1.2"
-}
+{
+  "task": "T1.1 mode dispatcher 确定性路由 (wl-search)",
+  "date": "2026-07-21",
+  "ruler": "scripts/validation/eval/dispatcher_ab.py",
+  "env": "云 10.89.7.120, DashScope turbo/plus/max, 14 gold 查询(7通道x2)",
+  "change": "wl-search '第一步:先判断问题类型'(纯LLM猜测) -> '确定性路由(按顺序首匹配信号,零猜测)' 7行表+默认rag_search; sync packages->.qoder diff空",
+  "matrix": {
+    "weak":   {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
+    "mid":    {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3},
+    "strong": {"old_acc": 0.857, "new_acc": 1.0, "gain_pp": 14.3}
+  },
+  "old_failures": [
+    "status 字段有哪些枚举值 -> gold=db, got=rag_search (旧表 db 只在括号注)",
+    "盘点考勤有哪些代码和字段 -> gold=context_pack, got=rag_search (旧表'盘点'非明确信号)"
+  ],
+  "insight": "三档同错同对(2错全修)再次印证 T1.0: 结构好时弱模型=强模型; dispatcher 价值在表结构非模型档位",
+  "honest_caveats": [
+    "gold 仅14条 + 作者同时写 gold 和新表, 有小样本/作者重叠风险(新表可能贴 gold)",
+    "2 修复项(枚举->db/盘点->context_pack)确为旧表缺新表补的显式信号, 改进真实",
+    "🖥️ 宿主端到端(弱模型跑真/wl-search 是否真按表走) 待用户验"
+  ],
+  "status": "代码改+sync+自验A/B完成; 🖥️宿主端到端列入 §12 V1.2"
+}

package/templates/qoder/scripts/validation/eval/feature_fidelity_flywheel.py

@@ -1,143 +1,143 @@
-"""feature_fidelity_flywheel.py — FW10 BF5/BF6 feature-scoped 业务保真飞轮。
-
-关键：precision 改成 vs【该 feature 真业务实体集】(非任意真实体)。
-之前 code/search 量"列出的是不是真实体"(可列真但不相关的实体也得分)→ 业务保真虚高。
-BF5/BF6 量"列出的是不是【这个 feature 的】真实体"——真业务相关，非泛实体。
-  - BF6 search feature相关率 = 返回实体 ∈ feature_gold / 返回总数
-  - BF5 code feature精准 = 改实体 ∈ feature_gold / 改总数
-bare(无接地→泛/编造) vs +该feature gold接地。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/feature_fidelity_flywheel.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODEL = "qwen-turbo"
-FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
-_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
-_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
-
-
-def chat(prompt, max_tokens=400, timeout=80):
-    body = json.dumps({"model": MODEL,
-        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)助手，只答真实存在的对象。"},
-                     {"role": "user", "content": prompt}], "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return ""
-
-
-def _q(s, sql, p=None):
-    try: return s.execute(text(sql), p or {}).fetchall()
-    except Exception:
-        try: s.rollback()
-        except Exception: pass
-        return []
-
-
-def feature_gold(theme, k=20):
-    """该 feature 的真业务实体集（BF5/BF6 的 gold）。"""
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
-                "AND type IN ('CONTROLLER','SERVICE','TABLE','COLUMN','FUNCTION') LIMIT :n",
-              {"p": f"%{theme}%", "n": k})
-    s.close()
-    return [(c, n) for c, n in rows if c]
-
-
-def extract_names(ans):
-    out = set()
-    for line in ans.splitlines():
-        line = _LINE_CLEAN.sub("", line).strip()
-        if line and len(line) <= 60:
-            m = _TOK.findall(line)
-            if m: out.add(m[0])
-    return out
-
-
-def feature_precision(ans, gold):
-    """feature 保真：列出的 ∈ 该feature gold 比例（非任意真实体）。"""
-    cands = extract_names(ans)
-    if not cands: return 0.0, 0
-    gold_canon = {c for c, _ in gold}
-    hit = sum(1 for c in cands if c in gold_canon)
-    # 也算 cn 命中（ans 文本含 gold cn）
-    # precision = 命中gold的候选 / 总候选
-    feat_hit = 0
-    for c in cands:
-        if c in gold_canon:
-            feat_hit += 1
-    return round(feat_hit / len(cands), 3), len(cands)
-
-
-def run():
-    rep = {"features": []}
-    for feat, theme in FEATURES:
-        gold = feature_gold(theme, 20)
-        gold_ctx = "\n".join("%s (%s)" % (c, n) for c, n in gold) or "(无)"
-        # BF6 search
-        b_search = chat(f"找 ICS 系统「{feat}」相关的具体对象（类/表/字段/接口），逐行列出，勿编造。")
-        s_search = chat(f"找 ICS 系统「{feat}」相关对象。知识图谱检索到（均真实）：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
-        bP_s, bN_s = feature_precision(b_search, gold)
-        sP_s, sN_s = feature_precision(s_search, gold)
-        # BF5 code
-        b_code = chat(f"给「{feat}」加功能，逐行列出需修改的代码类/方法名，勿编造。")
-        s_code = chat(f"给「{feat}」加功能。get_impact 检索到真实实体：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
-        bP_c, bN_c = feature_precision(b_code, gold)
-        sP_c, sN_c = feature_precision(s_code, gold)
-        rep["features"].append({"feature": feat, "gold_n": len(gold),
-                                "BF6_bare": bP_s, "BF6_scaffold": sP_s,
-                                "BF5_bare": bP_c, "BF5_scaffold": sP_c})
-        print("  %s(gold=%d): BF6 search 相关率 bare=%.0f%%→scaff=%.0f%% | BF5 code 精准 bare=%.0f%%→scaff=%.0f%%" % (
-            feat, len(gold), bP_s*100, sP_s*100, bP_c*100, sP_c*100))
-    n = len(FEATURES) or 1
-    rep["agg"] = {
-        "BF6_bare": round(sum(f["BF6_bare"] for f in rep["features"])/n, 3),
-        "BF6_scaffold": round(sum(f["BF6_scaffold"] for f in rep["features"])/n, 3),
-        "BF5_bare": round(sum(f["BF5_bare"] for f in rep["features"])/n, 3),
-        "BF5_scaffold": round(sum(f["BF5_scaffold"] for f in rep["features"])/n, 3),
-    }
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    a = r["agg"]
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("\n## FW10 BF5/BF6 feature-scoped 业务保真飞轮（列的是【该feature】真实体，非任意真实体）")
-    print("| 指标 | bare | +该feature gold接地 | 目标 |")
-    print("|---|---|---|---|")
-    print("| BF6 search feature相关率 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF6_bare"]*100, a["BF6_scaffold"]*100))
-    print("| BF5 code feature精准 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF5_bare"]*100, a["BF5_scaffold"]*100))
-    b6 = a["BF6_scaffold"] >= 0.85; b5 = a["BF5_scaffold"] >= 0.85
-    print("\nBF6达标?" , "✓" if b6 else "✗", "| BF5达标?", "✓" if b5 else "✗",
-          "| 不达标续转(更强feature接地/拒非feature实体)")
-
-
-if __name__ == "__main__":
-    main()
+"""feature_fidelity_flywheel.py — FW10 BF5/BF6 feature-scoped 业务保真飞轮。
+
+关键：precision 改成 vs【该 feature 真业务实体集】(非任意真实体)。
+之前 code/search 量"列出的是不是真实体"(可列真但不相关的实体也得分)→ 业务保真虚高。
+BF5/BF6 量"列出的是不是【这个 feature 的】真实体"——真业务相关，非泛实体。
+  - BF6 search feature相关率 = 返回实体 ∈ feature_gold / 返回总数
+  - BF5 code feature精准 = 改实体 ∈ feature_gold / 改总数
+bare(无接地→泛/编造) vs +该feature gold接地。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/feature_fidelity_flywheel.py
+"""
+import os
+import re
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from sqlalchemy import text  # noqa: E402
+from app.db import SessionLocal  # noqa: E402
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODEL = "qwen-turbo"
+FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
+_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
+_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
+
+
+def chat(prompt, max_tokens=400, timeout=80):
+    body = json.dumps({"model": MODEL,
+        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)助手，只答真实存在的对象。"},
+                     {"role": "user", "content": prompt}], "temperature": 0.0, "max_tokens": max_tokens}).encode()
+    for a in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if a < 2: time.sleep(1.0*(a+1))
+            else: return ""
+
+
+def _q(s, sql, p=None):
+    try: return s.execute(text(sql), p or {}).fetchall()
+    except Exception:
+        try: s.rollback()
+        except Exception: pass
+        return []
+
+
+def feature_gold(theme, k=20):
+    """该 feature 的真业务实体集（BF5/BF6 的 gold）。"""
+    s = SessionLocal()
+    rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
+                "AND type IN ('CONTROLLER','SERVICE','TABLE','COLUMN','FUNCTION') LIMIT :n",
+              {"p": f"%{theme}%", "n": k})
+    s.close()
+    return [(c, n) for c, n in rows if c]
+
+
+def extract_names(ans):
+    out = set()
+    for line in ans.splitlines():
+        line = _LINE_CLEAN.sub("", line).strip()
+        if line and len(line) <= 60:
+            m = _TOK.findall(line)
+            if m: out.add(m[0])
+    return out
+
+
+def feature_precision(ans, gold):
+    """feature 保真：列出的 ∈ 该feature gold 比例（非任意真实体）。"""
+    cands = extract_names(ans)
+    if not cands: return 0.0, 0
+    gold_canon = {c for c, _ in gold}
+    hit = sum(1 for c in cands if c in gold_canon)
+    # 也算 cn 命中（ans 文本含 gold cn）
+    # precision = 命中gold的候选 / 总候选
+    feat_hit = 0
+    for c in cands:
+        if c in gold_canon:
+            feat_hit += 1
+    return round(feat_hit / len(cands), 3), len(cands)
+
+
+def run():
+    rep = {"features": []}
+    for feat, theme in FEATURES:
+        gold = feature_gold(theme, 20)
+        gold_ctx = "\n".join("%s (%s)" % (c, n) for c, n in gold) or "(无)"
+        # BF6 search
+        b_search = chat(f"找 ICS 系统「{feat}」相关的具体对象（类/表/字段/接口），逐行列出，勿编造。")
+        s_search = chat(f"找 ICS 系统「{feat}」相关对象。知识图谱检索到（均真实）：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
+        bP_s, bN_s = feature_precision(b_search, gold)
+        sP_s, sN_s = feature_precision(s_search, gold)
+        # BF5 code
+        b_code = chat(f"给「{feat}」加功能，逐行列出需修改的代码类/方法名，勿编造。")
+        s_code = chat(f"给「{feat}」加功能。get_impact 检索到真实实体：\n{gold_ctx}\n请把检索结果实体全部逐行列出，直接用名称，勿编造。")
+        bP_c, bN_c = feature_precision(b_code, gold)
+        sP_c, sN_c = feature_precision(s_code, gold)
+        rep["features"].append({"feature": feat, "gold_n": len(gold),
+                                "BF6_bare": bP_s, "BF6_scaffold": sP_s,
+                                "BF5_bare": bP_c, "BF5_scaffold": sP_c})
+        print("  %s(gold=%d): BF6 search 相关率 bare=%.0f%%→scaff=%.0f%% | BF5 code 精准 bare=%.0f%%→scaff=%.0f%%" % (
+            feat, len(gold), bP_s*100, sP_s*100, bP_c*100, sP_c*100))
+    n = len(FEATURES) or 1
+    rep["agg"] = {
+        "BF6_bare": round(sum(f["BF6_bare"] for f in rep["features"])/n, 3),
+        "BF6_scaffold": round(sum(f["BF6_scaffold"] for f in rep["features"])/n, 3),
+        "BF5_bare": round(sum(f["BF5_bare"] for f in rep["features"])/n, 3),
+        "BF5_scaffold": round(sum(f["BF5_scaffold"] for f in rep["features"])/n, 3),
+    }
+    return rep
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    a = r["agg"]
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    print("\n## FW10 BF5/BF6 feature-scoped 业务保真飞轮（列的是【该feature】真实体，非任意真实体）")
+    print("| 指标 | bare | +该feature gold接地 | 目标 |")
+    print("|---|---|---|---|")
+    print("| BF6 search feature相关率 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF6_bare"]*100, a["BF6_scaffold"]*100))
+    print("| BF5 code feature精准 | %.0f%% | %.0f%% | ≥85%% |" % (a["BF5_bare"]*100, a["BF5_scaffold"]*100))
+    b6 = a["BF6_scaffold"] >= 0.85; b5 = a["BF5_scaffold"] >= 0.85
+    print("\nBF6达标?" , "✓" if b6 else "✗", "| BF5达标?", "✓" if b5 else "✗",
+          "| 不达标续转(更强feature接地/拒非feature实体)")
+
+
+if __name__ == "__main__":
+    main()