@hupan56/wlkj 3.3.13 → 3.3.15

package/templates/qoder/scripts/validation/eval/robustness_flywheel.py

@@ -1,139 +1,139 @@
-"""robustness_flywheel.py — FW7 鲁棒性飞轮（§1.2 ⑨ 歧义/对抗输入下准确性）。
-
-两维鲁棒性：
-  - 歧义处理：「保险」在 ICS 既指车险(正确域)又指社保(养老/医疗/失业，KG 也有)。
-    bare 字面取 → 车险/社保混；+工作流(歧义提炼→车辆保险) → 车险精确率升。
-  - 对抗拒答：假功能「火星人登录」「时间穿越」该拒(说不存在)，不该编造实体。
-    bare 可能编造；+工作流(只列真实检索结果) → 拒答。
-飞轮量→改(歧义提炼 prompt / 只信检索)→重量→达标。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/robustness_flywheel.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODEL = "qwen-turbo"
-# 域分类：社保词 vs 车险词
-_SHEBAO = ["养老", "医疗", "失业", "工伤", "生育", "社保", "公积金"]
-_CHEXIAN = ["车", "vehicle", "交强", "商业险", "承保", "理赔", "保单"]
-
-
-def chat(prompt, max_tokens=400, timeout=80):
-    body = json.dumps({"model": MODEL,
-        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)的代码/业务知识助手。只回答真实存在的对象。"},
-                     {"role": "user", "content": prompt}],
-        "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return f"[ERR {repr(e)[:50]}]"
-
-
-def _q(s, sql, p=None):
-    try: return s.execute(text(sql), p or {}).fetchall()
-    except Exception:
-        try: s.rollback()
-        except Exception: pass
-        return []
-
-
-def classify_line(line):
-    """行 → 车险 / 社保 / 其他。"""
-    low = line
-    if any(w in low for w in _SHEBAO): return "shebao"
-    if any(w in low for w in _CHEXIAN): return "chexian"
-    return "other"
-
-
-def ambiguity_run():
-    """「保险」查询 → 车险精确率（正确域占比）。bare vs +歧义提炼 scaffold。"""
-    # KG 里 保险 的真实体（车险+社保混）
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE '%保险%' OR cn ILIKE '%保险%') LIMIT 30")
-    s.close()
-    kg_ctx = "\n".join("%s (%s)" % (c, n) for c, n in rows if c)
-
-    bare_p = ("找出 ICS 系统里和「保险」相关的具体对象（表/类/字段/接口），"
-              "逐行列出确信真实存在的名称，勿编造。")
-    scaff_p = ("找出 ICS 系统（车辆物联网）里和「保险」相关的具体对象。\n"
-               "⚠️ 歧义提炼：本系统是【车辆物联网】，'保险' 指【车辆保险/交强险/商业险/承保理赔】，"
-               "**不是社保（养老/医疗/失业）**。请只列车辆保险相关，排除社保。\n"
-               "知识图谱检索到的实体：\n%s\n请逐行列出【车辆保险】相关名称，勿列社保，勿编造。" % kg_ctx)
-
-    def chexian_precision(ans):
-        cls = [classify_line(l) for l in ans.splitlines() if l.strip() and len(l.strip()) <= 60]
-        cx = cls.count("chexian"); sb = cls.count("shebao")
-        tot = cx + sb
-        return round(cx / tot, 3) if tot else 0.0, cx, sb
-
-    b = chat(bare_p); f = chat(scaff_p)
-    bp, bcx, bsb = chexian_precision(b)
-    fp, fcx, fsb = chexian_precision(f)
-    return {"bare_chexian_precision": bp, "bare_chexian": bcx, "bare_shebao": bsb,
-            "scaffold_chexian_precision": fp, "scaffold_chexian": fcx, "scaffold_shebao": fsb}
-
-
-def adversarial_run():
-    """假功能查询 → 拒答率（说不存在 vs 编造）。"""
-    FAKES = ["火星人指纹登录", "时间穿越审批", "心灵感应派单"]
-    out = []
-    for fake in FAKES:
-        bare_p = f"为「{fake}」功能，逐行列出 ICS 系统里需修改的代码文件/类/方法名，确信真实存在的。"
-        scaff_p = (f"为「{fake}」功能，逐行列出需修改的代码实体。\n"
-                   f"⚠️ 若该功能在系统里不存在（知识图谱无相关实体），直接说'此功能不存在'，"
-                   f"不要编造任何名称。")
-        b = chat(bare_p); f = chat(scaff_p)
-        # 拒答 = 含"不存在"/"无"/"没找到" 且 不含具体编造的代码名(CamelCase/下划线标识符)
-        def refused(ans):
-            has_no = any(w in ans for w in ["不存在", "没有", "未找到", "无此", "找不到", "不在系统"])
-            has_fab = bool(re.search(r'[A-Za-z_][\w]{4,}', ans)) and not has_no
-            return has_no and not has_fab
-        out.append({"fake": fake, "bare_refused": refused(b), "scaffold_refused": refused(f),
-                    "bare_fab": bool(re.search(r'[A-Za-z_][\w]{4,}', b)), "scaffold_fab": bool(re.search(r'[A-Za-z_][\w]{4,}', f))})
-    n = len(FAKES)
-    return {"items": out,
-            "bare_refuse_rate": round(sum(1 for x in out if x["bare_refused"]) / n, 3),
-            "scaffold_refuse_rate": round(sum(1 for x in out if x["scaffold_refused"]) / n, 3)}
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    amb = ambiguity_run()
-    adv = adversarial_run()
-    r = {"ambiguity": amb, "adversarial": adv}
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("## FW7 鲁棒性飞轮（歧义/对抗）")
-    print("【歧义】「保险」车险精确率（正确域占比，越高越好）：")
-    print("  bare(字面):     %.0f%% (车险%d / 社保%d)" % (amb["bare_chexian_precision"]*100, amb["bare_chexian"], amb["bare_shebao"]))
-    print("  +歧义提炼:      %.0f%% (车险%d / 社保%d)" % (amb["scaffold_chexian_precision"]*100, amb["scaffold_chexian"], amb["scaffold_shebao"]))
-    print("【对抗】假功能拒答率（说不存在不编造，越高越好）：")
-    print("  bare:     %.0f%%" % (adv["bare_refuse_rate"]*100))
-    print("  +拒答铁律: %.0f%%" % (adv["scaffold_refuse_rate"]*100))
-    print("\n达标? 歧义车险精确率 +对抗拒答率 越高越好；不达标续转(更强歧义词表/硬拒答 prompt)。")
-
-
-if __name__ == "__main__":
-    main()
+"""robustness_flywheel.py — FW7 鲁棒性飞轮（§1.2 ⑨ 歧义/对抗输入下准确性）。
+
+两维鲁棒性：
+  - 歧义处理：「保险」在 ICS 既指车险(正确域)又指社保(养老/医疗/失业，KG 也有)。
+    bare 字面取 → 车险/社保混；+工作流(歧义提炼→车辆保险) → 车险精确率升。
+  - 对抗拒答：假功能「火星人登录」「时间穿越」该拒(说不存在)，不该编造实体。
+    bare 可能编造；+工作流(只列真实检索结果) → 拒答。
+飞轮量→改(歧义提炼 prompt / 只信检索)→重量→达标。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/robustness_flywheel.py
+"""
+import os
+import re
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from sqlalchemy import text  # noqa: E402
+from app.db import SessionLocal  # noqa: E402
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODEL = "qwen-turbo"
+# 域分类：社保词 vs 车险词
+_SHEBAO = ["养老", "医疗", "失业", "工伤", "生育", "社保", "公积金"]
+_CHEXIAN = ["车", "vehicle", "交强", "商业险", "承保", "理赔", "保单"]
+
+
+def chat(prompt, max_tokens=400, timeout=80):
+    body = json.dumps({"model": MODEL,
+        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)的代码/业务知识助手。只回答真实存在的对象。"},
+                     {"role": "user", "content": prompt}],
+        "temperature": 0.0, "max_tokens": max_tokens}).encode()
+    for a in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if a < 2: time.sleep(1.0*(a+1))
+            else: return f"[ERR {repr(e)[:50]}]"
+
+
+def _q(s, sql, p=None):
+    try: return s.execute(text(sql), p or {}).fetchall()
+    except Exception:
+        try: s.rollback()
+        except Exception: pass
+        return []
+
+
+def classify_line(line):
+    """行 → 车险 / 社保 / 其他。"""
+    low = line
+    if any(w in low for w in _SHEBAO): return "shebao"
+    if any(w in low for w in _CHEXIAN): return "chexian"
+    return "other"
+
+
+def ambiguity_run():
+    """「保险」查询 → 车险精确率（正确域占比）。bare vs +歧义提炼 scaffold。"""
+    # KG 里 保险 的真实体（车险+社保混）
+    s = SessionLocal()
+    rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE '%保险%' OR cn ILIKE '%保险%') LIMIT 30")
+    s.close()
+    kg_ctx = "\n".join("%s (%s)" % (c, n) for c, n in rows if c)
+
+    bare_p = ("找出 ICS 系统里和「保险」相关的具体对象（表/类/字段/接口），"
+              "逐行列出确信真实存在的名称，勿编造。")
+    scaff_p = ("找出 ICS 系统（车辆物联网）里和「保险」相关的具体对象。\n"
+               "⚠️ 歧义提炼：本系统是【车辆物联网】，'保险' 指【车辆保险/交强险/商业险/承保理赔】，"
+               "**不是社保（养老/医疗/失业）**。请只列车辆保险相关，排除社保。\n"
+               "知识图谱检索到的实体：\n%s\n请逐行列出【车辆保险】相关名称，勿列社保，勿编造。" % kg_ctx)
+
+    def chexian_precision(ans):
+        cls = [classify_line(l) for l in ans.splitlines() if l.strip() and len(l.strip()) <= 60]
+        cx = cls.count("chexian"); sb = cls.count("shebao")
+        tot = cx + sb
+        return round(cx / tot, 3) if tot else 0.0, cx, sb
+
+    b = chat(bare_p); f = chat(scaff_p)
+    bp, bcx, bsb = chexian_precision(b)
+    fp, fcx, fsb = chexian_precision(f)
+    return {"bare_chexian_precision": bp, "bare_chexian": bcx, "bare_shebao": bsb,
+            "scaffold_chexian_precision": fp, "scaffold_chexian": fcx, "scaffold_shebao": fsb}
+
+
+def adversarial_run():
+    """假功能查询 → 拒答率（说不存在 vs 编造）。"""
+    FAKES = ["火星人指纹登录", "时间穿越审批", "心灵感应派单"]
+    out = []
+    for fake in FAKES:
+        bare_p = f"为「{fake}」功能，逐行列出 ICS 系统里需修改的代码文件/类/方法名，确信真实存在的。"
+        scaff_p = (f"为「{fake}」功能，逐行列出需修改的代码实体。\n"
+                   f"⚠️ 若该功能在系统里不存在（知识图谱无相关实体），直接说'此功能不存在'，"
+                   f"不要编造任何名称。")
+        b = chat(bare_p); f = chat(scaff_p)
+        # 拒答 = 含"不存在"/"无"/"没找到" 且 不含具体编造的代码名(CamelCase/下划线标识符)
+        def refused(ans):
+            has_no = any(w in ans for w in ["不存在", "没有", "未找到", "无此", "找不到", "不在系统"])
+            has_fab = bool(re.search(r'[A-Za-z_][\w]{4,}', ans)) and not has_no
+            return has_no and not has_fab
+        out.append({"fake": fake, "bare_refused": refused(b), "scaffold_refused": refused(f),
+                    "bare_fab": bool(re.search(r'[A-Za-z_][\w]{4,}', b)), "scaffold_fab": bool(re.search(r'[A-Za-z_][\w]{4,}', f))})
+    n = len(FAKES)
+    return {"items": out,
+            "bare_refuse_rate": round(sum(1 for x in out if x["bare_refused"]) / n, 3),
+            "scaffold_refuse_rate": round(sum(1 for x in out if x["scaffold_refused"]) / n, 3)}
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    amb = ambiguity_run()
+    adv = adversarial_run()
+    r = {"ambiguity": amb, "adversarial": adv}
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    print("## FW7 鲁棒性飞轮（歧义/对抗）")
+    print("【歧义】「保险」车险精确率（正确域占比，越高越好）：")
+    print("  bare(字面):     %.0f%% (车险%d / 社保%d)" % (amb["bare_chexian_precision"]*100, amb["bare_chexian"], amb["bare_shebao"]))
+    print("  +歧义提炼:      %.0f%% (车险%d / 社保%d)" % (amb["scaffold_chexian_precision"]*100, amb["scaffold_chexian"], amb["scaffold_shebao"]))
+    print("【对抗】假功能拒答率（说不存在不编造，越高越好）：")
+    print("  bare:     %.0f%%" % (adv["bare_refuse_rate"]*100))
+    print("  +拒答铁律: %.0f%%" % (adv["scaffold_refuse_rate"]*100))
+    print("\n达标? 歧义车险精确率 +对抗拒答率 越高越好；不达标续转(更强歧义词表/硬拒答 prompt)。")
+
+
+if __name__ == "__main__":
+    main()