@hupan56/wlkj 3.3.13 → 3.3.15

package/templates/qoder/scripts/validation/eval/bf2_content_fidelity.py

@@ -1,110 +1,110 @@
-"""bf2_content_fidelity.py — FW10 BF2 PRD 内容保真（生成PRD正文 vs 真实 ICS PRD 语义相似）。
-
-BF2 = 生成 PRD 正文 → rag_search(kind='prd') 找最近真 PRD chunk → 相似度。
-高=生成内容是真业务域(车辆/保险等,贴近真PRD正文)；低=泛泛/编造(不贴近真PRD)。
-bare(无接地) vs +schema接地。复用 rag_search 打分。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/bf2_content_fidelity.py
-"""
-import os
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from app.routers import mcp_handlers as H  # noqa: E402
-try:
-    H._init_handlers()
-except Exception:
-    pass
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODEL = "qwen-turbo"
-FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
-
-
-def chat(prompt, max_tokens=800, timeout=90):
-    body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.2, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return ""
-
-
-def content_fidelity(prd_text):
-    """生成 PRD 正文 → rag_search(kind=prd) 最近真 PRD chunk 平均分。"""
-    if not prd_text or len(prd_text) < 20:
-        return 0.0, 0
-    # 用 PRD 正文做语义查询（截断前 500 字，避免过长）
-    q = prd_text[:500].replace("\n", " ")
-    r = H.dispatch("rag_search", {"query": q, "kind": "prd", "top_k": 3}) or {}
-    items = r.get("items") or []
-    scores = [float(it.get("score", 0)) for it in items if it.get("score") is not None]
-    avg = round(sum(scores) / len(scores), 3) if scores else 0.0
-    return avg, len(items)
-
-
-def real_prd_template(theme):
-    """取一条真 ICS PRD 正文 chunk 做 house-style 写作范本（BF2 杠杆）。"""
-    r = H.dispatch("rag_search", {"query": theme, "kind": "prd", "top_k": 1}) or {}
-    items = r.get("items") or []
-    if items:
-        return (items[0].get("text") or "")[:600]
-    return ""
-
-
-def run():
-    rep = {"features": []}
-    for feat, theme in FEATURES:
-        tpl = real_prd_template(theme)
-        bare = chat(f"你是产品经理。为「{feat}」写一份 PRD 需求文档（含背景/功能/字段/流程）。直接输出正文。")
-        # BF2 杠杆：喂真 PRD 正文做写作范本（学 ICS house style/系统级深度，非泛产品文）
-        scaff = chat(f"你是 ICS(车辆物联网)产品经理。为「{feat}」写 PRD。\n"
-                     f"【写作范本接地】以下是真实 ICS PRD 的正文（学它的写法/系统级字段级深度/业务术语，"
-                     f"不要泛泛产品宣传文）：\n{tpl}\n\n参考这个 house style 为「{feat}」写 PRD，"
-                     f"字段/流程贴合 {theme} 真业务。直接输出正文。") if tpl else chat(
-            f"为「{feat}」写 PRD，贴合 {theme} 真业务。直接输出。")
-        b_score, b_n = content_fidelity(bare)
-        s_score, s_n = content_fidelity(scaff)
-        rep["features"].append({"feature": feat, "bare_score": b_score, "scaffold_score": s_score,
-                                "bare_hits": b_n, "scaffold_hits": s_n, "has_template": bool(tpl)})
-        print("  %s: 内容保真 bare=%.3f → +真PRD范本=%.3f (范本%d字)" % (
-            feat, b_score, s_score, len(tpl)))
-    n = len(FEATURES) or 1
-    rep["agg"] = {"bare": round(sum(f["bare_score"] for f in rep["features"])/n, 3),
-                  "scaffold": round(sum(f["scaffold_score"] for f in rep["features"])/n, 3)}
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    a = r["agg"]
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("\n## FW10 BF2 PRD 内容保真（生成PRD正文 vs 真实ICS PRD 语义相似度）")
-    print("| 模式 | 内容保真(语义cos,越接近真PRD越高) |")
-    print("|---|---|")
-    print("| bare(无业务接地) | %.3f |" % a["bare"])
-    print("| +真业务接地 | %.3f |" % a["scaffold"])
-    print("\n注：分数是 rag_search(kind=prd) 的 top-3 平均相似度（0-1）。"
-          "高=生成内容贴近真实 ICS PRD 正文（真业务域）；低=泛泛/偏离。")
-
-
-if __name__ == "__main__":
-    main()
+"""bf2_content_fidelity.py — FW10 BF2 PRD 内容保真（生成PRD正文 vs 真实 ICS PRD 语义相似）。
+
+BF2 = 生成 PRD 正文 → rag_search(kind='prd') 找最近真 PRD chunk → 相似度。
+高=生成内容是真业务域(车辆/保险等,贴近真PRD正文)；低=泛泛/编造(不贴近真PRD)。
+bare(无接地) vs +schema接地。复用 rag_search 打分。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/bf2_content_fidelity.py
+"""
+import os
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from app.routers import mcp_handlers as H  # noqa: E402
+try:
+    H._init_handlers()
+except Exception:
+    pass
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODEL = "qwen-turbo"
+FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
+
+
+def chat(prompt, max_tokens=800, timeout=90):
+    body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.2, "max_tokens": max_tokens}).encode()
+    for a in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if a < 2: time.sleep(1.0*(a+1))
+            else: return ""
+
+
+def content_fidelity(prd_text):
+    """生成 PRD 正文 → rag_search(kind=prd) 最近真 PRD chunk 平均分。"""
+    if not prd_text or len(prd_text) < 20:
+        return 0.0, 0
+    # 用 PRD 正文做语义查询（截断前 500 字，避免过长）
+    q = prd_text[:500].replace("\n", " ")
+    r = H.dispatch("rag_search", {"query": q, "kind": "prd", "top_k": 3}) or {}
+    items = r.get("items") or []
+    scores = [float(it.get("score", 0)) for it in items if it.get("score") is not None]
+    avg = round(sum(scores) / len(scores), 3) if scores else 0.0
+    return avg, len(items)
+
+
+def real_prd_template(theme):
+    """取一条真 ICS PRD 正文 chunk 做 house-style 写作范本（BF2 杠杆）。"""
+    r = H.dispatch("rag_search", {"query": theme, "kind": "prd", "top_k": 1}) or {}
+    items = r.get("items") or []
+    if items:
+        return (items[0].get("text") or "")[:600]
+    return ""
+
+
+def run():
+    rep = {"features": []}
+    for feat, theme in FEATURES:
+        tpl = real_prd_template(theme)
+        bare = chat(f"你是产品经理。为「{feat}」写一份 PRD 需求文档（含背景/功能/字段/流程）。直接输出正文。")
+        # BF2 杠杆：喂真 PRD 正文做写作范本（学 ICS house style/系统级深度，非泛产品文）
+        scaff = chat(f"你是 ICS(车辆物联网)产品经理。为「{feat}」写 PRD。\n"
+                     f"【写作范本接地】以下是真实 ICS PRD 的正文（学它的写法/系统级字段级深度/业务术语，"
+                     f"不要泛泛产品宣传文）：\n{tpl}\n\n参考这个 house style 为「{feat}」写 PRD，"
+                     f"字段/流程贴合 {theme} 真业务。直接输出正文。") if tpl else chat(
+            f"为「{feat}」写 PRD，贴合 {theme} 真业务。直接输出。")
+        b_score, b_n = content_fidelity(bare)
+        s_score, s_n = content_fidelity(scaff)
+        rep["features"].append({"feature": feat, "bare_score": b_score, "scaffold_score": s_score,
+                                "bare_hits": b_n, "scaffold_hits": s_n, "has_template": bool(tpl)})
+        print("  %s: 内容保真 bare=%.3f → +真PRD范本=%.3f (范本%d字)" % (
+            feat, b_score, s_score, len(tpl)))
+    n = len(FEATURES) or 1
+    rep["agg"] = {"bare": round(sum(f["bare_score"] for f in rep["features"])/n, 3),
+                  "scaffold": round(sum(f["scaffold_score"] for f in rep["features"])/n, 3)}
+    return rep
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    a = r["agg"]
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    print("\n## FW10 BF2 PRD 内容保真（生成PRD正文 vs 真实ICS PRD 语义相似度）")
+    print("| 模式 | 内容保真(语义cos,越接近真PRD越高) |")
+    print("|---|---|")
+    print("| bare(无业务接地) | %.3f |" % a["bare"])
+    print("| +真业务接地 | %.3f |" % a["scaffold"])
+    print("\n注：分数是 rag_search(kind=prd) 的 top-3 平均相似度（0-1）。"
+          "高=生成内容贴近真实 ICS PRD 正文（真业务域）；低=泛泛/偏离。")
+
+
+if __name__ == "__main__":
+    main()

package/templates/qoder/scripts/validation/eval/bf2_llmjudge.py

@@ -1,104 +1,104 @@
-"""bf2_llmjudge.py — FW10 BF2 换度量：LLM-judge 量 PRD 内容业务保真（余弦不分好坏，换强模型裁判）。
-
-BF2 余弦~0.03 不分好坏→换 LLM-judge：qwen-max 对照真 ICS PRD 范本，给生成 PRD 的【业务保真度】
-打 1-5 分（5=完全 ICS 业务写法 系统级/字段级/真流程；1=通用产品宣传文）。能区分 bare(泛) vs +范本(ICS)。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/bf2_llmjudge.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from app.routers import mcp_handlers as H  # noqa: E402
-try:
-    H._init_handlers()
-except Exception:
-    pass
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-GEN_MODEL = "qwen-turbo"   # 生成（弱模型，卖点）
-JUDGE_MODEL = "qwen-max"   # 裁判（强模型）
-FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
-
-
-def chat(model, prompt, max_tokens=800, timeout=90):
-    body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.0 if model == JUDGE_MODEL else 0.2, "max_tokens": max_tokens}).encode()
-    for a in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if a < 2: time.sleep(1.0*(a+1))
-            else: return ""
-
-
-def real_prd(theme):
-    r = H.dispatch("rag_search", {"query": theme, "kind": "prd", "top_k": 1}) or {}
-    items = r.get("items") or []
-    return (items[0].get("text") or "")[:600] if items else ""
-
-
-def judge(prd, ref, feat):
-    """qwen-max 打业务保真分 1-5。"""
-    p = ("你是 ICS（车辆物联网系统）资深产品专家。对照下面【真实 ICS PRD 范本】，给这份【生成 PRD】"
-         "的业务保真度打分 1-5：\n"
-         "5=完全 ICS 业务写法（系统级/字段级深度、真实业务流程、车辆物联网术语）\n"
-         "3=部分 ICS（有业务要素但有泛化）\n"
-         "1=通用产品宣传文（泛泛、非 ICS 真业务）\n\n"
-         f"【真实 ICS PRD 范本】：\n{ref}\n\n"
-         f"【生成 PRD（{feat}）】：\n{prd[:1000]}\n\n"
-         f"只输出一个整数分（1-5），不要解释。")
-    out = chat(JUDGE_MODEL, p, max_tokens=10)
-    m = re.search(r"[1-5]", out)
-    return int(m.group()) if m else 0
-
-
-def run():
-    rep = {"features": []}
-    for feat, theme in FEATURES:
-        ref = real_prd(theme)
-        bare = chat(GEN_MODEL, f"你是产品经理。为「{feat}」写一份 PRD 需求文档。直接输出正文。")
-        scaff = (chat(GEN_MODEL, f"你是 ICS 产品经理。为「{feat}」写 PRD。\n"
-                     f"【写作范本】真 ICS PRD 正文（学其系统级/字段级写法，勿泛产品文）：\n{ref}\n直接输出 PRD 正文。")
-                 if ref else bare)
-        b = judge(bare, ref, feat)
-        s = judge(scaff, ref, feat)
-        rep["features"].append({"feature": feat, "bare_score": b, "scaffold_score": s})
-        print("  %s: 业务保真分 bare=%d/5 → +真PRD范本=%d/5" % (feat, b, s))
-    n = len(FEATURES) or 1
-    rep["agg"] = {"bare": round(sum(f["bare_score"] for f in rep["features"])/n, 2),
-                  "scaffold": round(sum(f["scaffold_score"] for f in rep["features"])/n, 2)}
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    a = r["agg"]
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    print("\n## FW10 BF2（换 LLM-judge 度量）PRD 内容业务保真")
-    print("| 模式 | 业务保真分(/5, qwen-max 裁判 vs 真 PRD) |")
-    print("|---|---|")
-    print("| bare(通用) | %.2f |" % a["bare"])
-    print("| +真PRD范本(ICS house style) | %.2f |" % a["scaffold"])
-    print("\n目标≥4/5。LLM-judge 能区分（余弦不能）：bare 通用文低分，+范本 ICS 写法高分。")
-
-
-if __name__ == "__main__":
-    main()
+"""bf2_llmjudge.py — FW10 BF2 换度量：LLM-judge 量 PRD 内容业务保真（余弦不分好坏，换强模型裁判）。
+
+BF2 余弦~0.03 不分好坏→换 LLM-judge：qwen-max 对照真 ICS PRD 范本，给生成 PRD 的【业务保真度】
+打 1-5 分（5=完全 ICS 业务写法 系统级/字段级/真流程；1=通用产品宣传文）。能区分 bare(泛) vs +范本(ICS)。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/bf2_llmjudge.py
+"""
+import os
+import re
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from app.routers import mcp_handlers as H  # noqa: E402
+try:
+    H._init_handlers()
+except Exception:
+    pass
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+GEN_MODEL = "qwen-turbo"   # 生成（弱模型，卖点）
+JUDGE_MODEL = "qwen-max"   # 裁判（强模型）
+FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
+
+
+def chat(model, prompt, max_tokens=800, timeout=90):
+    body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0 if model == JUDGE_MODEL else 0.2, "max_tokens": max_tokens}).encode()
+    for a in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if a < 2: time.sleep(1.0*(a+1))
+            else: return ""
+
+
+def real_prd(theme):
+    r = H.dispatch("rag_search", {"query": theme, "kind": "prd", "top_k": 1}) or {}
+    items = r.get("items") or []
+    return (items[0].get("text") or "")[:600] if items else ""
+
+
+def judge(prd, ref, feat):
+    """qwen-max 打业务保真分 1-5。"""
+    p = ("你是 ICS（车辆物联网系统）资深产品专家。对照下面【真实 ICS PRD 范本】，给这份【生成 PRD】"
+         "的业务保真度打分 1-5：\n"
+         "5=完全 ICS 业务写法（系统级/字段级深度、真实业务流程、车辆物联网术语）\n"
+         "3=部分 ICS（有业务要素但有泛化）\n"
+         "1=通用产品宣传文（泛泛、非 ICS 真业务）\n\n"
+         f"【真实 ICS PRD 范本】：\n{ref}\n\n"
+         f"【生成 PRD（{feat}）】：\n{prd[:1000]}\n\n"
+         f"只输出一个整数分（1-5），不要解释。")
+    out = chat(JUDGE_MODEL, p, max_tokens=10)
+    m = re.search(r"[1-5]", out)
+    return int(m.group()) if m else 0
+
+
+def run():
+    rep = {"features": []}
+    for feat, theme in FEATURES:
+        ref = real_prd(theme)
+        bare = chat(GEN_MODEL, f"你是产品经理。为「{feat}」写一份 PRD 需求文档。直接输出正文。")
+        scaff = (chat(GEN_MODEL, f"你是 ICS 产品经理。为「{feat}」写 PRD。\n"
+                     f"【写作范本】真 ICS PRD 正文（学其系统级/字段级写法，勿泛产品文）：\n{ref}\n直接输出 PRD 正文。")
+                 if ref else bare)
+        b = judge(bare, ref, feat)
+        s = judge(scaff, ref, feat)
+        rep["features"].append({"feature": feat, "bare_score": b, "scaffold_score": s})
+        print("  %s: 业务保真分 bare=%d/5 → +真PRD范本=%d/5" % (feat, b, s))
+    n = len(FEATURES) or 1
+    rep["agg"] = {"bare": round(sum(f["bare_score"] for f in rep["features"])/n, 2),
+                  "scaffold": round(sum(f["scaffold_score"] for f in rep["features"])/n, 2)}
+    return rep
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    a = r["agg"]
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    print("\n## FW10 BF2（换 LLM-judge 度量）PRD 内容业务保真")
+    print("| 模式 | 业务保真分(/5, qwen-max 裁判 vs 真 PRD) |")
+    print("|---|---|")
+    print("| bare(通用) | %.2f |" % a["bare"])
+    print("| +真PRD范本(ICS house style) | %.2f |" % a["scaffold"])
+    print("\n目标≥4/5。LLM-judge 能区分（余弦不能）：bare 通用文低分，+范本 ICS 写法高分。")
+
+
+if __name__ == "__main__":
+    main()