@hupan56/wlkj 3.3.13 → 3.3.15

package/templates/qoder/scripts/validation/eval/speed_accuracy_flywheel.py

@@ -1,188 +1,188 @@
-"""speed_accuracy_flywheel.py — FW6 又快又准对齐飞轮（准确性 × 速度 × token Pareto）。
-
-GOAL=又快又准。之前只测"准"(P/R/完整性/路由),没测"快"(latency)。本尺子对齐测：
-  - 准确性：grounding 精度 P(反幻觉) + 召回 R
-  - 速度：latency P50/P95（API 代理真测 wall-clock）
-  - token：prompt+completion total
-跨 {weak/mid/strong} × {bare/scaffold}, 搜 wl-search 任务(端到端: 检索+LLM)。
-找 Pareto 最优档（够准的最快最省 token 配置）→ 卖点"弱模型+工作流又快又准"的速度证据。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/speed_accuracy_flywheel.py
-"""
-import os
-import re
-import sys
-import json
-import time
-import statistics
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-from sqlalchemy import text  # noqa: E402
-from app.db import SessionLocal  # noqa: E402
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
-THEMES = ["考勤", "排班", "加班", "车辆", "保险"]  # 5 主题(K=5 量 latency)
-RETRIEVE_K = 45  # FW1 已证 K=45 达标
-ACC_TARGET_P = 0.85  # 准确性目标（精度/召回）
-_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
-_CODE_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
-
-
-def chat_timed(model, prompt, max_tokens=500, timeout=90):
-    """返回 (text, total_tokens, latency_ms)。"""
-    body = json.dumps({"model": model,
-        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)的代码/业务知识助手。只回答真实存在的对象，绝不编造。"},
-                     {"role": "user", "content": prompt}],
-        "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    t0 = time.time()
-    for attempt in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            d = json.loads(urllib.request.urlopen(req, timeout=timeout).read())
-            return d["choices"][0]["message"]["content"], int(d.get("usage", {}).get("total_tokens", 0)), round((time.time() - t0) * 1000)
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if attempt < 2:
-                time.sleep(1.0 * (attempt + 1))
-            else:
-                return f"[ERR {repr(e)[:50]}]", 0, round((time.time() - t0) * 1000)
-
-
-def _q(s, sql, params=None):
-    try:
-        return s.execute(text(sql), params or {}).fetchall()
-    except Exception:
-        try:
-            s.rollback()
-        except Exception:
-            pass
-        return []
-
-
-def load_universe_and_gold():
-    s = SessionLocal()
-    rows = _q(s, "SELECT canonical, cn FROM entities WHERE canonical IS NOT NULL")
-    UNI = set()
-    for c, n in rows:
-        if c: UNI.add(c.strip())
-        if n: UNI.add(n.strip())
-    GOLD = {}
-    for th in THEMES:
-        rs = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) LIMIT 40", {"p": f"%{th}%"})
-        GOLD[th] = [(c, n) for c, n in rs if c]
-    s.close()
-    return UNI, GOLD
-
-
-def retrieve(theme, k=RETRIEVE_K):
-    s = SessionLocal()
-    rs = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) LIMIT :k",
-            {"p": f"%{theme}%", "k": k})
-    s.close()
-    return [(c, n) for c, n in rs if c]
-
-
-def score(ans, gold, UNI):
-    cands = set()
-    for line in ans.splitlines():
-        line = _LINE_CLEAN.sub("", line).strip()
-        if line and len(line) <= 60:
-            m = _CODE_TOK.findall(line)
-            if m:
-                cands.add(m[0])
-    real = {x for x in cands if x in UNI}
-    precision = (len(real) / len(cands)) if cands else 0.0
-    hit = sum(1 for c, n in gold if (c and c in ans) or (n and n in ans))
-    recall = (hit / len(gold)) if gold else 0.0
-    return precision, recall
-
-
-def prompt_bare(theme):
-    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象（表/Java类/接口/按钮等）。\n"
-            f"逐行列出你确信真实存在的名称，每行一个，不要编造。")
-
-
-def prompt_scaffold(theme, ctx):
-    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象。\n"
-            f"知识图谱检索到相关实体（均真实存在）：\n{ctx}\n\n"
-            f"请把检索结果里的实体【尽量全部逐行列出】，每行一个，直接用名称，勿筛选/省略，勿编造。")
-
-
-def pctile(vals, p):
-    if not vals:
-        return 0
-    vals = sorted(vals)
-    k = max(0, min(len(vals) - 1, int(round((p / 100.0) * (len(vals) - 1)))))
-    return vals[k]
-
-
-def run_cell(model, mode, UNI, GOLD):
-    """跑 5 主题，返回 (P, R, p50_lat, p95_lat, avg_tok)。"""
-    Ps, Rs, lats, toks = [], [], [], []
-    for th in THEMES:
-        gold = GOLD[th]
-        if mode == "bare":
-            p = prompt_bare(th)
-        else:
-            ents = retrieve(th)
-            ctx = "\n".join("%s (%s)" % (c, n) for c, n in ents)
-            p = prompt_scaffold(th, ctx)
-        ans, tok, lat = chat_timed(model, p)
-        prec, rec = score(ans, gold, UNI)
-        Ps.append(prec); Rs.append(rec); lats.append(lat); toks.append(tok)
-    n = len(THEMES) or 1
-    return {"P": round(sum(Ps)/n, 3), "R": round(sum(Rs)/n, 3),
-            "p50_lat": pctile(lats, 50), "p95_lat": pctile(lats, 95),
-            "avg_tok": round(sum(toks)/n)}
-
-
-def run():
-    UNI, GOLD = load_universe_and_gold()
-    cells = {}
-    for tier, model in MODELS.items():
-        for mode in ("bare", "scaffold"):
-            c = run_cell(model, mode, UNI, GOLD)
-            cells[f"{tier}/{mode}"] = c
-            print("  %s/%s: P=%.2f R=%.2f p50=%dms p95=%dms tok=%d" % (
-                tier, mode, c["P"], c["R"], c["p50_lat"], c["p95_lat"], c["avg_tok"]))
-    return {"cells": cells, "acc_target": ACC_TARGET_P}
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    c = r["cells"]
-    print("\n## FW6 又快又准对齐矩阵（准确性 P/R × 速度 P50/P95 × token）")
-    print("| 档 | 准 P | 全 R | 速 P50 | 速 P95 | token | 准达标? |")
-    print("|---|---|---|---|---|---|---|")
-    for tier in MODELS:
-        for mode in ("bare", "scaffold"):
-            x = c[f"{tier}/{mode}"]
-            acc_ok = x["P"] >= ACC_TARGET_P and x["R"] >= ACC_TARGET_P
-            print("| %s/%s | %.2f | %.2f | %dms | %dms | %d | %s |" % (
-                tier, mode, x["P"], x["R"], x["p50_lat"], x["p95_lat"], x["avg_tok"], "✓" if acc_ok else "✗"))
-    # Pareto：够准的档里最快最省
-    qual = [(k, v) for k, v in c.items() if v["P"] >= ACC_TARGET_P and v["R"] >= ACC_TARGET_P]
-    print("\n准达标的档:", [k for k, _ in qual] or "无")
-    if qual:
-        knee = min(qual, key=lambda kv: kv[1]["p50_lat"] + kv[1]["avg_tok"] * 0.5)
-        print("Pareto 最优档（准达标里 最快+最省）: %s → P%.2f R%.2f p50=%dms tok=%d" % (
-            knee[0], knee[1]["P"], knee[1]["R"], knee[1]["p50_lat"], knee[1]["avg_tok"]))
-        print("→ 卖点速度证据：弱模型+工作流既准又快又省，无需上强模型（强模型更慢更贵不更准）。")
-
-
-if __name__ == "__main__":
-    main()
+"""speed_accuracy_flywheel.py — FW6 又快又准对齐飞轮（准确性 × 速度 × token Pareto）。
+
+GOAL=又快又准。之前只测"准"(P/R/完整性/路由),没测"快"(latency)。本尺子对齐测：
+  - 准确性：grounding 精度 P(反幻觉) + 召回 R
+  - 速度：latency P50/P95（API 代理真测 wall-clock）
+  - token：prompt+completion total
+跨 {weak/mid/strong} × {bare/scaffold}, 搜 wl-search 任务(端到端: 检索+LLM)。
+找 Pareto 最优档（够准的最快最省 token 配置）→ 卖点"弱模型+工作流又快又准"的速度证据。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/speed_accuracy_flywheel.py
+"""
+import os
+import re
+import sys
+import json
+import time
+import statistics
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from sqlalchemy import text  # noqa: E402
+from app.db import SessionLocal  # noqa: E402
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
+THEMES = ["考勤", "排班", "加班", "车辆", "保险"]  # 5 主题(K=5 量 latency)
+RETRIEVE_K = 45  # FW1 已证 K=45 达标
+ACC_TARGET_P = 0.85  # 准确性目标（精度/召回）
+_LINE_CLEAN = re.compile(r"^[\-\*\d\.\)、\s]+")
+_CODE_TOK = re.compile(r"[A-Za-z_][\w./\-:]{2,}")
+
+
+def chat_timed(model, prompt, max_tokens=500, timeout=90):
+    """返回 (text, total_tokens, latency_ms)。"""
+    body = json.dumps({"model": model,
+        "messages": [{"role": "system", "content": "你是车辆物联网系统(ICS)的代码/业务知识助手。只回答真实存在的对象，绝不编造。"},
+                     {"role": "user", "content": prompt}],
+        "temperature": 0.0, "max_tokens": max_tokens}).encode()
+    t0 = time.time()
+    for attempt in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            d = json.loads(urllib.request.urlopen(req, timeout=timeout).read())
+            return d["choices"][0]["message"]["content"], int(d.get("usage", {}).get("total_tokens", 0)), round((time.time() - t0) * 1000)
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if attempt < 2:
+                time.sleep(1.0 * (attempt + 1))
+            else:
+                return f"[ERR {repr(e)[:50]}]", 0, round((time.time() - t0) * 1000)
+
+
+def _q(s, sql, params=None):
+    try:
+        return s.execute(text(sql), params or {}).fetchall()
+    except Exception:
+        try:
+            s.rollback()
+        except Exception:
+            pass
+        return []
+
+
+def load_universe_and_gold():
+    s = SessionLocal()
+    rows = _q(s, "SELECT canonical, cn FROM entities WHERE canonical IS NOT NULL")
+    UNI = set()
+    for c, n in rows:
+        if c: UNI.add(c.strip())
+        if n: UNI.add(n.strip())
+    GOLD = {}
+    for th in THEMES:
+        rs = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) LIMIT 40", {"p": f"%{th}%"})
+        GOLD[th] = [(c, n) for c, n in rs if c]
+    s.close()
+    return UNI, GOLD
+
+
+def retrieve(theme, k=RETRIEVE_K):
+    s = SessionLocal()
+    rs = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) LIMIT :k",
+            {"p": f"%{theme}%", "k": k})
+    s.close()
+    return [(c, n) for c, n in rs if c]
+
+
+def score(ans, gold, UNI):
+    cands = set()
+    for line in ans.splitlines():
+        line = _LINE_CLEAN.sub("", line).strip()
+        if line and len(line) <= 60:
+            m = _CODE_TOK.findall(line)
+            if m:
+                cands.add(m[0])
+    real = {x for x in cands if x in UNI}
+    precision = (len(real) / len(cands)) if cands else 0.0
+    hit = sum(1 for c, n in gold if (c and c in ans) or (n and n in ans))
+    recall = (hit / len(gold)) if gold else 0.0
+    return precision, recall
+
+
+def prompt_bare(theme):
+    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象（表/Java类/接口/按钮等）。\n"
+            f"逐行列出你确信真实存在的名称，每行一个，不要编造。")
+
+
+def prompt_scaffold(theme, ctx):
+    return (f"在我们的车辆物联网系统(ICS)里，找出和「{theme}」相关的具体对象。\n"
+            f"知识图谱检索到相关实体（均真实存在）：\n{ctx}\n\n"
+            f"请把检索结果里的实体【尽量全部逐行列出】，每行一个，直接用名称，勿筛选/省略，勿编造。")
+
+
+def pctile(vals, p):
+    if not vals:
+        return 0
+    vals = sorted(vals)
+    k = max(0, min(len(vals) - 1, int(round((p / 100.0) * (len(vals) - 1)))))
+    return vals[k]
+
+
+def run_cell(model, mode, UNI, GOLD):
+    """跑 5 主题，返回 (P, R, p50_lat, p95_lat, avg_tok)。"""
+    Ps, Rs, lats, toks = [], [], [], []
+    for th in THEMES:
+        gold = GOLD[th]
+        if mode == "bare":
+            p = prompt_bare(th)
+        else:
+            ents = retrieve(th)
+            ctx = "\n".join("%s (%s)" % (c, n) for c, n in ents)
+            p = prompt_scaffold(th, ctx)
+        ans, tok, lat = chat_timed(model, p)
+        prec, rec = score(ans, gold, UNI)
+        Ps.append(prec); Rs.append(rec); lats.append(lat); toks.append(tok)
+    n = len(THEMES) or 1
+    return {"P": round(sum(Ps)/n, 3), "R": round(sum(Rs)/n, 3),
+            "p50_lat": pctile(lats, 50), "p95_lat": pctile(lats, 95),
+            "avg_tok": round(sum(toks)/n)}
+
+
+def run():
+    UNI, GOLD = load_universe_and_gold()
+    cells = {}
+    for tier, model in MODELS.items():
+        for mode in ("bare", "scaffold"):
+            c = run_cell(model, mode, UNI, GOLD)
+            cells[f"{tier}/{mode}"] = c
+            print("  %s/%s: P=%.2f R=%.2f p50=%dms p95=%dms tok=%d" % (
+                tier, mode, c["P"], c["R"], c["p50_lat"], c["p95_lat"], c["avg_tok"]))
+    return {"cells": cells, "acc_target": ACC_TARGET_P}
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    c = r["cells"]
+    print("\n## FW6 又快又准对齐矩阵（准确性 P/R × 速度 P50/P95 × token）")
+    print("| 档 | 准 P | 全 R | 速 P50 | 速 P95 | token | 准达标? |")
+    print("|---|---|---|---|---|---|---|")
+    for tier in MODELS:
+        for mode in ("bare", "scaffold"):
+            x = c[f"{tier}/{mode}"]
+            acc_ok = x["P"] >= ACC_TARGET_P and x["R"] >= ACC_TARGET_P
+            print("| %s/%s | %.2f | %.2f | %dms | %dms | %d | %s |" % (
+                tier, mode, x["P"], x["R"], x["p50_lat"], x["p95_lat"], x["avg_tok"], "✓" if acc_ok else "✗"))
+    # Pareto：够准的档里最快最省
+    qual = [(k, v) for k, v in c.items() if v["P"] >= ACC_TARGET_P and v["R"] >= ACC_TARGET_P]
+    print("\n准达标的档:", [k for k, _ in qual] or "无")
+    if qual:
+        knee = min(qual, key=lambda kv: kv[1]["p50_lat"] + kv[1]["avg_tok"] * 0.5)
+        print("Pareto 最优档（准达标里 最快+最省）: %s → P%.2f R%.2f p50=%dms tok=%d" % (
+            knee[0], knee[1]["P"], knee[1]["R"], knee[1]["p50_lat"], knee[1]["avg_tok"]))
+        print("→ 卖点速度证据：弱模型+工作流既准又快又省，无需上强模型（强模型更慢更贵不更准）。")
+
+
+if __name__ == "__main__":
+    main()

package/templates/qoder/scripts/validation/eval/task_flywheel.py

@@ -1,124 +1,124 @@
-"""task_flywheel.py — FW4 wl-task 操作类动作路由飞轮。
-
-操作类维度(SPEC §1.1)：成功率(操作成功/总试)/正确性(操作对对象)。
-可代理的是【动作路由正确性】：自然语言任务短语 → 正确动作(create/list/show/start/finish/...)。
-bare("判断该走哪个动作") vs +wl-task dispatcher 表 → 路由准确率。
-成功率(脚本真成功)/归属正确性(不改错队友任务)=执行流→🖥️。
-
-跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/task_flywheel.py
-"""
-import os
-import sys
-import json
-import time
-import urllib.request
-import urllib.error
-
-for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
-    if os.path.isdir(_p) and _p not in sys.path:
-        sys.path.insert(0, _p)
-
-CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
-CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
-MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
-ACTIONS = ["create", "list", "show", "start", "finish", "archive", "rank", "plan", "sync", "zentao"]
-
-# gold 任务短语 → 正确动作（含语义类：无动作词靠意图）
-GOLD = [
-    ("建任务 报价单导出", "create"),
-    ("把R-231报价单导出拆成任务", "create"),
-    ("看任务", "list"),
-    ("接下来干啥", "list"),
-    ("看下登录页怎么样了", "show"),
-    ("开始做登录", "start"),
-    ("做完了", "finish"),
-    ("今天干了3小时", "zentao"),
-    ("归档 老需求", "archive"),
-    ("排优先级", "rank"),
-    ("排期 甘特图", "plan"),
-    ("PRD发禅道", "zentao"),
-    ("同步到禅道", "sync"),
-]
-
-SPEC_OLD = "按用户这句话的意图，判断 /wl-task 该走哪个动作（建/看/启动/完成/归档/排优先级/排期/同步/禅道）。"
-SPEC_NEW = """按 $ARGUMENTS 第一个词/意图确定性选动作（按表查，不要"判断"）：
-- create / "建任务" / "拆成任务" / "把XX建成任务" → create
-- list / "看任务" / "有哪些任务" / "接下来干啥" / "现在要做哪些" → list
-- show / "看下XX" / "XX怎么样了" → show
-- start / "开始做" / "启动XX" → start
-- finish / "做完了" / "完成了" → finish
-- archive / "归档" → archive
-- rank / "排优先级" / "排一下" → rank
-- plan / "排期" / "甘特图" / "排个日期" → plan
-- sync / "同步到禅道" → sync
-- zentao / "PRD发禅道" / "记工时" / "改任务状态" / "今天干了X小时" → zentao"""
-
-
-def chat(model, prompt, max_tokens=30, timeout=60):
-    body = json.dumps({"model": model,
-        "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.0, "max_tokens": max_tokens}).encode()
-    for attempt in range(3):
-        try:
-            req = urllib.request.Request(CHAT_URL, data=body,
-                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
-            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"].strip()
-        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
-            if attempt < 2:
-                time.sleep(1.0 * (attempt + 1))
-            else:
-                return f"[ERR {repr(e)[:50]}]"
-
-
-def _norm(ans):
-    a = ans.lower()
-    for act in ACTIONS:
-        if act in a:
-            return act
-    return "__none__"
-
-
-def run_spec(model, spec, label):
-    correct = 0
-    for query, gold in GOLD:
-        prompt = (f"你是任务管理命令 /wl-task 的路由器。严格按【路由规则】判断这条操作走哪个动作。\n\n"
-                  f"【路由规则】\n{spec}\n\n【操作】{query}\n\n"
-                  f"只输出一个动作名，从这些里选: {', '.join(ACTIONS)}。不要解释。")
-        got = _norm(chat(model, prompt))
-        ok = (got == gold)
-        correct += int(ok)
-        print("    [%s] %-22s gold=%-8s got=%-8s %s" % (label, query[:22], gold, got, "✓" if ok else "✗"))
-    return round(correct / len(GOLD), 3)
-
-
-def run():
-    rep = {"models": MODELS, "n": len(GOLD), "matrix": {}}
-    for tier, model in MODELS.items():
-        old = run_spec(model, SPEC_OLD, "old")
-        new = run_spec(model, SPEC_NEW, "new")
-        rep["matrix"][tier] = {"old_acc": old, "new_acc": new, "gain_pp": round((new - old) * 100, 1)}
-        print("  >> %s(%s): old=%.1f%% new=%.1f%% gain=%+.1fpp" % (tier, model, old*100, new*100, (new-old)*100))
-    return rep
-
-
-def main():
-    import argparse
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--json", action="store_true")
-    args = ap.parse_args()
-    r = run()
-    if args.json:
-        print(json.dumps(r, ensure_ascii=False, indent=2))
-        return
-    m = r["matrix"]
-    print("\n## FW4 wl-task 操作类动作路由飞轮（选对动作准确率）")
-    print("| 模型档 | 旧'判断意图' | +dispatcher 表 | 增益 |")
-    print("|---|---|---|---|")
-    for t in MODELS:
-        c = m[t]
-        print("| %s(%s) | %.1f%% | %.1f%% | %+.1fpp |" % (t, MODELS[t], c["old_acc"]*100, c["new_acc"]*100, c["gain_pp"]))
-    print("\n成功率(脚本真成功)/归属正确性(不改错队友任务) = 执行流维度 → 🖥️ 宿主 wl-task 真跑")
-
-
-if __name__ == "__main__":
-    main()
+"""task_flywheel.py — FW4 wl-task 操作类动作路由飞轮。
+
+操作类维度(SPEC §1.1)：成功率(操作成功/总试)/正确性(操作对对象)。
+可代理的是【动作路由正确性】：自然语言任务短语 → 正确动作(create/list/show/start/finish/...)。
+bare("判断该走哪个动作") vs +wl-task dispatcher 表 → 路由准确率。
+成功率(脚本真成功)/归属正确性(不改错队友任务)=执行流→🖥️。
+
+跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/task_flywheel.py
+"""
+import os
+import sys
+import json
+import time
+import urllib.request
+import urllib.error
+
+for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
+    if os.path.isdir(_p) and _p not in sys.path:
+        sys.path.insert(0, _p)
+
+CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
+CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
+MODELS = {"weak": "qwen-turbo", "mid": "qwen-plus", "strong": "qwen-max"}
+ACTIONS = ["create", "list", "show", "start", "finish", "archive", "rank", "plan", "sync", "zentao"]
+
+# gold 任务短语 → 正确动作（含语义类：无动作词靠意图）
+GOLD = [
+    ("建任务 报价单导出", "create"),
+    ("把R-231报价单导出拆成任务", "create"),
+    ("看任务", "list"),
+    ("接下来干啥", "list"),
+    ("看下登录页怎么样了", "show"),
+    ("开始做登录", "start"),
+    ("做完了", "finish"),
+    ("今天干了3小时", "zentao"),
+    ("归档 老需求", "archive"),
+    ("排优先级", "rank"),
+    ("排期 甘特图", "plan"),
+    ("PRD发禅道", "zentao"),
+    ("同步到禅道", "sync"),
+]
+
+SPEC_OLD = "按用户这句话的意图，判断 /wl-task 该走哪个动作（建/看/启动/完成/归档/排优先级/排期/同步/禅道）。"
+SPEC_NEW = """按 $ARGUMENTS 第一个词/意图确定性选动作（按表查，不要"判断"）：
+- create / "建任务" / "拆成任务" / "把XX建成任务" → create
+- list / "看任务" / "有哪些任务" / "接下来干啥" / "现在要做哪些" → list
+- show / "看下XX" / "XX怎么样了" → show
+- start / "开始做" / "启动XX" → start
+- finish / "做完了" / "完成了" → finish
+- archive / "归档" → archive
+- rank / "排优先级" / "排一下" → rank
+- plan / "排期" / "甘特图" / "排个日期" → plan
+- sync / "同步到禅道" → sync
+- zentao / "PRD发禅道" / "记工时" / "改任务状态" / "今天干了X小时" → zentao"""
+
+
+def chat(model, prompt, max_tokens=30, timeout=60):
+    body = json.dumps({"model": model,
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0, "max_tokens": max_tokens}).encode()
+    for attempt in range(3):
+        try:
+            req = urllib.request.Request(CHAT_URL, data=body,
+                headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
+            return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"].strip()
+        except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
+            if attempt < 2:
+                time.sleep(1.0 * (attempt + 1))
+            else:
+                return f"[ERR {repr(e)[:50]}]"
+
+
+def _norm(ans):
+    a = ans.lower()
+    for act in ACTIONS:
+        if act in a:
+            return act
+    return "__none__"
+
+
+def run_spec(model, spec, label):
+    correct = 0
+    for query, gold in GOLD:
+        prompt = (f"你是任务管理命令 /wl-task 的路由器。严格按【路由规则】判断这条操作走哪个动作。\n\n"
+                  f"【路由规则】\n{spec}\n\n【操作】{query}\n\n"
+                  f"只输出一个动作名，从这些里选: {', '.join(ACTIONS)}。不要解释。")
+        got = _norm(chat(model, prompt))
+        ok = (got == gold)
+        correct += int(ok)
+        print("    [%s] %-22s gold=%-8s got=%-8s %s" % (label, query[:22], gold, got, "✓" if ok else "✗"))
+    return round(correct / len(GOLD), 3)
+
+
+def run():
+    rep = {"models": MODELS, "n": len(GOLD), "matrix": {}}
+    for tier, model in MODELS.items():
+        old = run_spec(model, SPEC_OLD, "old")
+        new = run_spec(model, SPEC_NEW, "new")
+        rep["matrix"][tier] = {"old_acc": old, "new_acc": new, "gain_pp": round((new - old) * 100, 1)}
+        print("  >> %s(%s): old=%.1f%% new=%.1f%% gain=%+.1fpp" % (tier, model, old*100, new*100, (new-old)*100))
+    return rep
+
+
+def main():
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args()
+    r = run()
+    if args.json:
+        print(json.dumps(r, ensure_ascii=False, indent=2))
+        return
+    m = r["matrix"]
+    print("\n## FW4 wl-task 操作类动作路由飞轮（选对动作准确率）")
+    print("| 模型档 | 旧'判断意图' | +dispatcher 表 | 增益 |")
+    print("|---|---|---|---|")
+    for t in MODELS:
+        c = m[t]
+        print("| %s(%s) | %.1f%% | %.1f%% | %+.1fpp |" % (t, MODELS[t], c["old_acc"]*100, c["new_acc"]*100, c["gain_pp"]))
+    print("\n成功率(脚本真成功)/归属正确性(不改错队友任务) = 执行流维度 → 🖥️ 宿主 wl-task 真跑")
+
+
+if __name__ == "__main__":
+    main()