@hupan56/wlkj 3.3.13 → 3.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +0 -7
- package/package.json +29 -29
- package/templates/qoder/commands/optional/wl-spec.md +0 -8
- package/templates/qoder/commands/optional/wl-status.md +0 -8
- package/templates/qoder/commands/wl-code.md +0 -8
- package/templates/qoder/commands/wl-commit.md +0 -8
- package/templates/qoder/commands/wl-init.md +129 -129
- package/templates/qoder/commands/wl-search.md +0 -8
- package/templates/qoder/commands/wl-task.md +0 -8
- package/templates/qoder/commands/wl-test.md +0 -8
- package/templates/qoder/config.yaml +2 -3
- package/templates/qoder/contracts/insight.md +55 -55
- package/templates/qoder/hooks/pre-tool-use-commit.py +124 -124
- package/templates/qoder/hooks/session-start.py +384 -384
- package/templates/qoder/hooks/stop-eval.py +383 -383
- package/templates/qoder/scripts/capability/adapters/mcp.py +322 -322
- package/templates/qoder/scripts/capability/registry.py +245 -245
- package/templates/qoder/scripts/deployment/setup/install_qoderwork.py +14 -0
- package/templates/qoder/scripts/deployment/setup/setup.py +0 -11
- package/templates/qoder/scripts/domain/task/wlkj_panel.py +1503 -1503
- package/templates/qoder/scripts/foundation/bootstrap.py +145 -156
- package/templates/qoder/scripts/foundation/io/context_cache.py +94 -94
- package/templates/qoder/scripts/orchestration/wlkj.py +295 -306
- package/templates/qoder/scripts/tool_guide.md +70 -70
- package/templates/qoder/scripts/validation/eval/alignment_matrix.py +176 -176
- package/templates/qoder/scripts/validation/eval/bf2_content_fidelity.py +110 -110
- package/templates/qoder/scripts/validation/eval/bf2_llmjudge.py +104 -104
- package/templates/qoder/scripts/validation/eval/bf_score.py +218 -218
- package/templates/qoder/scripts/validation/eval/code_flywheel.py +150 -150
- package/templates/qoder/scripts/validation/eval/dispatcher_ab.py +156 -156
- package/templates/qoder/scripts/validation/eval/dispatcher_ab_2026-07-21.json +23 -23
- package/templates/qoder/scripts/validation/eval/feature_fidelity_flywheel.py +143 -143
- package/templates/qoder/scripts/validation/eval/gradient_matrix.py +261 -261
- package/templates/qoder/scripts/validation/eval/gradient_matrix_baseline_2026-07-21.json +33 -33
- package/templates/qoder/scripts/validation/eval/metrics_dashboard.py +105 -105
- package/templates/qoder/scripts/validation/eval/multi_turn_flywheel.py +118 -118
- package/templates/qoder/scripts/validation/eval/prd_fidelity_flywheel.py +128 -128
- package/templates/qoder/scripts/validation/eval/prd_flywheel.py +148 -148
- package/templates/qoder/scripts/validation/eval/prototype_fidelity_flywheel.py +166 -166
- package/templates/qoder/scripts/validation/eval/recall_flywheel.py +148 -148
- package/templates/qoder/scripts/validation/eval/robustness_flywheel.py +139 -139
- package/templates/qoder/scripts/validation/eval/speed_accuracy_flywheel.py +188 -188
- package/templates/qoder/scripts/validation/eval/task_flywheel.py +124 -124
- package/templates/qoder/scripts/validation/eval/token_flywheel.py +88 -88
- package/templates/qoder/scripts/validation/metrics/eval_code_ac.py +177 -177
- package/templates/qoder/scripts/validation/metrics/lint_cases.py +170 -170
- package/templates/qoder/scripts/validation/test/test_context_cache.py +78 -78
- package/templates/qoder/scripts/validation/test/test_lint_cases.py +60 -60
- package/templates/qoder/scripts/validation/test/test_pre_tool_use_commit.py +70 -70
- package/templates/qoder/commands/wl-data.md +0 -46
- package/templates/qoder/commands/wl-fix.md +0 -47
- package/templates/qoder/commands/wl-knowledge.md +0 -41
- package/templates/qoder/commands/wl-review.md +0 -49
- package/templates/workspace/specs/REQ-TEST-CTX/.context-cache.json +0 -1
|
@@ -1,128 +1,128 @@
|
|
|
1
|
-
"""prd_fidelity_flywheel.py — FW9 PRD 业务保真度飞轮(vs 真实 ICS schema,非结构代理)。
|
|
2
|
-
|
|
3
|
-
用户根因:之前 FW2 量的是"6 章齐"(结构),不是"PRD 是不是这个业务的"(保真)。
|
|
4
|
-
本尺子量 **schema 保真**:PRD 引用的表/列是不是 ICS 真实业务表列(db_tables/db_columns),
|
|
5
|
-
而非泛泛/编造。bare(无 schema grounding→编造表名) vs +真schema grounding(喂真表真列)。
|
|
6
|
-
保真率 = PRD 提的表/列 ∈ 真实 schema 的比例;召回 = 该功能真表被提的比例。
|
|
7
|
-
|
|
8
|
-
(原型颜色保真=eval_prd A2 读宿主真色板,云 KG 无设计系统表→那格 🖥️ 宿主;本尺子量可代理的 schema 保真)
|
|
9
|
-
|
|
10
|
-
跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/prd_fidelity_flywheel.py
|
|
11
|
-
"""
|
|
12
|
-
import os
|
|
13
|
-
import re
|
|
14
|
-
import sys
|
|
15
|
-
import json
|
|
16
|
-
import time
|
|
17
|
-
import urllib.request
|
|
18
|
-
import urllib.error
|
|
19
|
-
|
|
20
|
-
for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
|
|
21
|
-
if os.path.isdir(_p) and _p not in sys.path:
|
|
22
|
-
sys.path.insert(0, _p)
|
|
23
|
-
|
|
24
|
-
from sqlalchemy import text # noqa: E402
|
|
25
|
-
from app.db import SessionLocal # noqa: E402
|
|
26
|
-
|
|
27
|
-
CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
|
|
28
|
-
CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
|
|
29
|
-
MODEL = "qwen-turbo"
|
|
30
|
-
FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
|
|
31
|
-
# 表名 token(t_xxx / tb_xxx / report_xxx 等)
|
|
32
|
-
_TABLE_RE = re.compile(r'\b(t_[a-z_]+|tb_[a-z_]+|report_[a-z_]+|salary_[a-z_]+|[a-z]+_[a-z_]+_table)\b', re.IGNORECASE)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def chat(prompt, max_tokens=900, timeout=90):
|
|
36
|
-
body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
|
|
37
|
-
"temperature": 0.2, "max_tokens": max_tokens}).encode()
|
|
38
|
-
for a in range(3):
|
|
39
|
-
try:
|
|
40
|
-
req = urllib.request.Request(CHAT_URL, data=body,
|
|
41
|
-
headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
|
|
42
|
-
return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
|
|
43
|
-
except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
|
|
44
|
-
if a < 2: time.sleep(1.0*(a+1))
|
|
45
|
-
else: return f"[ERR {repr(e)[:50]}]"
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def _q(s, sql, p=None):
|
|
49
|
-
try: return s.execute(text(sql), p or {}).fetchall()
|
|
50
|
-
except Exception:
|
|
51
|
-
try: s.rollback()
|
|
52
|
-
except Exception: pass
|
|
53
|
-
return []
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def real_schema(theme):
|
|
57
|
-
"""该功能的真实业务表(db_tables) + 列(db_columns) = 业务保真 gold。"""
|
|
58
|
-
s = SessionLocal()
|
|
59
|
-
tabs = [r[0] for r in _q(s, "SELECT table_name FROM db_tables WHERE table_name ILIKE :p OR table_comment ILIKE :p LIMIT 15", {"p": f"%{theme}%"})]
|
|
60
|
-
cols = [r[0] for r in _q(s, "SELECT column_name FROM db_columns WHERE column_name ILIKE :p OR column_comment ILIKE :p LIMIT 15", {"p": f"%{theme}%"})]
|
|
61
|
-
all_tabs = set(r[0] for r in _q(s, "SELECT table_name FROM db_tables")) # 全量真表(保真校验集)
|
|
62
|
-
s.close()
|
|
63
|
-
return tabs, cols, all_tabs
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def extract_tables(prd):
|
|
67
|
-
return set(m.lower() for m in _TABLE_RE.findall(prd))
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def fidelity(prd, gold_tabs, all_tabs):
|
|
71
|
-
"""保真率 = PRD 提的表 ∈ 真实 schema 的比例;召回 = gold 表被提的比例。"""
|
|
72
|
-
mentioned = extract_tables(prd)
|
|
73
|
-
if not mentioned:
|
|
74
|
-
return 0.0, 0.0, 0
|
|
75
|
-
real_mentioned = mentioned & all_tabs
|
|
76
|
-
precision = len(real_mentioned) / len(mentioned) # 保真:提的表是真的比例
|
|
77
|
-
recall = len(mentioned & set(t.lower() for t in gold_tabs)) / max(len(gold_tabs), 1) # 召回:gold 表被提
|
|
78
|
-
return round(precision, 3), round(recall, 3), len(mentioned)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
def run():
|
|
82
|
-
rep = {"features": []}
|
|
83
|
-
for feat, theme in FEATURES:
|
|
84
|
-
gtabs, gcols, all_tabs = real_schema(theme)
|
|
85
|
-
schema_ctx = "真实业务表:\n" + "\n".join(gtabs[:12]) + "\n真实业务列:\n" + ", ".join(gcols[:12])
|
|
86
|
-
bare = chat(f"你是产品经理。为 ICS 系统「{feat}」写 PRD(含字段设计/数据表)。直接输出正文。")
|
|
87
|
-
scaff = chat(f"你是 ICS(车辆物联网)产品经理。为「{feat}」写 PRD(含字段设计/数据表)。\n"
|
|
88
|
-
f"【业务 schema 接地】以下是该功能的真实业务表/列(均为真实存在):\n{schema_ctx}\n"
|
|
89
|
-
f"铁律:字段设计/数据表章节【必须把上面列出的真实业务表全部引用】,用真实表名,勿编造/勿省略。"
|
|
90
|
-
f"直接输出 PRD 正文。")
|
|
91
|
-
bP, bR, bN = fidelity(bare, gtabs, all_tabs)
|
|
92
|
-
sP, sR, sN = fidelity(scaff, gtabs, all_tabs)
|
|
93
|
-
rep["features"].append({"feature": feat, "gold_tabs": len(gtabs),
|
|
94
|
-
"bare_fidelity": bP, "bare_recall": bR, "bare_mentioned": bN,
|
|
95
|
-
"scaffold_fidelity": sP, "scaffold_recall": sR, "scaffold_mentioned": sN})
|
|
96
|
-
print(" %s: gold表=%d | bare 保真=%.2f 召回=%.2f(提%d) | scaffold 保真=%.2f 召回=%.2f(提%d)" % (
|
|
97
|
-
feat, len(gtabs), bP, bR, bN, sP, sR, sN))
|
|
98
|
-
n = len(FEATURES) or 1
|
|
99
|
-
rep["agg"] = {
|
|
100
|
-
"bare_fidelity": round(sum(f["bare_fidelity"] for f in rep["features"])/n, 3),
|
|
101
|
-
"scaffold_fidelity": round(sum(f["scaffold_fidelity"] for f in rep["features"])/n, 3),
|
|
102
|
-
"bare_recall": round(sum(f["bare_recall"] for f in rep["features"])/n, 3),
|
|
103
|
-
"scaffold_recall": round(sum(f["scaffold_recall"] for f in rep["features"])/n, 3),
|
|
104
|
-
}
|
|
105
|
-
return rep
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
def main():
|
|
109
|
-
import argparse
|
|
110
|
-
ap = argparse.ArgumentParser()
|
|
111
|
-
ap.add_argument("--json", action="store_true")
|
|
112
|
-
args = ap.parse_args()
|
|
113
|
-
r = run()
|
|
114
|
-
a = r["agg"]
|
|
115
|
-
if args.json:
|
|
116
|
-
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
117
|
-
return
|
|
118
|
-
print("\n## FW9 PRD 业务保真度飞轮(PRD 提的表 vs 真实 ICS schema)")
|
|
119
|
-
print("| 维度 | bare(无schema接地) | +真schema接地 |")
|
|
120
|
-
print("|---|---|---|")
|
|
121
|
-
print("| 保真率(提的表是真的) | %.0f%% | %.0f%% |" % (a["bare_fidelity"]*100, a["scaffold_fidelity"]*100))
|
|
122
|
-
print("| 召回(gold真表被提) | %.0f%% | %.0f%% |" % (a["bare_recall"]*100, a["scaffold_recall"]*100))
|
|
123
|
-
print("\n业务保真=PRD 引用的是 ICS 真实业务表(t_insurance_log 等),非编造/泛泛。"
|
|
124
|
-
"bare 编造表名→保真低;scaffold 接真 schema→保真高。")
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if __name__ == "__main__":
|
|
128
|
-
main()
|
|
1
|
+
"""prd_fidelity_flywheel.py — FW9 PRD 业务保真度飞轮(vs 真实 ICS schema,非结构代理)。
|
|
2
|
+
|
|
3
|
+
用户根因:之前 FW2 量的是"6 章齐"(结构),不是"PRD 是不是这个业务的"(保真)。
|
|
4
|
+
本尺子量 **schema 保真**:PRD 引用的表/列是不是 ICS 真实业务表列(db_tables/db_columns),
|
|
5
|
+
而非泛泛/编造。bare(无 schema grounding→编造表名) vs +真schema grounding(喂真表真列)。
|
|
6
|
+
保真率 = PRD 提的表/列 ∈ 真实 schema 的比例;召回 = 该功能真表被提的比例。
|
|
7
|
+
|
|
8
|
+
(原型颜色保真=eval_prd A2 读宿主真色板,云 KG 无设计系统表→那格 🖥️ 宿主;本尺子量可代理的 schema 保真)
|
|
9
|
+
|
|
10
|
+
跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/prd_fidelity_flywheel.py
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
import json
|
|
16
|
+
import time
|
|
17
|
+
import urllib.request
|
|
18
|
+
import urllib.error
|
|
19
|
+
|
|
20
|
+
for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
|
|
21
|
+
if os.path.isdir(_p) and _p not in sys.path:
|
|
22
|
+
sys.path.insert(0, _p)
|
|
23
|
+
|
|
24
|
+
from sqlalchemy import text # noqa: E402
|
|
25
|
+
from app.db import SessionLocal # noqa: E402
|
|
26
|
+
|
|
27
|
+
CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
|
|
28
|
+
CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
|
|
29
|
+
MODEL = "qwen-turbo"
|
|
30
|
+
FEATURES = [("车辆保险管理", "保险"), ("异常记录管理", "异常"), ("合同维护", "合同")]
|
|
31
|
+
# 表名 token(t_xxx / tb_xxx / report_xxx 等)
|
|
32
|
+
_TABLE_RE = re.compile(r'\b(t_[a-z_]+|tb_[a-z_]+|report_[a-z_]+|salary_[a-z_]+|[a-z]+_[a-z_]+_table)\b', re.IGNORECASE)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def chat(prompt, max_tokens=900, timeout=90):
|
|
36
|
+
body = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": prompt}],
|
|
37
|
+
"temperature": 0.2, "max_tokens": max_tokens}).encode()
|
|
38
|
+
for a in range(3):
|
|
39
|
+
try:
|
|
40
|
+
req = urllib.request.Request(CHAT_URL, data=body,
|
|
41
|
+
headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
|
|
42
|
+
return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
|
|
43
|
+
except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
|
|
44
|
+
if a < 2: time.sleep(1.0*(a+1))
|
|
45
|
+
else: return f"[ERR {repr(e)[:50]}]"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _q(s, sql, p=None):
|
|
49
|
+
try: return s.execute(text(sql), p or {}).fetchall()
|
|
50
|
+
except Exception:
|
|
51
|
+
try: s.rollback()
|
|
52
|
+
except Exception: pass
|
|
53
|
+
return []
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def real_schema(theme):
|
|
57
|
+
"""该功能的真实业务表(db_tables) + 列(db_columns) = 业务保真 gold。"""
|
|
58
|
+
s = SessionLocal()
|
|
59
|
+
tabs = [r[0] for r in _q(s, "SELECT table_name FROM db_tables WHERE table_name ILIKE :p OR table_comment ILIKE :p LIMIT 15", {"p": f"%{theme}%"})]
|
|
60
|
+
cols = [r[0] for r in _q(s, "SELECT column_name FROM db_columns WHERE column_name ILIKE :p OR column_comment ILIKE :p LIMIT 15", {"p": f"%{theme}%"})]
|
|
61
|
+
all_tabs = set(r[0] for r in _q(s, "SELECT table_name FROM db_tables")) # 全量真表(保真校验集)
|
|
62
|
+
s.close()
|
|
63
|
+
return tabs, cols, all_tabs
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def extract_tables(prd):
|
|
67
|
+
return set(m.lower() for m in _TABLE_RE.findall(prd))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def fidelity(prd, gold_tabs, all_tabs):
|
|
71
|
+
"""保真率 = PRD 提的表 ∈ 真实 schema 的比例;召回 = gold 表被提的比例。"""
|
|
72
|
+
mentioned = extract_tables(prd)
|
|
73
|
+
if not mentioned:
|
|
74
|
+
return 0.0, 0.0, 0
|
|
75
|
+
real_mentioned = mentioned & all_tabs
|
|
76
|
+
precision = len(real_mentioned) / len(mentioned) # 保真:提的表是真的比例
|
|
77
|
+
recall = len(mentioned & set(t.lower() for t in gold_tabs)) / max(len(gold_tabs), 1) # 召回:gold 表被提
|
|
78
|
+
return round(precision, 3), round(recall, 3), len(mentioned)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def run():
|
|
82
|
+
rep = {"features": []}
|
|
83
|
+
for feat, theme in FEATURES:
|
|
84
|
+
gtabs, gcols, all_tabs = real_schema(theme)
|
|
85
|
+
schema_ctx = "真实业务表:\n" + "\n".join(gtabs[:12]) + "\n真实业务列:\n" + ", ".join(gcols[:12])
|
|
86
|
+
bare = chat(f"你是产品经理。为 ICS 系统「{feat}」写 PRD(含字段设计/数据表)。直接输出正文。")
|
|
87
|
+
scaff = chat(f"你是 ICS(车辆物联网)产品经理。为「{feat}」写 PRD(含字段设计/数据表)。\n"
|
|
88
|
+
f"【业务 schema 接地】以下是该功能的真实业务表/列(均为真实存在):\n{schema_ctx}\n"
|
|
89
|
+
f"铁律:字段设计/数据表章节【必须把上面列出的真实业务表全部引用】,用真实表名,勿编造/勿省略。"
|
|
90
|
+
f"直接输出 PRD 正文。")
|
|
91
|
+
bP, bR, bN = fidelity(bare, gtabs, all_tabs)
|
|
92
|
+
sP, sR, sN = fidelity(scaff, gtabs, all_tabs)
|
|
93
|
+
rep["features"].append({"feature": feat, "gold_tabs": len(gtabs),
|
|
94
|
+
"bare_fidelity": bP, "bare_recall": bR, "bare_mentioned": bN,
|
|
95
|
+
"scaffold_fidelity": sP, "scaffold_recall": sR, "scaffold_mentioned": sN})
|
|
96
|
+
print(" %s: gold表=%d | bare 保真=%.2f 召回=%.2f(提%d) | scaffold 保真=%.2f 召回=%.2f(提%d)" % (
|
|
97
|
+
feat, len(gtabs), bP, bR, bN, sP, sR, sN))
|
|
98
|
+
n = len(FEATURES) or 1
|
|
99
|
+
rep["agg"] = {
|
|
100
|
+
"bare_fidelity": round(sum(f["bare_fidelity"] for f in rep["features"])/n, 3),
|
|
101
|
+
"scaffold_fidelity": round(sum(f["scaffold_fidelity"] for f in rep["features"])/n, 3),
|
|
102
|
+
"bare_recall": round(sum(f["bare_recall"] for f in rep["features"])/n, 3),
|
|
103
|
+
"scaffold_recall": round(sum(f["scaffold_recall"] for f in rep["features"])/n, 3),
|
|
104
|
+
}
|
|
105
|
+
return rep
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def main():
|
|
109
|
+
import argparse
|
|
110
|
+
ap = argparse.ArgumentParser()
|
|
111
|
+
ap.add_argument("--json", action="store_true")
|
|
112
|
+
args = ap.parse_args()
|
|
113
|
+
r = run()
|
|
114
|
+
a = r["agg"]
|
|
115
|
+
if args.json:
|
|
116
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
117
|
+
return
|
|
118
|
+
print("\n## FW9 PRD 业务保真度飞轮(PRD 提的表 vs 真实 ICS schema)")
|
|
119
|
+
print("| 维度 | bare(无schema接地) | +真schema接地 |")
|
|
120
|
+
print("|---|---|---|")
|
|
121
|
+
print("| 保真率(提的表是真的) | %.0f%% | %.0f%% |" % (a["bare_fidelity"]*100, a["scaffold_fidelity"]*100))
|
|
122
|
+
print("| 召回(gold真表被提) | %.0f%% | %.0f%% |" % (a["bare_recall"]*100, a["scaffold_recall"]*100))
|
|
123
|
+
print("\n业务保真=PRD 引用的是 ICS 真实业务表(t_insurance_log 等),非编造/泛泛。"
|
|
124
|
+
"bare 编造表名→保真低;scaffold 接真 schema→保真高。")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
if __name__ == "__main__":
|
|
128
|
+
main()
|
|
@@ -1,148 +1,148 @@
|
|
|
1
|
-
"""prd_flywheel.py — FW2 wl-prd 生成类质量飞轮(完整性/准确性 → 目标)。
|
|
2
|
-
|
|
3
|
-
生成类维度(SPEC §1.1):完整性(必要段数) / 准确性(gold 对齐) / 可用性(EVA≥80%)。
|
|
4
|
-
飞轮:弱模型(bare prompt) vs +wl-prd scaffold(6章强制 + KG grounding) → 量
|
|
5
|
-
- 完整性 = 6 章命中 / 6(QUICK_SECTIONS,纯 regex)
|
|
6
|
-
- 现实锚定 = PRD 里出现的【KG 真实体名】数(bare 编造→少;scaffold 接地→多)
|
|
7
|
-
→ 改 scaffold(强制章 + 接地)→ 重量 → 直到 完整性=6/6 + 锚定≥阈值。
|
|
8
|
-
|
|
9
|
-
云端跑(DashScope 生 + PG 真实体)。eval_prd.py 是宿主 DuckDB 侧,本尺子是其云端同构代理。
|
|
10
|
-
跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/prd_flywheel.py
|
|
11
|
-
"""
|
|
12
|
-
import os
|
|
13
|
-
import re
|
|
14
|
-
import sys
|
|
15
|
-
import json
|
|
16
|
-
import time
|
|
17
|
-
import urllib.request
|
|
18
|
-
import urllib.error
|
|
19
|
-
|
|
20
|
-
for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
|
|
21
|
-
if os.path.isdir(_p) and _p not in sys.path:
|
|
22
|
-
sys.path.insert(0, _p)
|
|
23
|
-
|
|
24
|
-
from sqlalchemy import text # noqa: E402
|
|
25
|
-
from app.db import SessionLocal # noqa: E402
|
|
26
|
-
|
|
27
|
-
CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
|
|
28
|
-
CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
|
|
29
|
-
MODEL = "qwen-turbo" # 弱模型(卖点:弱模型+工作流)
|
|
30
|
-
QUICK_SECTIONS = ["功能入口", "需求背景", "需求说明", "影响范围", "验收标准", "不在本次范围"]
|
|
31
|
-
FEATURES = ["车辆保养提醒", "考勤异常导出", "保险续保预警"]
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def chat(prompt, max_tokens=900, timeout=90):
|
|
35
|
-
body = json.dumps({"model": MODEL,
|
|
36
|
-
"messages": [{"role": "user", "content": prompt}],
|
|
37
|
-
"temperature": 0.2, "max_tokens": max_tokens}).encode()
|
|
38
|
-
for attempt in range(3):
|
|
39
|
-
try:
|
|
40
|
-
req = urllib.request.Request(CHAT_URL, data=body,
|
|
41
|
-
headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
|
|
42
|
-
return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
|
|
43
|
-
except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
|
|
44
|
-
if attempt < 2:
|
|
45
|
-
time.sleep(1.0 * (attempt + 1))
|
|
46
|
-
else:
|
|
47
|
-
return f"[ERR {repr(e)[:60]}]"
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
def _q(s, sql, params=None):
|
|
51
|
-
try:
|
|
52
|
-
return s.execute(text(sql), params or {}).fetchall()
|
|
53
|
-
except Exception:
|
|
54
|
-
try:
|
|
55
|
-
s.rollback()
|
|
56
|
-
except Exception:
|
|
57
|
-
pass
|
|
58
|
-
return []
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def real_entities(theme, k=15):
|
|
62
|
-
"""KG 里该主题的真实实体 (canonical, cn)(现实锚定 gold)。"""
|
|
63
|
-
s = SessionLocal()
|
|
64
|
-
rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
|
|
65
|
-
"AND canonical IS NOT NULL LIMIT :n", {"p": f"%{theme}%", "n": k})
|
|
66
|
-
s.close()
|
|
67
|
-
return [(r[0], r[1]) for r in rows if r[0]]
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def completeness(prd_text):
|
|
71
|
-
"""6 章命中数(regex,接受 #/##/粗体/裸文本)。"""
|
|
72
|
-
hit = 0
|
|
73
|
-
for sec in QUICK_SECTIONS:
|
|
74
|
-
if re.search(r'#{0,3}\s*\*{0,2}' + sec, prd_text):
|
|
75
|
-
hit += 1
|
|
76
|
-
return hit
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
def reality_anchor(prd_text, reals):
|
|
80
|
-
"""PRD 里出现的 KG 真实体数(canonical 或 cn 命中任一即算;cn 中文名更易在散文出现)。"""
|
|
81
|
-
n = 0
|
|
82
|
-
for canon, cn in reals:
|
|
83
|
-
if (canon and canon in prd_text) or (cn and cn in prd_text):
|
|
84
|
-
n += 1
|
|
85
|
-
return n
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def gen_bare(feature):
|
|
89
|
-
return (f"你是产品经理。为「{feature}」功能写一份 PRD 需求文档。\n"
|
|
90
|
-
f"直接输出 PRD 正文。")
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def gen_scaffold(feature, grounding):
|
|
94
|
-
return (f"你是车辆物联网系统(ICS)的产品经理。为「{feature}」写一份 PRD。\n"
|
|
95
|
-
f"知识图谱检索到该功能相关的真实系统对象(字段/类/接口,均为真实存在):\n{grounding}\n\n"
|
|
96
|
-
f"**铁律**:PRD 必须【完整含以下 6 章,每章一个 ## 标题,缺一不可】:\n"
|
|
97
|
-
f"## 功能入口\n## 需求背景\n## 需求说明\n## 影响范围\n## 验收标准\n## 不在本次范围\n"
|
|
98
|
-
f"PRD 里的字段名/接口名【必须用上面检索到的真实名称】,不要编造。直接输出 PRD 正文。")
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
def run():
|
|
102
|
-
rep = {"model": MODEL, "features": []}
|
|
103
|
-
for feat in FEATURES:
|
|
104
|
-
reals = real_entities(feat.split("提醒")[0].split("导出")[0].split("预警")[0] or feat, 15)
|
|
105
|
-
ctx = "\n".join("%s (%s)" % (c, n) for c, n in reals[:15]) if reals else "(无检索结果)"
|
|
106
|
-
bare = chat(gen_bare(feat))
|
|
107
|
-
scaff = chat(gen_scaffold(feat, ctx))
|
|
108
|
-
b_cmp = completeness(bare)
|
|
109
|
-
s_cmp = completeness(scaff)
|
|
110
|
-
b_real = reality_anchor(bare, reals)
|
|
111
|
-
s_real = reality_anchor(scaff, reals)
|
|
112
|
-
rep["features"].append({"feature": feat, "real_n": len(reals),
|
|
113
|
-
"bare_completeness": b_cmp, "scaffold_completeness": s_cmp,
|
|
114
|
-
"bare_reality": b_real, "scaffold_reality": s_real})
|
|
115
|
-
print(" %s: 完整性 bare=%d/6 scaff=%d/6 | 锚定 bare=%d scaff=%d (gold=%d)" % (
|
|
116
|
-
feat, b_cmp, s_cmp, b_real, s_real, len(reals)))
|
|
117
|
-
n = len(FEATURES) or 1
|
|
118
|
-
rep["agg"] = {
|
|
119
|
-
"bare_completeness": round(sum(f["bare_completeness"] for f in rep["features"]) / n, 2),
|
|
120
|
-
"scaffold_completeness": round(sum(f["scaffold_completeness"] for f in rep["features"]) / n, 2),
|
|
121
|
-
"bare_reality": round(sum(f["bare_reality"] for f in rep["features"]) / n, 2),
|
|
122
|
-
"scaffold_reality": round(sum(f["scaffold_reality"] for f in rep["features"]) / n, 2),
|
|
123
|
-
}
|
|
124
|
-
return rep
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def main():
|
|
128
|
-
import argparse
|
|
129
|
-
ap = argparse.ArgumentParser()
|
|
130
|
-
ap.add_argument("--json", action="store_true")
|
|
131
|
-
args = ap.parse_args()
|
|
132
|
-
r = run()
|
|
133
|
-
a = r["agg"]
|
|
134
|
-
if args.json:
|
|
135
|
-
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
136
|
-
return
|
|
137
|
-
print("\n## FW2 wl-prd 生成类飞轮(完整性 6/6 + 现实锚定)")
|
|
138
|
-
print("| 维度 | bare(弱模型裸) | +wl-prd scaffold | 目标 |")
|
|
139
|
-
print("|---|---|---|---|")
|
|
140
|
-
print("| 完整性(章/6) | %.2f | %.2f | 6.0 |" % (a["bare_completeness"], a["scaffold_completeness"]))
|
|
141
|
-
print("| 现实锚定(真实体命中) | %.2f | %.2f | 高 |" % (a["bare_reality"], a["scaffold_reality"]))
|
|
142
|
-
cmp_met = a["scaffold_completeness"] >= 6.0
|
|
143
|
-
print("\n完整性达标(≥6/6)?" , "✓" if cmp_met else "✗ → 下轮:强章模板/Stop hook 拦缺章")
|
|
144
|
-
print("锚定提升:", "%+.2f" % (a["scaffold_reality"] - a["bare_reality"]), "(scaffold 接地 vs bare 编造)")
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
if __name__ == "__main__":
|
|
148
|
-
main()
|
|
1
|
+
"""prd_flywheel.py — FW2 wl-prd 生成类质量飞轮(完整性/准确性 → 目标)。
|
|
2
|
+
|
|
3
|
+
生成类维度(SPEC §1.1):完整性(必要段数) / 准确性(gold 对齐) / 可用性(EVA≥80%)。
|
|
4
|
+
飞轮:弱模型(bare prompt) vs +wl-prd scaffold(6章强制 + KG grounding) → 量
|
|
5
|
+
- 完整性 = 6 章命中 / 6(QUICK_SECTIONS,纯 regex)
|
|
6
|
+
- 现实锚定 = PRD 里出现的【KG 真实体名】数(bare 编造→少;scaffold 接地→多)
|
|
7
|
+
→ 改 scaffold(强制章 + 接地)→ 重量 → 直到 完整性=6/6 + 锚定≥阈值。
|
|
8
|
+
|
|
9
|
+
云端跑(DashScope 生 + PG 真实体)。eval_prd.py 是宿主 DuckDB 侧,本尺子是其云端同构代理。
|
|
10
|
+
跑法: docker exec -i wlinkj-workspace-backend-1 python /app/scripts/validation/eval/prd_flywheel.py
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
import json
|
|
16
|
+
import time
|
|
17
|
+
import urllib.request
|
|
18
|
+
import urllib.error
|
|
19
|
+
|
|
20
|
+
for _p in ("/app", os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))):
|
|
21
|
+
if os.path.isdir(_p) and _p not in sys.path:
|
|
22
|
+
sys.path.insert(0, _p)
|
|
23
|
+
|
|
24
|
+
from sqlalchemy import text # noqa: E402
|
|
25
|
+
from app.db import SessionLocal # noqa: E402
|
|
26
|
+
|
|
27
|
+
CHAT_URL = os.environ.get("ALI_CHAT_BASE", "").rstrip("/") + "/chat/completions"
|
|
28
|
+
CHAT_KEY = os.environ.get("DASHSCOPE_API_KEY", "")
|
|
29
|
+
MODEL = "qwen-turbo" # 弱模型(卖点:弱模型+工作流)
|
|
30
|
+
QUICK_SECTIONS = ["功能入口", "需求背景", "需求说明", "影响范围", "验收标准", "不在本次范围"]
|
|
31
|
+
FEATURES = ["车辆保养提醒", "考勤异常导出", "保险续保预警"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def chat(prompt, max_tokens=900, timeout=90):
|
|
35
|
+
body = json.dumps({"model": MODEL,
|
|
36
|
+
"messages": [{"role": "user", "content": prompt}],
|
|
37
|
+
"temperature": 0.2, "max_tokens": max_tokens}).encode()
|
|
38
|
+
for attempt in range(3):
|
|
39
|
+
try:
|
|
40
|
+
req = urllib.request.Request(CHAT_URL, data=body,
|
|
41
|
+
headers={"Authorization": f"Bearer {CHAT_KEY}", "Content-Type": "application/json"})
|
|
42
|
+
return json.loads(urllib.request.urlopen(req, timeout=timeout).read())["choices"][0]["message"]["content"]
|
|
43
|
+
except (urllib.error.URLError, TimeoutError, OSError, KeyError) as e:
|
|
44
|
+
if attempt < 2:
|
|
45
|
+
time.sleep(1.0 * (attempt + 1))
|
|
46
|
+
else:
|
|
47
|
+
return f"[ERR {repr(e)[:60]}]"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _q(s, sql, params=None):
|
|
51
|
+
try:
|
|
52
|
+
return s.execute(text(sql), params or {}).fetchall()
|
|
53
|
+
except Exception:
|
|
54
|
+
try:
|
|
55
|
+
s.rollback()
|
|
56
|
+
except Exception:
|
|
57
|
+
pass
|
|
58
|
+
return []
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def real_entities(theme, k=15):
|
|
62
|
+
"""KG 里该主题的真实实体 (canonical, cn)(现实锚定 gold)。"""
|
|
63
|
+
s = SessionLocal()
|
|
64
|
+
rows = _q(s, "SELECT canonical, cn FROM entities WHERE (canonical ILIKE :p OR cn ILIKE :p) "
|
|
65
|
+
"AND canonical IS NOT NULL LIMIT :n", {"p": f"%{theme}%", "n": k})
|
|
66
|
+
s.close()
|
|
67
|
+
return [(r[0], r[1]) for r in rows if r[0]]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def completeness(prd_text):
|
|
71
|
+
"""6 章命中数(regex,接受 #/##/粗体/裸文本)。"""
|
|
72
|
+
hit = 0
|
|
73
|
+
for sec in QUICK_SECTIONS:
|
|
74
|
+
if re.search(r'#{0,3}\s*\*{0,2}' + sec, prd_text):
|
|
75
|
+
hit += 1
|
|
76
|
+
return hit
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def reality_anchor(prd_text, reals):
|
|
80
|
+
"""PRD 里出现的 KG 真实体数(canonical 或 cn 命中任一即算;cn 中文名更易在散文出现)。"""
|
|
81
|
+
n = 0
|
|
82
|
+
for canon, cn in reals:
|
|
83
|
+
if (canon and canon in prd_text) or (cn and cn in prd_text):
|
|
84
|
+
n += 1
|
|
85
|
+
return n
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def gen_bare(feature):
|
|
89
|
+
return (f"你是产品经理。为「{feature}」功能写一份 PRD 需求文档。\n"
|
|
90
|
+
f"直接输出 PRD 正文。")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def gen_scaffold(feature, grounding):
|
|
94
|
+
return (f"你是车辆物联网系统(ICS)的产品经理。为「{feature}」写一份 PRD。\n"
|
|
95
|
+
f"知识图谱检索到该功能相关的真实系统对象(字段/类/接口,均为真实存在):\n{grounding}\n\n"
|
|
96
|
+
f"**铁律**:PRD 必须【完整含以下 6 章,每章一个 ## 标题,缺一不可】:\n"
|
|
97
|
+
f"## 功能入口\n## 需求背景\n## 需求说明\n## 影响范围\n## 验收标准\n## 不在本次范围\n"
|
|
98
|
+
f"PRD 里的字段名/接口名【必须用上面检索到的真实名称】,不要编造。直接输出 PRD 正文。")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def run():
|
|
102
|
+
rep = {"model": MODEL, "features": []}
|
|
103
|
+
for feat in FEATURES:
|
|
104
|
+
reals = real_entities(feat.split("提醒")[0].split("导出")[0].split("预警")[0] or feat, 15)
|
|
105
|
+
ctx = "\n".join("%s (%s)" % (c, n) for c, n in reals[:15]) if reals else "(无检索结果)"
|
|
106
|
+
bare = chat(gen_bare(feat))
|
|
107
|
+
scaff = chat(gen_scaffold(feat, ctx))
|
|
108
|
+
b_cmp = completeness(bare)
|
|
109
|
+
s_cmp = completeness(scaff)
|
|
110
|
+
b_real = reality_anchor(bare, reals)
|
|
111
|
+
s_real = reality_anchor(scaff, reals)
|
|
112
|
+
rep["features"].append({"feature": feat, "real_n": len(reals),
|
|
113
|
+
"bare_completeness": b_cmp, "scaffold_completeness": s_cmp,
|
|
114
|
+
"bare_reality": b_real, "scaffold_reality": s_real})
|
|
115
|
+
print(" %s: 完整性 bare=%d/6 scaff=%d/6 | 锚定 bare=%d scaff=%d (gold=%d)" % (
|
|
116
|
+
feat, b_cmp, s_cmp, b_real, s_real, len(reals)))
|
|
117
|
+
n = len(FEATURES) or 1
|
|
118
|
+
rep["agg"] = {
|
|
119
|
+
"bare_completeness": round(sum(f["bare_completeness"] for f in rep["features"]) / n, 2),
|
|
120
|
+
"scaffold_completeness": round(sum(f["scaffold_completeness"] for f in rep["features"]) / n, 2),
|
|
121
|
+
"bare_reality": round(sum(f["bare_reality"] for f in rep["features"]) / n, 2),
|
|
122
|
+
"scaffold_reality": round(sum(f["scaffold_reality"] for f in rep["features"]) / n, 2),
|
|
123
|
+
}
|
|
124
|
+
return rep
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def main():
|
|
128
|
+
import argparse
|
|
129
|
+
ap = argparse.ArgumentParser()
|
|
130
|
+
ap.add_argument("--json", action="store_true")
|
|
131
|
+
args = ap.parse_args()
|
|
132
|
+
r = run()
|
|
133
|
+
a = r["agg"]
|
|
134
|
+
if args.json:
|
|
135
|
+
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
136
|
+
return
|
|
137
|
+
print("\n## FW2 wl-prd 生成类飞轮(完整性 6/6 + 现实锚定)")
|
|
138
|
+
print("| 维度 | bare(弱模型裸) | +wl-prd scaffold | 目标 |")
|
|
139
|
+
print("|---|---|---|---|")
|
|
140
|
+
print("| 完整性(章/6) | %.2f | %.2f | 6.0 |" % (a["bare_completeness"], a["scaffold_completeness"]))
|
|
141
|
+
print("| 现实锚定(真实体命中) | %.2f | %.2f | 高 |" % (a["bare_reality"], a["scaffold_reality"]))
|
|
142
|
+
cmp_met = a["scaffold_completeness"] >= 6.0
|
|
143
|
+
print("\n完整性达标(≥6/6)?" , "✓" if cmp_met else "✗ → 下轮:强章模板/Stop hook 拦缺章")
|
|
144
|
+
print("锚定提升:", "%+.2f" % (a["scaffold_reality"] - a["bare_reality"]), "(scaffold 接地 vs bare 编造)")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
main()
|