@hupan56/wlkj 3.3.14 → 3.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +16 -1
- package/package.json +1 -1
- package/templates/qoder/commands/optional/wl-spec.md +4 -2
- package/templates/qoder/commands/wl-commit.md +3 -1
- package/templates/qoder/commands/wl-init.md +2 -2
- package/templates/qoder/commands/wl-prd.md +24 -0
- package/templates/qoder/commands/wl-search.md +18 -11
- package/templates/qoder/commands/wl-task.md +3 -1
- package/templates/qoder/config.yaml +0 -6
- package/templates/qoder/contracts/insight.md +55 -0
- package/templates/qoder/hooks/pre-tool-use-commit.py +124 -0
- package/templates/qoder/hooks/session-start.py +20 -1
- package/templates/qoder/hooks/stop-eval.py +129 -0
- package/templates/qoder/scripts/capability/adapters/mcp.py +9 -1
- package/templates/qoder/scripts/capability/registry.py +0 -3
- package/templates/qoder/scripts/capability/registry_mcp.py +10 -4
- package/templates/qoder/scripts/deployment/setup/init_doctor.py +6 -2
- package/templates/qoder/scripts/deployment/setup/install_qoderwork.py +7 -7
- package/templates/qoder/scripts/deployment/setup/setup.py +5 -2
- package/templates/qoder/scripts/domain/integration/return_to_platform.py +4 -2
- package/templates/qoder/scripts/domain/integration/spec_upload.py +4 -4
- package/templates/qoder/scripts/domain/kg/build/build_entity_registry.py +12 -12
- package/templates/qoder/scripts/domain/kg/build/build_relations.py +12 -12
- package/templates/qoder/scripts/domain/kg/graph/kg_semantic.py +41 -5
- package/templates/qoder/scripts/domain/kg/kg.py +14 -0
- package/templates/qoder/scripts/domain/kg/switch_project.py +2 -2
- package/templates/qoder/scripts/domain/task/wlkj_panel.py +169 -14
- package/templates/qoder/scripts/foundation/io/context_cache.py +94 -0
- package/templates/qoder/scripts/orchestration/wlkj.py +106 -0
- package/templates/qoder/scripts/tool_guide.md +70 -0
- package/templates/qoder/scripts/validation/eval/alignment_matrix.py +176 -0
- package/templates/qoder/scripts/validation/eval/bf2_content_fidelity.py +110 -0
- package/templates/qoder/scripts/validation/eval/bf2_llmjudge.py +104 -0
- package/templates/qoder/scripts/validation/eval/bf_score.py +218 -0
- package/templates/qoder/scripts/validation/eval/code_flywheel.py +150 -0
- package/templates/qoder/scripts/validation/eval/dispatcher_ab.py +156 -0
- package/templates/qoder/scripts/validation/eval/dispatcher_ab_2026-07-21.json +23 -0
- package/templates/qoder/scripts/validation/eval/feature_fidelity_flywheel.py +143 -0
- package/templates/qoder/scripts/validation/eval/gradient_matrix.py +261 -0
- package/templates/qoder/scripts/validation/eval/gradient_matrix_baseline_2026-07-21.json +33 -0
- package/templates/qoder/scripts/validation/eval/metrics_dashboard.py +105 -0
- package/templates/qoder/scripts/validation/eval/multi_turn_flywheel.py +118 -0
- package/templates/qoder/scripts/validation/eval/prd_fidelity_flywheel.py +128 -0
- package/templates/qoder/scripts/validation/eval/prd_flywheel.py +148 -0
- package/templates/qoder/scripts/validation/eval/prototype_fidelity_flywheel.py +166 -0
- package/templates/qoder/scripts/validation/eval/recall_flywheel.py +148 -0
- package/templates/qoder/scripts/validation/eval/robustness_flywheel.py +139 -0
- package/templates/qoder/scripts/validation/eval/speed_accuracy_flywheel.py +188 -0
- package/templates/qoder/scripts/validation/eval/task_flywheel.py +124 -0
- package/templates/qoder/scripts/validation/eval/token_flywheel.py +88 -0
- package/templates/qoder/scripts/validation/metrics/eval_code_ac.py +177 -0
- package/templates/qoder/scripts/validation/metrics/lint_cases.py +170 -0
- package/templates/qoder/scripts/validation/test/test_context_cache.py +78 -0
- package/templates/qoder/scripts/validation/test/test_lint_cases.py +60 -0
- package/templates/qoder/scripts/validation/test/test_pre_tool_use_commit.py +70 -0
- package/templates/qoder/settings.json +8 -0
- package/templates/qoder/skills/wl-spec/SKILL.md +1 -1
- package/templates/qoder/agents/design-agent.md +0 -20
- package/templates/qoder/agents/spec-generator.md +0 -21
- package/templates/qoder/scripts/capability/smoke_test_report.json.new +0 -94
- package/templates/qoder/scripts/domain/kg/extract/asset/__init__.py +0 -10
- package/templates/qoder/scripts/domain/kg/extract/asset/asset_tree.py +0 -57
- package/templates/qoder/scripts/domain/kg/extract/asset/discussion_importer.py +0 -62
- package/templates/qoder/scripts/domain/kg/extract/asset/prd_importer.py +0 -146
- package/templates/qoder/scripts/domain/kg/extract/asset/prototype_importer.py +0 -64
- package/templates/qoder/scripts/domain/kg/extract/asset/returns_importer.py +0 -52
- package/templates/qoder/scripts/domain/kg/extract/build_goal3.py +0 -104
- package/templates/qoder/scripts/domain/kg/extract/build_goal4.py +0 -55
- package/templates/qoder/scripts/domain/kg/extract/build_goal5.py +0 -95
- package/templates/qoder/scripts/domain/kg/extract/db/__init__.py +0 -8
- package/templates/qoder/scripts/domain/kg/extract/db/data_profile.py +0 -22
- package/templates/qoder/scripts/domain/kg/extract/db/fk_extractor.py +0 -55
- package/templates/qoder/scripts/domain/kg/extract/db/schema_extractor.py +0 -90
- package/templates/qoder/scripts/domain/kg/extract/inference/__init__.py +0 -9
- package/templates/qoder/scripts/domain/kg/extract/inference/community_summarizer.py +0 -206
- package/templates/qoder/scripts/domain/kg/extract/inference/embed_builder.py +0 -132
- package/templates/qoder/scripts/domain/kg/extract/inference/naming_matcher.py +0 -80
- package/templates/qoder/scripts/domain/kg/extract/inference/promote.py +0 -59
- package/templates/qoder/scripts/domain/kg/extract/inference/recompute.py +0 -93
- package/templates/qoder/scripts/domain/kg/extract/inference/weak_link.py +0 -421
- package/templates/qoder/scripts/domain/kg/extract/mybatis/__init__.py +0 -9
- package/templates/qoder/scripts/domain/kg/extract/mybatis/all.py +0 -79
- package/templates/qoder/scripts/domain/kg/extract/mybatis/mapper_parser.py +0 -99
- package/templates/qoder/scripts/domain/kg/extract/mybatis/relation_builder.py +0 -69
- package/templates/qoder/scripts/domain/kg/extract/mybatis/sql_extractor.py +0 -78
- package/templates/qoder/scripts/domain/kg/extract/prd/__init__.py +0 -8
- package/templates/qoder/scripts/domain/kg/extract/prd/prd_chunk_embed.py +0 -105
- package/templates/qoder/scripts/domain/kg/extract/prd/prd_llm_extract.py +0 -153
- package/templates/qoder/scripts/domain/kg/extract/prd/req_anchor.py +0 -120
- package/templates/qoder/scripts/domain/kg/graph/kg_link_db.py +0 -235
- package/templates/qoder/scripts/domain/kg/search/enrich_prompt.py +0 -238
- package/templates/qoder/scripts/domain/kg/server/perf_bench.py +0 -197
- package/templates/qoder/scripts/domain/kg/sync_repowiki.py +0 -109
- package/templates/qoder/scripts/engine/poller.py +0 -219
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""GOAL 5 统一入口: L3 弱关联推理 + C 方案重算。
|
|
3
|
-
|
|
4
|
-
CLI:
|
|
5
|
-
# 限量推理弱关联 (前500个无强关联实体)
|
|
6
|
-
python -m domain.kg.extract.build_goal5 --infer --limit 500
|
|
7
|
-
# C方案重算 (过期弱边续期/删除)
|
|
8
|
-
python -m domain.kg.extract.build_goal5 --recompute
|
|
9
|
-
# 升格 (删冗余弱边)
|
|
10
|
-
python -m domain.kg.extract.build_goal5 --promote
|
|
11
|
-
"""
|
|
12
|
-
from __future__ import annotations
|
|
13
|
-
import os, sys
|
|
14
|
-
|
|
15
|
-
_THIS = os.path.dirname(os.path.abspath(__file__))
|
|
16
|
-
for _i in range(8):
|
|
17
|
-
_p = os.path.dirname(_THIS)
|
|
18
|
-
if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
|
|
19
|
-
sys.path.insert(0, _p); break
|
|
20
|
-
_THIS = _p
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def run_infer(limit: int = 500, logger=print):
|
|
24
|
-
"""推理弱关联 (限量)。"""
|
|
25
|
-
from domain.kg.extract.inference import weak_link
|
|
26
|
-
from domain.kg.extract.java import pg_upsert
|
|
27
|
-
from sqlalchemy import text
|
|
28
|
-
|
|
29
|
-
engine, _ = pg_upsert._get_pg_engine()
|
|
30
|
-
if engine is None:
|
|
31
|
-
return {'error': 'PG 不可用'}
|
|
32
|
-
|
|
33
|
-
with engine.connect() as conn:
|
|
34
|
-
# 取无强关联的代码实体
|
|
35
|
-
entities = conn.execute(text(f"""
|
|
36
|
-
SELECT id, repo_id, type, canonical, cn, props FROM entities
|
|
37
|
-
WHERE type IN ('FUNCTION','BUTTON','ENDPOINT','CONTROLLER','SERVICE',
|
|
38
|
-
'MAPPER','ENTITY','TABLE','COLUMN','FEATURE')
|
|
39
|
-
AND id NOT IN (
|
|
40
|
-
SELECT from_id FROM edges
|
|
41
|
-
WHERE source IN ('explicit','commit')
|
|
42
|
-
AND edge_type IN ('implements_req','anchors')
|
|
43
|
-
)
|
|
44
|
-
LIMIT {int(limit)}
|
|
45
|
-
""")).all()
|
|
46
|
-
# 取全部 anchors
|
|
47
|
-
anchors = conn.execute(text("SELECT id, title, kind FROM anchors")).all()
|
|
48
|
-
|
|
49
|
-
ent_list = [{'id': r[0], 'repo_id': r[1], 'type': r[2],
|
|
50
|
-
'canonical': r[3], 'cn': r[4], 'props': r[5] or {}} for r in entities]
|
|
51
|
-
anchor_list = [{'id': r[0], 'title': r[1], 'kind': r[2]} for r in anchors]
|
|
52
|
-
logger(' [goal5] %d 实体待推理, %d 锚点' % (len(ent_list), len(anchor_list)))
|
|
53
|
-
|
|
54
|
-
edges, st = weak_link.infer_batch(ent_list, anchor_list, logger=logger)
|
|
55
|
-
|
|
56
|
-
# 写 PG (用 pg_upsert, 它会处理 expires_at/fingerprint 字段)
|
|
57
|
-
upsert_r = pg_upsert.upsert_batch([], edges) if edges else {'ok': False, 'edges_ok': 0}
|
|
58
|
-
st['upsert'] = upsert_r
|
|
59
|
-
return st
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def run_recompute(logger=print):
|
|
63
|
-
"""C 方案重算。"""
|
|
64
|
-
from domain.kg.extract.inference import recompute
|
|
65
|
-
return recompute.recompute_expired(logger=logger)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def run_promote(logger=print):
|
|
69
|
-
"""弱边升格。"""
|
|
70
|
-
from domain.kg.extract.inference import promote
|
|
71
|
-
return promote.promote_strong_over_weak(logger=logger)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if __name__ == '__main__':
|
|
75
|
-
import argparse, json
|
|
76
|
-
ap = argparse.ArgumentParser()
|
|
77
|
-
ap.add_argument('--infer', action='store_true')
|
|
78
|
-
ap.add_argument('--recompute', action='store_true')
|
|
79
|
-
ap.add_argument('--promote', action='store_true')
|
|
80
|
-
ap.add_argument('--limit', type=int, default=500)
|
|
81
|
-
args = ap.parse_args()
|
|
82
|
-
try:
|
|
83
|
-
from dotenv import load_dotenv
|
|
84
|
-
for p in ['wlinkj-workspace/backend/.env', '.env']:
|
|
85
|
-
if os.path.isfile(p): load_dotenv(p); break
|
|
86
|
-
except: pass
|
|
87
|
-
|
|
88
|
-
if args.infer:
|
|
89
|
-
print(json.dumps(run_infer(args.limit), ensure_ascii=False, indent=2, default=str))
|
|
90
|
-
elif args.recompute:
|
|
91
|
-
print(json.dumps(run_recompute(), ensure_ascii=False, indent=2, default=str))
|
|
92
|
-
elif args.promote:
|
|
93
|
-
print(json.dumps(run_promote(), ensure_ascii=False, indent=2, default=str))
|
|
94
|
-
else:
|
|
95
|
-
ap.print_help()
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""数据特征 profile (选做): 表行数、列 null 率、枚举分布。
|
|
3
|
-
|
|
4
|
-
价值: 让 KG 知道"这张表有 50 万行""这个字段 null 率 80%"——数据质量知识。
|
|
5
|
-
本轮 best-effort, 连不上 DB 跳过。
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
from . import schema_extractor as _se
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def profile_tables(repo_id: str = 'fywl-ics', sample_size: int = 100, logger=print):
|
|
12
|
-
"""采样表的数据特征。Returns: list[dict] 连不上返回 []。"""
|
|
13
|
-
if not _se._can_connect():
|
|
14
|
-
return []
|
|
15
|
-
try:
|
|
16
|
-
import pymysql
|
|
17
|
-
except ImportError:
|
|
18
|
-
return []
|
|
19
|
-
# 本轮简化: 只返回行数 (table_rows 已在 schema_extractor 取过)
|
|
20
|
-
# 完整版可加: SELECT COUNT(*), 列 null 率, DISTINCT 值分布
|
|
21
|
-
logger(' [db-profile] 本轮跳过详细 profile (行数已在 schema_extractor 取)')
|
|
22
|
-
return []
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""外键关系提取器: KEY_COLUMN_USAGE → references 边。
|
|
3
|
-
|
|
4
|
-
让 table_impact 真正能做"改表影响谁"的跨表追溯。
|
|
5
|
-
"""
|
|
6
|
-
from __future__ import annotations
|
|
7
|
-
from . import schema_extractor as _se
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def extract_foreign_keys(repo_id: str = 'fywl-ics', logger=print):
|
|
11
|
-
"""连 MySQL → 提取外键关系 → references 边。
|
|
12
|
-
|
|
13
|
-
Returns: [{from_id, to_id, edge_type:'references', ...}] 连不上返回 []
|
|
14
|
-
"""
|
|
15
|
-
if not _se._can_connect():
|
|
16
|
-
return []
|
|
17
|
-
try:
|
|
18
|
-
import pymysql
|
|
19
|
-
except ImportError:
|
|
20
|
-
return []
|
|
21
|
-
|
|
22
|
-
conn = pymysql.connect(host=_se.MYSQL_HOST, port=_se.MYSQL_PORT, user=_se.MYSQL_USER,
|
|
23
|
-
password=_se.MYSQL_PASSWORD, charset='utf8mb4',
|
|
24
|
-
cursorclass=pymysql.cursors.DictCursor,
|
|
25
|
-
connect_timeout=5, read_timeout=30)
|
|
26
|
-
edges = []
|
|
27
|
-
try:
|
|
28
|
-
cur = conn.cursor()
|
|
29
|
-
cur.execute("SELECT schema_name FROM information_schema.schemata "
|
|
30
|
-
"WHERE schema_name LIKE %s", (_se.DB_NAME_PREFIX + '%',))
|
|
31
|
-
dbs = [r['schema_name'] for r in cur.fetchall()]
|
|
32
|
-
for db in dbs:
|
|
33
|
-
# 外键关系
|
|
34
|
-
cur.execute("""
|
|
35
|
-
SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
|
|
36
|
-
FROM information_schema.KEY_COLUMN_USAGE
|
|
37
|
-
WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL
|
|
38
|
-
""", (db,))
|
|
39
|
-
for r in cur.fetchall():
|
|
40
|
-
from_tbl = r['TABLE_NAME'].lower()
|
|
41
|
-
to_tbl = r['REFERENCED_TABLE_NAME'].lower()
|
|
42
|
-
edges.append({
|
|
43
|
-
'from_id': 'TBL:%s:%s' % (repo_id, from_tbl),
|
|
44
|
-
'to_id': 'TBL:%s:%s' % (repo_id, to_tbl),
|
|
45
|
-
'edge_type': 'references',
|
|
46
|
-
'from_repo': repo_id, 'to_repo': repo_id,
|
|
47
|
-
'confidence': 1.0, 'source': 'db_fk',
|
|
48
|
-
'props': {'column': r['COLUMN_NAME'],
|
|
49
|
-
'ref_column': r['REFERENCED_COLUMN_NAME'],
|
|
50
|
-
'schema': db},
|
|
51
|
-
})
|
|
52
|
-
logger(' [db-fk] %d 条外键关系' % len(edges))
|
|
53
|
-
finally:
|
|
54
|
-
conn.close()
|
|
55
|
-
return edges
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""连 MySQL information_schema → 灌 db_tables / db_columns。
|
|
3
|
-
|
|
4
|
-
复用 kg_build_db.py 的 MySQL 连接配置 (内网 10.54.6.9, fywl_ics_* 库)。
|
|
5
|
-
连不上内网时 best-effort 返回空, 不阻塞 (GOAL 3 主链路是 mapper XML)。
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
import os
|
|
9
|
-
import sys
|
|
10
|
-
import socket
|
|
11
|
-
|
|
12
|
-
_THIS = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
-
for _i in range(8):
|
|
14
|
-
_p = os.path.dirname(_THIS)
|
|
15
|
-
if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
|
|
16
|
-
sys.path.insert(0, _p); break
|
|
17
|
-
_THIS = _p
|
|
18
|
-
|
|
19
|
-
# 复用 kg_build_db 的配置
|
|
20
|
-
MYSQL_HOST = os.environ.get('MYSQL_HOST', '10.54.6.9')
|
|
21
|
-
MYSQL_PORT = int(os.environ.get('MYSQL_PORT', '3306'))
|
|
22
|
-
MYSQL_USER = os.environ.get('MYSQL_USER', 'fywl_ics_test')
|
|
23
|
-
MYSQL_PASSWORD = os.environ.get('MYSQL_PASSWORD', 'Fywl_ics_test56476')
|
|
24
|
-
DB_NAME_PREFIX = os.environ.get('DB_NAME_PREFIX', 'fywl_ics_')
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def _can_connect(timeout=3):
|
|
28
|
-
"""探测内网 MySQL 是否可达。"""
|
|
29
|
-
try:
|
|
30
|
-
with socket.create_connection((MYSQL_HOST, MYSQL_PORT), timeout=timeout):
|
|
31
|
-
return True
|
|
32
|
-
except Exception:
|
|
33
|
-
return False
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def extract_all_schema(repo_id: str = 'fywl-ics', logger=print):
|
|
37
|
-
"""连 MySQL → 灌 db_tables + db_columns。
|
|
38
|
-
|
|
39
|
-
Returns: (tables: list[dict], columns: list[dict]) 连不上返回 ([], [])
|
|
40
|
-
"""
|
|
41
|
-
if not _can_connect():
|
|
42
|
-
logger(' [db] ⚠️ 内网 MySQL 不可达 (%s:%s), 跳过 schema 提取' % (MYSQL_HOST, MYSQL_PORT))
|
|
43
|
-
return [], []
|
|
44
|
-
try:
|
|
45
|
-
import pymysql
|
|
46
|
-
except ImportError:
|
|
47
|
-
logger(' [db] pymysql 未装, 跳过')
|
|
48
|
-
return [], []
|
|
49
|
-
|
|
50
|
-
logger(' [db] 连接 MySQL %s:%s ...' % (MYSQL_HOST, MYSQL_PORT))
|
|
51
|
-
conn = pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER,
|
|
52
|
-
password=MYSQL_PASSWORD, charset='utf8mb4',
|
|
53
|
-
cursorclass=pymysql.cursors.DictCursor,
|
|
54
|
-
connect_timeout=5, read_timeout=30)
|
|
55
|
-
tables, columns = [], []
|
|
56
|
-
try:
|
|
57
|
-
cur = conn.cursor()
|
|
58
|
-
cur.execute("SELECT schema_name FROM information_schema.schemata "
|
|
59
|
-
"WHERE schema_name LIKE %s ORDER BY schema_name",
|
|
60
|
-
(DB_NAME_PREFIX + '%'))
|
|
61
|
-
dbs = [r['schema_name'] for r in cur.fetchall()]
|
|
62
|
-
logger(' [db] 发现 %d 个业务库: %s' % (len(dbs), dbs[:3]))
|
|
63
|
-
|
|
64
|
-
for db in dbs:
|
|
65
|
-
# 表
|
|
66
|
-
cur.execute("SELECT table_name, table_comment, table_rows "
|
|
67
|
-
"FROM information_schema.tables WHERE table_schema=%s", (db,))
|
|
68
|
-
for r in cur.fetchall():
|
|
69
|
-
tables.append({
|
|
70
|
-
'repo_id': repo_id, 'schema_name': db,
|
|
71
|
-
'table_name': r['table_name'], 'table_comment': r.get('table_comment') or '',
|
|
72
|
-
'rows_count': r.get('table_rows'),
|
|
73
|
-
})
|
|
74
|
-
# 列
|
|
75
|
-
cur.execute("SELECT table_name, column_name, data_type, is_nullable, "
|
|
76
|
-
"column_comment, ordinal_position "
|
|
77
|
-
"FROM information_schema.columns WHERE table_schema=%s "
|
|
78
|
-
"ORDER BY table_name, ordinal_position", (db,))
|
|
79
|
-
for r in cur.fetchall():
|
|
80
|
-
columns.append({
|
|
81
|
-
'repo_id': repo_id, 'schema_name': db,
|
|
82
|
-
'table_name': r['table_name'], 'column_name': r['column_name'],
|
|
83
|
-
'data_type': r['data_type'], 'is_nullable': r['is_nullable'],
|
|
84
|
-
'column_comment': r.get('column_comment') or '',
|
|
85
|
-
'ordinal_position': r.get('ordinal_position'),
|
|
86
|
-
})
|
|
87
|
-
logger(' [db] 完成: %d 表 / %d 列' % (len(tables), len(columns)))
|
|
88
|
-
finally:
|
|
89
|
-
conn.close()
|
|
90
|
-
return tables, columns
|
|
@@ -1,206 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""D2: 补社区摘要 + 摘要向量。
|
|
3
|
-
|
|
4
|
-
对 graph_communities 表里 summary 为空的社区:
|
|
5
|
-
① 读 member_ids → 取成员的 canonical+cn → LLM 生成摘要
|
|
6
|
-
② 对摘要算 embedding → summary_vec
|
|
7
|
-
|
|
8
|
-
独立于现有 summarize_communities (不改动它), 直接补 PG。
|
|
9
|
-
"""
|
|
10
|
-
from __future__ import annotations
|
|
11
|
-
import os
|
|
12
|
-
import sys
|
|
13
|
-
import json
|
|
14
|
-
import time
|
|
15
|
-
import urllib.request
|
|
16
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
17
|
-
|
|
18
|
-
_THIS = os.path.dirname(os.path.abspath(__file__))
|
|
19
|
-
for _i in range(8):
|
|
20
|
-
_p = os.path.dirname(_THIS)
|
|
21
|
-
if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
|
|
22
|
-
sys.path.insert(0, _p); break
|
|
23
|
-
_THIS = _p
|
|
24
|
-
|
|
25
|
-
_SUMMARY_PROMPT = """你是软件架构师。下面是一个代码社区(聚类在一起的相关功能)的成员列表。
|
|
26
|
-
请用1-2句话概括这个社区负责什么业务功能。
|
|
27
|
-
|
|
28
|
-
成员列表(类名/端点/按钮, 括号内是中文含义):
|
|
29
|
-
%s
|
|
30
|
-
|
|
31
|
-
只返回摘要文本, 不要解释。"""
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def _call_qwen(prompt: str, timeout: int = 20) -> str:
|
|
35
|
-
key = os.environ.get('DASHSCOPE_API_KEY', '')
|
|
36
|
-
base = os.environ.get('ALI_CHAT_BASE', '').rstrip('/')
|
|
37
|
-
model = os.environ.get('ALI_CHAT_MODEL', 'qwen-plus')
|
|
38
|
-
if not key or not base:
|
|
39
|
-
return ''
|
|
40
|
-
req = urllib.request.Request(
|
|
41
|
-
base + '/chat/completions',
|
|
42
|
-
data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': prompt}],
|
|
43
|
-
'max_tokens': 200, 'temperature': 0.2}).encode('utf-8'),
|
|
44
|
-
headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
|
|
45
|
-
)
|
|
46
|
-
try:
|
|
47
|
-
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
48
|
-
return json.loads(resp.read())['choices'][0]['message']['content'].strip()
|
|
49
|
-
except Exception:
|
|
50
|
-
return ''
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def _embed(text: str) -> list:
|
|
54
|
-
"""调 DashScope embedding, 返回 1024 维向量。"""
|
|
55
|
-
key = os.environ.get('DASHSCOPE_API_KEY', '')
|
|
56
|
-
if not key or not text:
|
|
57
|
-
return []
|
|
58
|
-
req = urllib.request.Request(
|
|
59
|
-
'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
|
|
60
|
-
data=json.dumps({'model': 'text-embedding-v3', 'input': [text[:500]], 'dimensions': 1024}).encode('utf-8'),
|
|
61
|
-
headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
|
|
62
|
-
)
|
|
63
|
-
try:
|
|
64
|
-
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
65
|
-
return json.loads(resp.read())['data'][0]['embedding']
|
|
66
|
-
except Exception:
|
|
67
|
-
return []
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def summarize_missing(limit: int = 2500, do_embed: bool = True, logger=print):
|
|
71
|
-
"""补全缺摘要的社区。
|
|
72
|
-
|
|
73
|
-
Args:
|
|
74
|
-
limit: 上限 (默认 2500, 覆盖全部 2206 个缺的)
|
|
75
|
-
do_embed: True 同时补 summary_vec
|
|
76
|
-
"""
|
|
77
|
-
from sqlalchemy import text
|
|
78
|
-
from domain.kg.extract.java import pg_upsert
|
|
79
|
-
|
|
80
|
-
engine, _ = pg_upsert._get_pg_engine()
|
|
81
|
-
if engine is None:
|
|
82
|
-
return {'error': 'PG 不可用'}
|
|
83
|
-
|
|
84
|
-
# 取缺摘要的社区
|
|
85
|
-
with engine.connect() as conn:
|
|
86
|
-
rows = conn.execute(text("""
|
|
87
|
-
SELECT id, level, size, member_ids FROM graph_communities
|
|
88
|
-
WHERE summary IS NULL OR summary = ''
|
|
89
|
-
LIMIT :lim
|
|
90
|
-
"""), {'lim': limit}).all()
|
|
91
|
-
logger(' [d2] %d 个社区待补摘要' % len(rows))
|
|
92
|
-
if not rows:
|
|
93
|
-
return {'summarized': 0}
|
|
94
|
-
|
|
95
|
-
# 批量取成员的 canonical+cn (一次查)
|
|
96
|
-
all_member_ids = set()
|
|
97
|
-
for r in rows:
|
|
98
|
-
try:
|
|
99
|
-
mids = json.loads(r[3]) if isinstance(r[3], str) else (r[3] or [])
|
|
100
|
-
all_member_ids.update(mids[:20]) # 每社区最多取20个成员
|
|
101
|
-
except Exception:
|
|
102
|
-
pass
|
|
103
|
-
|
|
104
|
-
member_info = {} # {entity_id: "canonical(cn)"}
|
|
105
|
-
if all_member_ids:
|
|
106
|
-
member_list = list(all_member_ids)[:5000] # 限 5000 避免查询太大
|
|
107
|
-
# 分 chunk 查
|
|
108
|
-
for ci in range(0, len(member_list), 200):
|
|
109
|
-
chunk = member_list[ci:ci+200]
|
|
110
|
-
arr = "ARRAY[" + ','.join("'" + eid.replace("'", "''") + "'" for eid in chunk) + "]"
|
|
111
|
-
with engine.connect() as conn:
|
|
112
|
-
mrows = conn.execute(text(
|
|
113
|
-
f"SELECT id, canonical, cn FROM entities WHERE id = ANY({arr})"
|
|
114
|
-
)).all()
|
|
115
|
-
for eid, canonical, cn in mrows:
|
|
116
|
-
desc = canonical or eid
|
|
117
|
-
if cn:
|
|
118
|
-
desc += '(' + cn[:20] + ')'
|
|
119
|
-
member_info[eid] = desc[:50]
|
|
120
|
-
|
|
121
|
-
logger(' [d2] 成员信息: %d 条' % len(member_info))
|
|
122
|
-
|
|
123
|
-
# 批量生成摘要 (4线程并发)
|
|
124
|
-
def make_summary(row):
|
|
125
|
-
cid, level, size, member_ids_raw = row
|
|
126
|
-
try:
|
|
127
|
-
mids = json.loads(member_ids_raw) if isinstance(member_ids_raw, str) else (member_ids_raw or [])
|
|
128
|
-
except Exception:
|
|
129
|
-
mids = []
|
|
130
|
-
members_desc = [member_info.get(mid, mid) for mid in mids[:15]]
|
|
131
|
-
if not members_desc:
|
|
132
|
-
members_desc = [mid for mid in mids[:15]]
|
|
133
|
-
if not members_desc:
|
|
134
|
-
return (cid, '空社区')
|
|
135
|
-
prompt = _SUMMARY_PROMPT % '\n'.join(members_desc)
|
|
136
|
-
summary = _call_qwen(prompt)
|
|
137
|
-
if not summary:
|
|
138
|
-
# 兜底: 实体名拼接
|
|
139
|
-
summary = '、'.join(members_desc[:5]) + '等相关功能'
|
|
140
|
-
return (cid, summary[:200])
|
|
141
|
-
|
|
142
|
-
# 并发跑
|
|
143
|
-
t0 = time.time()
|
|
144
|
-
batches = [rows[i:i+10] for i in range(0, len(rows), 10)]
|
|
145
|
-
all_summaries = []
|
|
146
|
-
for bi in range(0, len(batches), 5):
|
|
147
|
-
chunk = batches[bi:bi+5]
|
|
148
|
-
with ThreadPoolExecutor(max_workers=4) as ex:
|
|
149
|
-
results = list(ex.map(make_summary, [r for b in chunk for r in b]))
|
|
150
|
-
all_summaries += results
|
|
151
|
-
if (bi//5+1) % 10 == 0:
|
|
152
|
-
logger(' [d2] %d/%d 社区摘要生成 (%d)' % (len(all_summaries), len(rows), len(all_summaries)))
|
|
153
|
-
|
|
154
|
-
logger(' [d2] 摘要生成完成: %d / %.1fs' % (len(all_summaries), time.time()-t0))
|
|
155
|
-
|
|
156
|
-
# 写 PG (分块提交)
|
|
157
|
-
ok = 0
|
|
158
|
-
vec_ok = 0
|
|
159
|
-
for i in range(0, len(all_summaries), 100):
|
|
160
|
-
chunk = all_summaries[i:i+100]
|
|
161
|
-
with engine.begin() as conn:
|
|
162
|
-
for cid, summary in chunk:
|
|
163
|
-
try:
|
|
164
|
-
# 算摘要向量 (可选)
|
|
165
|
-
vec_str = ''
|
|
166
|
-
if do_embed:
|
|
167
|
-
vec = _embed(summary)
|
|
168
|
-
if vec:
|
|
169
|
-
# summary_vec 存成 JSONB (兼容无 pgvector)
|
|
170
|
-
vec_str = json.dumps(vec)
|
|
171
|
-
if vec_str:
|
|
172
|
-
# summary_vec 是 ARRAY 类型, 用 FLOAT[] 写入
|
|
173
|
-
arr_str = '{' + ','.join(str(float(v)) for v in vec[:1024]) + '}'
|
|
174
|
-
conn.execute(text("""
|
|
175
|
-
UPDATE graph_communities SET summary=:s, summary_vec=CAST(:arr AS FLOAT[])
|
|
176
|
-
WHERE id=:id
|
|
177
|
-
"""), {'s': summary, 'arr': arr_str, 'id': cid})
|
|
178
|
-
vec_ok += 1
|
|
179
|
-
else:
|
|
180
|
-
conn.execute(text("""
|
|
181
|
-
UPDATE graph_communities SET summary=:s WHERE id=:id
|
|
182
|
-
"""), {'s': summary, 'id': cid})
|
|
183
|
-
ok += 1
|
|
184
|
-
except Exception:
|
|
185
|
-
pass
|
|
186
|
-
if (i//100+1) % 5 == 0:
|
|
187
|
-
logger(' [d2] 写入 %d/%d (vec=%d)' % (ok, len(all_summaries), vec_ok))
|
|
188
|
-
|
|
189
|
-
dt = time.time() - t0
|
|
190
|
-
logger(' [d2] 完成: %d 摘要 / %d 向量 / %.1fs' % (ok, vec_ok, dt))
|
|
191
|
-
return {'summarized': ok, 'vectorized': vec_ok, 'seconds': round(dt, 1)}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
if __name__ == '__main__':
|
|
195
|
-
import argparse
|
|
196
|
-
ap = argparse.ArgumentParser()
|
|
197
|
-
ap.add_argument('--limit', type=int, default=2500)
|
|
198
|
-
ap.add_argument('--no-embed', action='store_true')
|
|
199
|
-
args = ap.parse_args()
|
|
200
|
-
try:
|
|
201
|
-
from dotenv import load_dotenv
|
|
202
|
-
for p in ['wlinkj-workspace/backend/.env', '.env']:
|
|
203
|
-
if os.path.isfile(p): load_dotenv(p); break
|
|
204
|
-
except: pass
|
|
205
|
-
r = summarize_missing(limit=args.limit, do_embed=not args.no_embed)
|
|
206
|
-
print(json.dumps(r, ensure_ascii=False, indent=2))
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""批量灌入实体向量 → embeddings 表 (GOAL A 核心)。
|
|
3
|
-
|
|
4
|
-
对关键实体 (anchors + SERVICE + TABLE + PRD) 算 DashScope embedding,
|
|
5
|
-
存进 embeddings.vec_json (JSONB 数组, 不依赖 pgvector)。
|
|
6
|
-
|
|
7
|
-
被 build_goal_a.py 调用, 也可独立 CLI。
|
|
8
|
-
"""
|
|
9
|
-
from __future__ import annotations
|
|
10
|
-
import os
|
|
11
|
-
import sys
|
|
12
|
-
import json
|
|
13
|
-
import time
|
|
14
|
-
import urllib.request
|
|
15
|
-
|
|
16
|
-
_THIS = os.path.dirname(os.path.abspath(__file__))
|
|
17
|
-
for _i in range(8):
|
|
18
|
-
_p = os.path.dirname(_THIS)
|
|
19
|
-
if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
|
|
20
|
-
sys.path.insert(0, _p); break
|
|
21
|
-
_THIS = _p
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def embed_batch(texts: list, batch_size: int = 10) -> list:
|
|
25
|
-
"""批量调 DashScope text-embedding-v3 → [[float]]。失败返回空。"""
|
|
26
|
-
key = os.environ.get('DASHSCOPE_API_KEY', '')
|
|
27
|
-
if not key or not texts:
|
|
28
|
-
return []
|
|
29
|
-
all_vecs = []
|
|
30
|
-
for i in range(0, len(texts), batch_size):
|
|
31
|
-
batch = texts[i:i + batch_size]
|
|
32
|
-
payload = json.dumps({
|
|
33
|
-
'model': 'text-embedding-v3', 'input': batch, 'dimensions': 1024,
|
|
34
|
-
}).encode('utf-8')
|
|
35
|
-
req = urllib.request.Request(
|
|
36
|
-
'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
|
|
37
|
-
data=payload,
|
|
38
|
-
headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
|
|
39
|
-
)
|
|
40
|
-
try:
|
|
41
|
-
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
42
|
-
r = json.loads(resp.read())
|
|
43
|
-
for d in sorted(r['data'], key=lambda x: x['index']):
|
|
44
|
-
all_vecs.append(d['embedding'])
|
|
45
|
-
except Exception as e:
|
|
46
|
-
# 这批失败 → 填 None 占位 (保持顺序)
|
|
47
|
-
all_vecs.extend([None] * len(batch))
|
|
48
|
-
return all_vecs
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def build_embeddings_for_entities(entity_filter: str = None, limit: int = 500,
|
|
52
|
-
kinds: list = None, logger=print):
|
|
53
|
-
"""对实体算 embedding → 灌 embeddings.vec_json。
|
|
54
|
-
|
|
55
|
-
Args:
|
|
56
|
-
entity_filter: SQL WHERE 条件 (如 "type='ANCHOR'")
|
|
57
|
-
limit: 上限 (控成本)
|
|
58
|
-
kinds: 实体类型白名单 (默认 anchors + SERVICE + TABLE + PRD)
|
|
59
|
-
"""
|
|
60
|
-
from domain.kg.extract.java import pg_upsert
|
|
61
|
-
from sqlalchemy import text
|
|
62
|
-
|
|
63
|
-
engine, _ = pg_upsert._get_pg_engine()
|
|
64
|
-
if engine is None:
|
|
65
|
-
return {'error': 'PG 不可用'}
|
|
66
|
-
|
|
67
|
-
if kinds is None:
|
|
68
|
-
kinds = ['ANCHOR', 'SERVICE', 'TABLE', 'PRD', 'FEATURE']
|
|
69
|
-
|
|
70
|
-
# 取实体 (跳过已 embed 过的)
|
|
71
|
-
where = "WHERE e.type IN :kinds AND e.id NOT IN (SELECT entity_id FROM embeddings)"
|
|
72
|
-
if entity_filter:
|
|
73
|
-
where += " AND " + entity_filter
|
|
74
|
-
sql = f"""
|
|
75
|
-
SELECT e.id, e.repo_id, e.type, e.canonical, e.cn
|
|
76
|
-
FROM entities e
|
|
77
|
-
{where.replace(':kinds', ':kinds')}
|
|
78
|
-
LIMIT :lim
|
|
79
|
-
"""
|
|
80
|
-
# 兼容 IN 参数
|
|
81
|
-
with engine.connect() as conn:
|
|
82
|
-
rows = conn.execute(text(sql.replace(':kinds', "('" + "','".join(kinds) + "')")),
|
|
83
|
-
{'lim': limit}).all()
|
|
84
|
-
|
|
85
|
-
logger(' [embed] %d 实体待向量化 (类型=%s)' % (len(rows), kinds))
|
|
86
|
-
if not rows:
|
|
87
|
-
return {'embedded': 0}
|
|
88
|
-
|
|
89
|
-
# 批量 embed
|
|
90
|
-
descs = [(r[3] or r[4] or r[0])[:200] for r in rows] # canonical 优先
|
|
91
|
-
t0 = time.time()
|
|
92
|
-
vecs = embed_batch(descs)
|
|
93
|
-
embed_time = time.time() - t0
|
|
94
|
-
valid = sum(1 for v in vecs if v)
|
|
95
|
-
logger(' [embed] embedding 完成: %d/%d 有效 / %.1fs' % (valid, len(vecs), embed_time))
|
|
96
|
-
|
|
97
|
-
# 灌 PG
|
|
98
|
-
ok = 0
|
|
99
|
-
with engine.begin() as conn:
|
|
100
|
-
for (eid, repo_id, etype, canonical, cn), vec in zip(rows, vecs):
|
|
101
|
-
if not vec:
|
|
102
|
-
continue
|
|
103
|
-
try:
|
|
104
|
-
conn.execute(text("""
|
|
105
|
-
INSERT INTO embeddings (entity_id, repo_id, kind, content, vec_json, model, embedded_at)
|
|
106
|
-
VALUES (:id, :repo, :kind, :content, CAST(:vec AS JSONB), 'text-embedding-v3', now())
|
|
107
|
-
ON CONFLICT (entity_id) DO UPDATE SET
|
|
108
|
-
vec_json=EXCLUDED.vec_json, content=EXCLUDED.content, embedded_at=now()
|
|
109
|
-
"""), {
|
|
110
|
-
'id': eid, 'repo': repo_id, 'kind': etype,
|
|
111
|
-
'content': (canonical or cn or eid)[:200],
|
|
112
|
-
'vec': json.dumps(vec),
|
|
113
|
-
})
|
|
114
|
-
ok += 1
|
|
115
|
-
except Exception:
|
|
116
|
-
pass
|
|
117
|
-
logger(' [embed] 灌入 %d 条向量' % ok)
|
|
118
|
-
return {'embedded': ok, 'time_sec': round(embed_time, 1)}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if __name__ == '__main__':
|
|
122
|
-
import argparse
|
|
123
|
-
ap = argparse.ArgumentParser()
|
|
124
|
-
ap.add_argument('--limit', type=int, default=500)
|
|
125
|
-
args = ap.parse_args()
|
|
126
|
-
try:
|
|
127
|
-
from dotenv import load_dotenv
|
|
128
|
-
for p in ['wlinkj-workspace/backend/.env', '.env']:
|
|
129
|
-
if os.path.isfile(p): load_dotenv(p); break
|
|
130
|
-
except: pass
|
|
131
|
-
r = build_embeddings_for_entities(limit=args.limit)
|
|
132
|
-
print(json.dumps(r, ensure_ascii=False, indent=2))
|