@hupan56/wlkj 3.2.0 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/bin/cli.js +117 -0
  2. package/package.json +2 -2
  3. package/templates/.qoder/.runtime/hook-errors.log +4 -0
  4. package/templates/qoder/commands/optional/wl-insight.md +276 -276
  5. package/templates/qoder/commands/optional/wl-report.md +1 -1
  6. package/templates/qoder/commands/optional/wl-spec.md +13 -1
  7. package/templates/qoder/commands/optional/wl-status.md +14 -1
  8. package/templates/qoder/commands/wl-code.md +111 -4
  9. package/templates/qoder/commands/wl-commit.md +12 -1
  10. package/templates/qoder/commands/wl-design.md +69 -5
  11. package/templates/qoder/commands/wl-init.md +129 -102
  12. package/templates/qoder/commands/wl-prd.md +162 -6
  13. package/templates/qoder/commands/wl-search.md +106 -20
  14. package/templates/qoder/commands/wl-task.md +617 -613
  15. package/templates/qoder/commands/wl-test.md +20 -1
  16. package/templates/qoder/contracts/spec.md +4 -0
  17. package/templates/qoder/hooks/post-tool-use.py +71 -0
  18. package/templates/qoder/hooks/pre-tool-use.py +136 -0
  19. package/templates/qoder/hooks/session-start.py +365 -397
  20. package/templates/qoder/scripts/capability/__pycache__/__init__.cpython-39.pyc +0 -0
  21. package/templates/qoder/scripts/capability/__pycache__/present_html.cpython-39.pyc +0 -0
  22. package/templates/qoder/scripts/capability/__pycache__/registry.cpython-39.pyc +0 -0
  23. package/templates/qoder/scripts/capability/__pycache__/registry_mcp.cpython-39.pyc +0 -0
  24. package/templates/qoder/scripts/capability/adapters/__init__.py +1 -1
  25. package/templates/qoder/scripts/capability/adapters/__pycache__/__init__.cpython-39.pyc +0 -0
  26. package/templates/qoder/scripts/capability/adapters/__pycache__/cli.cpython-39.pyc +0 -0
  27. package/templates/qoder/scripts/capability/adapters/__pycache__/mcp.cpython-39.pyc +0 -0
  28. package/templates/qoder/scripts/capability/adapters/__pycache__/qw.cpython-39.pyc +0 -0
  29. package/templates/qoder/scripts/capability/adapters/mcp.py +76 -100
  30. package/templates/qoder/scripts/capability/adapters/qw.py +295 -295
  31. package/templates/qoder/scripts/capability/caps/__init__.py +5 -5
  32. package/templates/qoder/scripts/capability/caps/__pycache__/__init__.cpython-39.pyc +0 -0
  33. package/templates/qoder/scripts/capability/caps/__pycache__/context.cpython-39.pyc +0 -0
  34. package/templates/qoder/scripts/capability/caps/__pycache__/cron.cpython-39.pyc +0 -0
  35. package/templates/qoder/scripts/capability/caps/__pycache__/identity.cpython-39.pyc +0 -0
  36. package/templates/qoder/scripts/capability/caps/__pycache__/memory.cpython-39.pyc +0 -0
  37. package/templates/qoder/scripts/capability/caps/__pycache__/notify.cpython-39.pyc +0 -0
  38. package/templates/qoder/scripts/capability/caps/__pycache__/present.cpython-39.pyc +0 -0
  39. package/templates/qoder/scripts/capability/caps/__pycache__/repo.cpython-39.pyc +0 -0
  40. package/templates/qoder/scripts/capability/caps/__pycache__/sandbox.cpython-39.pyc +0 -0
  41. package/templates/qoder/scripts/capability/caps/memory.py +1 -1
  42. package/templates/qoder/scripts/capability/caps/notify.py +64 -0
  43. package/templates/qoder/scripts/capability/caps/sandbox.py +38 -0
  44. package/templates/qoder/scripts/capability/present_html.py +68 -0
  45. package/templates/qoder/scripts/capability/registry.py +27 -25
  46. package/templates/qoder/scripts/capability/registry_mcp.py +67 -5
  47. package/templates/qoder/scripts/capability/smoke_test_report.json +34 -20
  48. package/templates/qoder/scripts/deployment/setup/carriers.py +3 -1
  49. package/templates/qoder/scripts/deployment/setup/init_doctor.py +10 -3
  50. package/templates/qoder/scripts/domain/__pycache__/__init__.cpython-39.pyc +0 -0
  51. package/templates/qoder/scripts/domain/integration/__init__.py +0 -0
  52. package/templates/qoder/scripts/domain/integration/__pycache__/__init__.cpython-39.pyc +0 -0
  53. package/templates/qoder/scripts/domain/integration/__pycache__/return_to_platform.cpython-39.pyc +0 -0
  54. package/templates/qoder/scripts/domain/integration/return_to_platform.py +395 -0
  55. package/templates/qoder/scripts/domain/integration/spec_upload.py +208 -0
  56. package/templates/qoder/scripts/domain/kg/build/kg_build.py +27 -3
  57. package/templates/qoder/scripts/domain/kg/extract/asset/__init__.py +10 -0
  58. package/templates/qoder/scripts/domain/kg/extract/asset/asset_tree.py +57 -0
  59. package/templates/qoder/scripts/domain/kg/extract/asset/discussion_importer.py +62 -0
  60. package/templates/qoder/scripts/domain/kg/extract/asset/prd_importer.py +146 -0
  61. package/templates/qoder/scripts/domain/kg/extract/asset/prototype_importer.py +64 -0
  62. package/templates/qoder/scripts/domain/kg/extract/asset/returns_importer.py +52 -0
  63. package/templates/qoder/scripts/domain/kg/extract/build_goal3.py +104 -0
  64. package/templates/qoder/scripts/domain/kg/extract/build_goal4.py +55 -0
  65. package/templates/qoder/scripts/domain/kg/extract/build_goal5.py +95 -0
  66. package/templates/qoder/scripts/domain/kg/extract/db/__init__.py +8 -0
  67. package/templates/qoder/scripts/domain/kg/extract/db/data_profile.py +22 -0
  68. package/templates/qoder/scripts/domain/kg/extract/db/fk_extractor.py +55 -0
  69. package/templates/qoder/scripts/domain/kg/extract/db/schema_extractor.py +90 -0
  70. package/templates/qoder/scripts/domain/kg/extract/extract.py +84 -0
  71. package/templates/qoder/scripts/domain/kg/extract/inference/__init__.py +9 -0
  72. package/templates/qoder/scripts/domain/kg/extract/inference/community_summarizer.py +206 -0
  73. package/templates/qoder/scripts/domain/kg/extract/inference/embed_builder.py +132 -0
  74. package/templates/qoder/scripts/domain/kg/extract/inference/naming_matcher.py +80 -0
  75. package/templates/qoder/scripts/domain/kg/extract/inference/promote.py +59 -0
  76. package/templates/qoder/scripts/domain/kg/extract/inference/recompute.py +93 -0
  77. package/templates/qoder/scripts/domain/kg/extract/inference/weak_link.py +421 -0
  78. package/templates/qoder/scripts/domain/kg/extract/java/__init__.py +15 -0
  79. package/templates/qoder/scripts/domain/kg/extract/java/_parser.py +271 -0
  80. package/templates/qoder/scripts/domain/kg/extract/java/all.py +145 -0
  81. package/templates/qoder/scripts/domain/kg/extract/java/build_java_to_pg.py +102 -0
  82. package/templates/qoder/scripts/domain/kg/extract/java/call_chain.py +49 -0
  83. package/templates/qoder/scripts/domain/kg/extract/java/class_extractor.py +141 -0
  84. package/templates/qoder/scripts/domain/kg/extract/java/domain_extractor.py +148 -0
  85. package/templates/qoder/scripts/domain/kg/extract/java/dubbo_extractor.py +33 -0
  86. package/templates/qoder/scripts/domain/kg/extract/java/endpoint_extractor.py +36 -0
  87. package/templates/qoder/scripts/domain/kg/extract/java/javadoc_extractor.py +110 -0
  88. package/templates/qoder/scripts/domain/kg/extract/java/llm_cn_filler.py +150 -0
  89. package/templates/qoder/scripts/domain/kg/extract/java/member_extractor.py +157 -0
  90. package/templates/qoder/scripts/domain/kg/extract/java/mybatisplus_extractor.py +34 -0
  91. package/templates/qoder/scripts/domain/kg/extract/java/pg_upsert.py +165 -0
  92. package/templates/qoder/scripts/domain/kg/extract/java/satoken_extractor.py +30 -0
  93. package/templates/qoder/scripts/domain/kg/extract/java/spring_extractor.py +39 -0
  94. package/templates/qoder/scripts/domain/kg/extract/java/validation_extractor.py +33 -0
  95. package/templates/qoder/scripts/domain/kg/extract/mybatis/__init__.py +9 -0
  96. package/templates/qoder/scripts/domain/kg/extract/mybatis/all.py +79 -0
  97. package/templates/qoder/scripts/domain/kg/extract/mybatis/mapper_parser.py +99 -0
  98. package/templates/qoder/scripts/domain/kg/extract/mybatis/relation_builder.py +69 -0
  99. package/templates/qoder/scripts/domain/kg/extract/mybatis/sql_extractor.py +78 -0
  100. package/templates/qoder/scripts/domain/kg/extract/prd/__init__.py +8 -0
  101. package/templates/qoder/scripts/domain/kg/extract/prd/prd_chunk_embed.py +105 -0
  102. package/templates/qoder/scripts/domain/kg/extract/prd/prd_llm_extract.py +153 -0
  103. package/templates/qoder/scripts/domain/kg/extract/prd/req_anchor.py +120 -0
  104. package/templates/qoder/scripts/domain/kg/graph/kg_semantic.py +4 -2
  105. package/templates/qoder/scripts/domain/kg/kg.py +42 -5
  106. package/templates/qoder/scripts/domain/kg/search/_remote.py +187 -0
  107. package/templates/qoder/scripts/domain/kg/search/context_pack.py +32 -2
  108. package/templates/qoder/scripts/domain/kg/search/search_index.py +74 -20
  109. package/templates/qoder/scripts/domain/kg/switch_project.py +158 -0
  110. package/templates/qoder/scripts/domain/kg/sync_repowiki.py +109 -0
  111. package/templates/qoder/scripts/domain/task/__pycache__/wlkj_panel.cpython-39.pyc +0 -0
  112. package/templates/qoder/scripts/domain/task/{zentao_panel.py → wlkj_panel.py} +315 -53
  113. package/templates/qoder/scripts/engine/poller.py +219 -0
  114. package/templates/qoder/scripts/foundation/__pycache__/__init__.cpython-39.pyc +0 -0
  115. package/templates/qoder/scripts/foundation/core/__pycache__/__init__.cpython-39.pyc +0 -0
  116. package/templates/qoder/scripts/foundation/core/__pycache__/paths.cpython-39.pyc +0 -0
  117. package/templates/qoder/scripts/foundation/integrations/active_task.py +2 -1
  118. package/templates/qoder/scripts/orchestration/wlkj.py +4 -0
  119. package/templates/qoder/scripts/protocol/__pycache__/__init__.cpython-39.pyc +0 -0
  120. package/templates/qoder/scripts/protocol/mcp/zentao_mcp_server.py +1 -1
  121. package/templates/qoder/scripts/protocol/transports/__pycache__/__init__.cpython-39.pyc +0 -0
  122. package/templates/qoder/scripts/protocol/transports/__pycache__/base.cpython-39.pyc +0 -0
  123. package/templates/qoder/scripts/protocol/transports/__pycache__/cli.cpython-39.pyc +0 -0
  124. package/templates/qoder/scripts/protocol/transports/__pycache__/http.cpython-39.pyc +0 -0
  125. package/templates/qoder/scripts/protocol/transports/__pycache__/stdio.cpython-39.pyc +0 -0
  126. package/templates/qoder/scripts/protocol/transports/http.py +7 -1
  127. package/templates/qoder/scripts/validation/metrics/__pycache__/present_board.cpython-39.pyc +0 -0
  128. package/templates/qoder/scripts/validation/metrics/present_board.py +180 -0
  129. package/templates/qoder/settings.json +10 -0
  130. package/templates/root/AGENTS.md +9 -10
@@ -0,0 +1,55 @@
1
+ # -*- coding: utf-8 -*-
2
+ """外键关系提取器: KEY_COLUMN_USAGE → references 边。
3
+
4
+ 让 table_impact 真正能做"改表影响谁"的跨表追溯。
5
+ """
6
+ from __future__ import annotations
7
+ from . import schema_extractor as _se
8
+
9
+
10
+ def extract_foreign_keys(repo_id: str = 'fywl-ics', logger=print):
11
+ """连 MySQL → 提取外键关系 → references 边。
12
+
13
+ Returns: [{from_id, to_id, edge_type:'references', ...}] 连不上返回 []
14
+ """
15
+ if not _se._can_connect():
16
+ return []
17
+ try:
18
+ import pymysql
19
+ except ImportError:
20
+ return []
21
+
22
+ conn = pymysql.connect(host=_se.MYSQL_HOST, port=_se.MYSQL_PORT, user=_se.MYSQL_USER,
23
+ password=_se.MYSQL_PASSWORD, charset='utf8mb4',
24
+ cursorclass=pymysql.cursors.DictCursor,
25
+ connect_timeout=5, read_timeout=30)
26
+ edges = []
27
+ try:
28
+ cur = conn.cursor()
29
+ cur.execute("SELECT schema_name FROM information_schema.schemata "
30
+ "WHERE schema_name LIKE %s", (_se.DB_NAME_PREFIX + '%',))
31
+ dbs = [r['schema_name'] for r in cur.fetchall()]
32
+ for db in dbs:
33
+ # 外键关系
34
+ cur.execute("""
35
+ SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
36
+ FROM information_schema.KEY_COLUMN_USAGE
37
+ WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL
38
+ """, (db,))
39
+ for r in cur.fetchall():
40
+ from_tbl = r['TABLE_NAME'].lower()
41
+ to_tbl = r['REFERENCED_TABLE_NAME'].lower()
42
+ edges.append({
43
+ 'from_id': 'TBL:%s:%s' % (repo_id, from_tbl),
44
+ 'to_id': 'TBL:%s:%s' % (repo_id, to_tbl),
45
+ 'edge_type': 'references',
46
+ 'from_repo': repo_id, 'to_repo': repo_id,
47
+ 'confidence': 1.0, 'source': 'db_fk',
48
+ 'props': {'column': r['COLUMN_NAME'],
49
+ 'ref_column': r['REFERENCED_COLUMN_NAME'],
50
+ 'schema': db},
51
+ })
52
+ logger(' [db-fk] %d 条外键关系' % len(edges))
53
+ finally:
54
+ conn.close()
55
+ return edges
@@ -0,0 +1,90 @@
1
+ # -*- coding: utf-8 -*-
2
+ """连 MySQL information_schema → 灌 db_tables / db_columns。
3
+
4
+ 复用 kg_build_db.py 的 MySQL 连接配置 (内网 10.54.6.9, fywl_ics_* 库)。
5
+ 连不上内网时 best-effort 返回空, 不阻塞 (GOAL 3 主链路是 mapper XML)。
6
+ """
7
+ from __future__ import annotations
8
+ import os
9
+ import sys
10
+ import socket
11
+
12
+ _THIS = os.path.dirname(os.path.abspath(__file__))
13
+ for _i in range(8):
14
+ _p = os.path.dirname(_THIS)
15
+ if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
16
+ sys.path.insert(0, _p); break
17
+ _THIS = _p
18
+
19
+ # 复用 kg_build_db 的配置
20
+ MYSQL_HOST = os.environ.get('MYSQL_HOST', '10.54.6.9')
21
+ MYSQL_PORT = int(os.environ.get('MYSQL_PORT', '3306'))
22
+ MYSQL_USER = os.environ.get('MYSQL_USER', 'fywl_ics_test')
23
+ MYSQL_PASSWORD = os.environ.get('MYSQL_PASSWORD', 'Fywl_ics_test56476')
24
+ DB_NAME_PREFIX = os.environ.get('DB_NAME_PREFIX', 'fywl_ics_')
25
+
26
+
27
+ def _can_connect(timeout=3):
28
+ """探测内网 MySQL 是否可达。"""
29
+ try:
30
+ with socket.create_connection((MYSQL_HOST, MYSQL_PORT), timeout=timeout):
31
+ return True
32
+ except Exception:
33
+ return False
34
+
35
+
36
+ def extract_all_schema(repo_id: str = 'fywl-ics', logger=print):
37
+ """连 MySQL → 灌 db_tables + db_columns。
38
+
39
+ Returns: (tables: list[dict], columns: list[dict]) 连不上返回 ([], [])
40
+ """
41
+ if not _can_connect():
42
+ logger(' [db] ⚠️ 内网 MySQL 不可达 (%s:%s), 跳过 schema 提取' % (MYSQL_HOST, MYSQL_PORT))
43
+ return [], []
44
+ try:
45
+ import pymysql
46
+ except ImportError:
47
+ logger(' [db] pymysql 未装, 跳过')
48
+ return [], []
49
+
50
+ logger(' [db] 连接 MySQL %s:%s ...' % (MYSQL_HOST, MYSQL_PORT))
51
+ conn = pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER,
52
+ password=MYSQL_PASSWORD, charset='utf8mb4',
53
+ cursorclass=pymysql.cursors.DictCursor,
54
+ connect_timeout=5, read_timeout=30)
55
+ tables, columns = [], []
56
+ try:
57
+ cur = conn.cursor()
58
+ cur.execute("SELECT schema_name FROM information_schema.schemata "
59
+ "WHERE schema_name LIKE %s ORDER BY schema_name",
60
+ (DB_NAME_PREFIX + '%'))
61
+ dbs = [r['schema_name'] for r in cur.fetchall()]
62
+ logger(' [db] 发现 %d 个业务库: %s' % (len(dbs), dbs[:3]))
63
+
64
+ for db in dbs:
65
+ # 表
66
+ cur.execute("SELECT table_name, table_comment, table_rows "
67
+ "FROM information_schema.tables WHERE table_schema=%s", (db,))
68
+ for r in cur.fetchall():
69
+ tables.append({
70
+ 'repo_id': repo_id, 'schema_name': db,
71
+ 'table_name': r['table_name'], 'table_comment': r.get('table_comment') or '',
72
+ 'rows_count': r.get('table_rows'),
73
+ })
74
+ # 列
75
+ cur.execute("SELECT table_name, column_name, data_type, is_nullable, "
76
+ "column_comment, ordinal_position "
77
+ "FROM information_schema.columns WHERE table_schema=%s "
78
+ "ORDER BY table_name, ordinal_position", (db,))
79
+ for r in cur.fetchall():
80
+ columns.append({
81
+ 'repo_id': repo_id, 'schema_name': db,
82
+ 'table_name': r['table_name'], 'column_name': r['column_name'],
83
+ 'data_type': r['data_type'], 'is_nullable': r['is_nullable'],
84
+ 'column_comment': r.get('column_comment') or '',
85
+ 'ordinal_position': r.get('ordinal_position'),
86
+ })
87
+ logger(' [db] 完成: %d 表 / %d 列' % (len(tables), len(columns)))
88
+ finally:
89
+ conn.close()
90
+ return tables, columns
@@ -428,3 +428,87 @@ def trace_clicks(vue_path, vue_content, api_table):
428
428
  traces.append(trace)
429
429
 
430
430
  return traces
431
+
432
+
433
+ # ============================================================
434
+ # GOAL 3 泛化: Carmg-H5 (Vant/H5) 风格 API + 调用提取
435
+ # ============================================================
436
+ # Carmg-H5 的 API 定义方式和 fywl-ui 不同:
437
+ # fywl-ui: enum Api { X='/url' } + requestClient.post(Api.X)
438
+ # Carmg-H5: export function fn(data){ const params={url:'/x',method:'post'}; return fetch(params) }
439
+ # 以及 van-button 的 @click/@tap 事件。
440
+ # 下面两个函数让 extract.py 能同时处理两种风格。
441
+
442
+ # Carmg-H5 API: export function fnName(...) { ... url: '/xxx' ... method: 'post' ... }
443
+ RE_H5_API_FN = re.compile(
444
+ r"export\s+function\s+(\w+)\s*\([^)]*\)\s*\{(?:(?!\}).)*?"
445
+ r"url:\s*['\"]([^'\"]+)['\"](?:(?!\}).)*?"
446
+ r"(?:method:\s*['\"](\w+)['\"])?",
447
+ re.DOTALL
448
+ )
449
+
450
+
451
+ def build_api_fn_table_h5(api_dir):
452
+ """扫 src/api/**/*.js (Carmg-H5 风格) → {fnName: {url, verb, file}}。
453
+
454
+ 识别 export function + url: 字面量模式。
455
+ """
456
+ table = {}
457
+ if not os.path.isdir(api_dir):
458
+ return table
459
+ for root, dirs, files in os.walk(api_dir):
460
+ dirs[:] = [d for d in dirs if d not in ('dist', 'node_modules', '__pycache__')]
461
+ for f in files:
462
+ if not (f.endswith('.js') or f.endswith('.ts')):
463
+ continue
464
+ fp = os.path.join(root, f)
465
+ try:
466
+ content = open(fp, encoding='utf-8', errors='ignore').read()
467
+ except Exception:
468
+ continue
469
+ for m in RE_H5_API_FN.finditer(content):
470
+ fn_name, url, verb = m.group(1), m.group(2), m.group(3) or 'get'
471
+ table[fn_name] = {'url': url, 'verb': verb.lower(), 'file': fp}
472
+ return table
473
+
474
+
475
+ def trace_clicks_h5(vue_path, vue_content, api_table):
476
+ """Vant/H5 风格的 click→api 追踪。
477
+
478
+ 和 trace_clicks 区别:
479
+ - 同时认 @click 和 @tap (Vant 用 @tap)
480
+ - API 调用是 import { fn } from '@/api/xx' + fn(args) (不是 requestClient.xx)
481
+ """
482
+ if not api_table:
483
+ return []
484
+ traces = []
485
+ # 收集 @click / @tap
486
+ clicks = list(re.finditer(r"@(?:click|tap)(?:\.\w+)?\s*=\s*['\"]([^'\"]+)['\"]", vue_content))
487
+ if not clicks:
488
+ return []
489
+ # 收集 import { fn1, fn2 } from '@/api/...'
490
+ imported = set()
491
+ for m in re.finditer(r"import\s*\{([^}]+)\}\s*from\s*['\"][^'\"]*api[^'\"]*['\"]", vue_content):
492
+ for n in re.split(r'[,\s]+', m.group(1)):
493
+ n = n.strip()
494
+ if n:
495
+ imported.add(n)
496
+ # script 里直接调用的 api 函数 (imported 集合内的)
497
+ for click in clicks:
498
+ handler = click.group(1).split('(')[0].strip()
499
+ if not handler:
500
+ continue
501
+ # 在 script 里找 handler 函数体, 看它调了哪些 imported api
502
+ handler_body = _get_fn_body(vue_content, handler)
503
+ if not handler_body:
504
+ continue
505
+ for api_fn in imported:
506
+ if api_fn in api_table and re.search(r'\b' + re.escape(api_fn) + r'\s*\(', handler_body):
507
+ info = api_table[api_fn]
508
+ traces.append({
509
+ 'vue_file': vue_path, 'button_text': handler,
510
+ 'handler': handler, 'api_fn': api_fn,
511
+ 'endpoint': info['url'], 'verb': info['verb'],
512
+ 'api_file': info['file'], 'confidence': 'high',
513
+ })
514
+ return traces
@@ -0,0 +1,9 @@
1
+ # -*- coding: utf-8 -*-
2
+ """L3 推断层 (GOAL 5): 跨源锚点关联。
3
+
4
+ 子模块:
5
+ weak_link embedding 弱关联推理 (向量召回, 需 pgvector; 不可用时降级命名匹配)
6
+ naming_matcher 命名同源匹配 (AssetController ↔ asset ↔ 资产管理)
7
+ recompute C 方案指纹重算 (30天过期弱边对比指纹)
8
+ promote 弱边升格 (出现强关联时弱边删除)
9
+ """
@@ -0,0 +1,206 @@
1
+ # -*- coding: utf-8 -*-
2
+ """D2: 补社区摘要 + 摘要向量。
3
+
4
+ 对 graph_communities 表里 summary 为空的社区:
5
+ ① 读 member_ids → 取成员的 canonical+cn → LLM 生成摘要
6
+ ② 对摘要算 embedding → summary_vec
7
+
8
+ 独立于现有 summarize_communities (不改动它), 直接补 PG。
9
+ """
10
+ from __future__ import annotations
11
+ import os
12
+ import sys
13
+ import json
14
+ import time
15
+ import urllib.request
16
+ from concurrent.futures import ThreadPoolExecutor
17
+
18
+ _THIS = os.path.dirname(os.path.abspath(__file__))
19
+ for _i in range(8):
20
+ _p = os.path.dirname(_THIS)
21
+ if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
22
+ sys.path.insert(0, _p); break
23
+ _THIS = _p
24
+
25
+ _SUMMARY_PROMPT = """你是软件架构师。下面是一个代码社区(聚类在一起的相关功能)的成员列表。
26
+ 请用1-2句话概括这个社区负责什么业务功能。
27
+
28
+ 成员列表(类名/端点/按钮, 括号内是中文含义):
29
+ %s
30
+
31
+ 只返回摘要文本, 不要解释。"""
32
+
33
+
34
+ def _call_qwen(prompt: str, timeout: int = 20) -> str:
35
+ key = os.environ.get('DASHSCOPE_API_KEY', '')
36
+ base = os.environ.get('ALI_CHAT_BASE', '').rstrip('/')
37
+ model = os.environ.get('ALI_CHAT_MODEL', 'qwen-plus')
38
+ if not key or not base:
39
+ return ''
40
+ req = urllib.request.Request(
41
+ base + '/chat/completions',
42
+ data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': prompt}],
43
+ 'max_tokens': 200, 'temperature': 0.2}).encode('utf-8'),
44
+ headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
45
+ )
46
+ try:
47
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
48
+ return json.loads(resp.read())['choices'][0]['message']['content'].strip()
49
+ except Exception:
50
+ return ''
51
+
52
+
53
+ def _embed(text: str) -> list:
54
+ """调 DashScope embedding, 返回 1024 维向量。"""
55
+ key = os.environ.get('DASHSCOPE_API_KEY', '')
56
+ if not key or not text:
57
+ return []
58
+ req = urllib.request.Request(
59
+ 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
60
+ data=json.dumps({'model': 'text-embedding-v3', 'input': [text[:500]], 'dimensions': 1024}).encode('utf-8'),
61
+ headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
62
+ )
63
+ try:
64
+ with urllib.request.urlopen(req, timeout=15) as resp:
65
+ return json.loads(resp.read())['data'][0]['embedding']
66
+ except Exception:
67
+ return []
68
+
69
+
70
+ def summarize_missing(limit: int = 2500, do_embed: bool = True, logger=print):
71
+ """补全缺摘要的社区。
72
+
73
+ Args:
74
+ limit: 上限 (默认 2500, 覆盖全部 2206 个缺的)
75
+ do_embed: True 同时补 summary_vec
76
+ """
77
+ from sqlalchemy import text
78
+ from domain.kg.extract.java import pg_upsert
79
+
80
+ engine, _ = pg_upsert._get_pg_engine()
81
+ if engine is None:
82
+ return {'error': 'PG 不可用'}
83
+
84
+ # 取缺摘要的社区
85
+ with engine.connect() as conn:
86
+ rows = conn.execute(text("""
87
+ SELECT id, level, size, member_ids FROM graph_communities
88
+ WHERE summary IS NULL OR summary = ''
89
+ LIMIT :lim
90
+ """), {'lim': limit}).all()
91
+ logger(' [d2] %d 个社区待补摘要' % len(rows))
92
+ if not rows:
93
+ return {'summarized': 0}
94
+
95
+ # 批量取成员的 canonical+cn (一次查)
96
+ all_member_ids = set()
97
+ for r in rows:
98
+ try:
99
+ mids = json.loads(r[3]) if isinstance(r[3], str) else (r[3] or [])
100
+ all_member_ids.update(mids[:20]) # 每社区最多取20个成员
101
+ except Exception:
102
+ pass
103
+
104
+ member_info = {} # {entity_id: "canonical(cn)"}
105
+ if all_member_ids:
106
+ member_list = list(all_member_ids)[:5000] # 限 5000 避免查询太大
107
+ # 分 chunk 查
108
+ for ci in range(0, len(member_list), 200):
109
+ chunk = member_list[ci:ci+200]
110
+ arr = "ARRAY[" + ','.join("'" + eid.replace("'", "''") + "'" for eid in chunk) + "]"
111
+ with engine.connect() as conn:
112
+ mrows = conn.execute(text(
113
+ f"SELECT id, canonical, cn FROM entities WHERE id = ANY({arr})"
114
+ )).all()
115
+ for eid, canonical, cn in mrows:
116
+ desc = canonical or eid
117
+ if cn:
118
+ desc += '(' + cn[:20] + ')'
119
+ member_info[eid] = desc[:50]
120
+
121
+ logger(' [d2] 成员信息: %d 条' % len(member_info))
122
+
123
+ # 批量生成摘要 (4线程并发)
124
+ def make_summary(row):
125
+ cid, level, size, member_ids_raw = row
126
+ try:
127
+ mids = json.loads(member_ids_raw) if isinstance(member_ids_raw, str) else (member_ids_raw or [])
128
+ except Exception:
129
+ mids = []
130
+ members_desc = [member_info.get(mid, mid) for mid in mids[:15]]
131
+ if not members_desc:
132
+ members_desc = [mid for mid in mids[:15]]
133
+ if not members_desc:
134
+ return (cid, '空社区')
135
+ prompt = _SUMMARY_PROMPT % '\n'.join(members_desc)
136
+ summary = _call_qwen(prompt)
137
+ if not summary:
138
+ # 兜底: 实体名拼接
139
+ summary = '、'.join(members_desc[:5]) + '等相关功能'
140
+ return (cid, summary[:200])
141
+
142
+ # 并发跑
143
+ t0 = time.time()
144
+ batches = [rows[i:i+10] for i in range(0, len(rows), 10)]
145
+ all_summaries = []
146
+ for bi in range(0, len(batches), 5):
147
+ chunk = batches[bi:bi+5]
148
+ with ThreadPoolExecutor(max_workers=4) as ex:
149
+ results = list(ex.map(make_summary, [r for b in chunk for r in b]))
150
+ all_summaries += results
151
+ if (bi//5+1) % 10 == 0:
152
+ logger(' [d2] %d/%d 社区摘要生成 (%d)' % (len(all_summaries), len(rows), len(all_summaries)))
153
+
154
+ logger(' [d2] 摘要生成完成: %d / %.1fs' % (len(all_summaries), time.time()-t0))
155
+
156
+ # 写 PG (分块提交)
157
+ ok = 0
158
+ vec_ok = 0
159
+ for i in range(0, len(all_summaries), 100):
160
+ chunk = all_summaries[i:i+100]
161
+ with engine.begin() as conn:
162
+ for cid, summary in chunk:
163
+ try:
164
+ # 算摘要向量 (可选)
165
+ vec_str = ''
166
+ if do_embed:
167
+ vec = _embed(summary)
168
+ if vec:
169
+ # summary_vec 存成 JSONB (兼容无 pgvector)
170
+ vec_str = json.dumps(vec)
171
+ if vec_str:
172
+ # summary_vec 是 ARRAY 类型, 用 FLOAT[] 写入
173
+ arr_str = '{' + ','.join(str(float(v)) for v in vec[:1024]) + '}'
174
+ conn.execute(text("""
175
+ UPDATE graph_communities SET summary=:s, summary_vec=CAST(:arr AS FLOAT[])
176
+ WHERE id=:id
177
+ """), {'s': summary, 'arr': arr_str, 'id': cid})
178
+ vec_ok += 1
179
+ else:
180
+ conn.execute(text("""
181
+ UPDATE graph_communities SET summary=:s WHERE id=:id
182
+ """), {'s': summary, 'id': cid})
183
+ ok += 1
184
+ except Exception:
185
+ pass
186
+ if (i//100+1) % 5 == 0:
187
+ logger(' [d2] 写入 %d/%d (vec=%d)' % (ok, len(all_summaries), vec_ok))
188
+
189
+ dt = time.time() - t0
190
+ logger(' [d2] 完成: %d 摘要 / %d 向量 / %.1fs' % (ok, vec_ok, dt))
191
+ return {'summarized': ok, 'vectorized': vec_ok, 'seconds': round(dt, 1)}
192
+
193
+
194
+ if __name__ == '__main__':
195
+ import argparse
196
+ ap = argparse.ArgumentParser()
197
+ ap.add_argument('--limit', type=int, default=2500)
198
+ ap.add_argument('--no-embed', action='store_true')
199
+ args = ap.parse_args()
200
+ try:
201
+ from dotenv import load_dotenv
202
+ for p in ['wlinkj-workspace/backend/.env', '.env']:
203
+ if os.path.isfile(p): load_dotenv(p); break
204
+ except: pass
205
+ r = summarize_missing(limit=args.limit, do_embed=not args.no_embed)
206
+ print(json.dumps(r, ensure_ascii=False, indent=2))
@@ -0,0 +1,132 @@
1
+ # -*- coding: utf-8 -*-
2
+ """批量灌入实体向量 → embeddings 表 (GOAL A 核心)。
3
+
4
+ 对关键实体 (anchors + SERVICE + TABLE + PRD) 算 DashScope embedding,
5
+ 存进 embeddings.vec_json (JSONB 数组, 不依赖 pgvector)。
6
+
7
+ 被 build_goal_a.py 调用, 也可独立 CLI。
8
+ """
9
+ from __future__ import annotations
10
+ import os
11
+ import sys
12
+ import json
13
+ import time
14
+ import urllib.request
15
+
16
+ _THIS = os.path.dirname(os.path.abspath(__file__))
17
+ for _i in range(8):
18
+ _p = os.path.dirname(_THIS)
19
+ if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
20
+ sys.path.insert(0, _p); break
21
+ _THIS = _p
22
+
23
+
24
+ def embed_batch(texts: list, batch_size: int = 10) -> list:
25
+ """批量调 DashScope text-embedding-v3 → [[float]]。失败返回空。"""
26
+ key = os.environ.get('DASHSCOPE_API_KEY', '')
27
+ if not key or not texts:
28
+ return []
29
+ all_vecs = []
30
+ for i in range(0, len(texts), batch_size):
31
+ batch = texts[i:i + batch_size]
32
+ payload = json.dumps({
33
+ 'model': 'text-embedding-v3', 'input': batch, 'dimensions': 1024,
34
+ }).encode('utf-8')
35
+ req = urllib.request.Request(
36
+ 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
37
+ data=payload,
38
+ headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
39
+ )
40
+ try:
41
+ with urllib.request.urlopen(req, timeout=30) as resp:
42
+ r = json.loads(resp.read())
43
+ for d in sorted(r['data'], key=lambda x: x['index']):
44
+ all_vecs.append(d['embedding'])
45
+ except Exception as e:
46
+ # 这批失败 → 填 None 占位 (保持顺序)
47
+ all_vecs.extend([None] * len(batch))
48
+ return all_vecs
49
+
50
+
51
+ def build_embeddings_for_entities(entity_filter: str = None, limit: int = 500,
52
+ kinds: list = None, logger=print):
53
+ """对实体算 embedding → 灌 embeddings.vec_json。
54
+
55
+ Args:
56
+ entity_filter: SQL WHERE 条件 (如 "type='ANCHOR'")
57
+ limit: 上限 (控成本)
58
+ kinds: 实体类型白名单 (默认 anchors + SERVICE + TABLE + PRD)
59
+ """
60
+ from domain.kg.extract.java import pg_upsert
61
+ from sqlalchemy import text
62
+
63
+ engine, _ = pg_upsert._get_pg_engine()
64
+ if engine is None:
65
+ return {'error': 'PG 不可用'}
66
+
67
+ if kinds is None:
68
+ kinds = ['ANCHOR', 'SERVICE', 'TABLE', 'PRD', 'FEATURE']
69
+
70
+ # 取实体 (跳过已 embed 过的)
71
+ where = "WHERE e.type IN :kinds AND e.id NOT IN (SELECT entity_id FROM embeddings)"
72
+ if entity_filter:
73
+ where += " AND " + entity_filter
74
+ sql = f"""
75
+ SELECT e.id, e.repo_id, e.type, e.canonical, e.cn
76
+ FROM entities e
77
+ {where.replace(':kinds', ':kinds')}
78
+ LIMIT :lim
79
+ """
80
+ # 兼容 IN 参数
81
+ with engine.connect() as conn:
82
+ rows = conn.execute(text(sql.replace(':kinds', "('" + "','".join(kinds) + "')")),
83
+ {'lim': limit}).all()
84
+
85
+ logger(' [embed] %d 实体待向量化 (类型=%s)' % (len(rows), kinds))
86
+ if not rows:
87
+ return {'embedded': 0}
88
+
89
+ # 批量 embed
90
+ descs = [(r[3] or r[4] or r[0])[:200] for r in rows] # canonical 优先
91
+ t0 = time.time()
92
+ vecs = embed_batch(descs)
93
+ embed_time = time.time() - t0
94
+ valid = sum(1 for v in vecs if v)
95
+ logger(' [embed] embedding 完成: %d/%d 有效 / %.1fs' % (valid, len(vecs), embed_time))
96
+
97
+ # 灌 PG
98
+ ok = 0
99
+ with engine.begin() as conn:
100
+ for (eid, repo_id, etype, canonical, cn), vec in zip(rows, vecs):
101
+ if not vec:
102
+ continue
103
+ try:
104
+ conn.execute(text("""
105
+ INSERT INTO embeddings (entity_id, repo_id, kind, content, vec_json, model, embedded_at)
106
+ VALUES (:id, :repo, :kind, :content, CAST(:vec AS JSONB), 'text-embedding-v3', now())
107
+ ON CONFLICT (entity_id) DO UPDATE SET
108
+ vec_json=EXCLUDED.vec_json, content=EXCLUDED.content, embedded_at=now()
109
+ """), {
110
+ 'id': eid, 'repo': repo_id, 'kind': etype,
111
+ 'content': (canonical or cn or eid)[:200],
112
+ 'vec': json.dumps(vec),
113
+ })
114
+ ok += 1
115
+ except Exception:
116
+ pass
117
+ logger(' [embed] 灌入 %d 条向量' % ok)
118
+ return {'embedded': ok, 'time_sec': round(embed_time, 1)}
119
+
120
+
121
+ if __name__ == '__main__':
122
+ import argparse
123
+ ap = argparse.ArgumentParser()
124
+ ap.add_argument('--limit', type=int, default=500)
125
+ args = ap.parse_args()
126
+ try:
127
+ from dotenv import load_dotenv
128
+ for p in ['wlinkj-workspace/backend/.env', '.env']:
129
+ if os.path.isfile(p): load_dotenv(p); break
130
+ except: pass
131
+ r = build_embeddings_for_entities(limit=args.limit)
132
+ print(json.dumps(r, ensure_ascii=False, indent=2))
@@ -0,0 +1,80 @@
1
+ # -*- coding: utf-8 -*-
2
+ """命名同源匹配: 不依赖 embedding, 用命名规则建弱关联。
3
+
4
+ AssetController ↔ asset 表 ↔ 资产管理 PRD → 词干相同 → 弱边。
5
+ 这是 embedding 不可用时的主路径, 也是 embedding 的补充 (硬规则置信度高)。
6
+ """
7
+ from __future__ import annotations
8
+ import re
9
+
10
+
11
+ def extract_stem(name: str) -> str:
12
+ """提取名称的词干 (去掉类型后缀 + 小写 + 驼峰拆词)。
13
+
14
+ AssetController → 'asset'
15
+ EmployeeOnline → 'employee online' → 'employee'
16
+ IQrCodeLoginService → 'qr code login'
17
+ test_demo → 'test demo'
18
+ 资产管理 → '资产管理'
19
+ """
20
+ if not name:
21
+ return ''
22
+ s = name.strip()
23
+ # 去掉常见类型后缀
24
+ for suffix in ['Controller', 'ServiceImpl', 'Service', 'Mapper',
25
+ 'Repository', 'Entity', 'Vo', 'BO', 'Bo', 'Dto', 'DTO',
26
+ 'Form', 'Body', 'Request', 'Response', 'Result']:
27
+ if s.endswith(suffix) and len(s) > len(suffix):
28
+ s = s[:-len(suffix)]
29
+ break
30
+ # 去掉接口前缀 I
31
+ if s.startswith('I') and len(s) > 1 and s[1].isupper():
32
+ s = s[1:]
33
+ # 驼峰拆词
34
+ words = re.findall(r'[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|[\u4e00-\u9fff]+', s)
35
+ if not words:
36
+ return s.lower()
37
+ # 中文直接返回, 英文取第一个词 (主词)
38
+ if any('\u4e00' <= c <= '\u9fff' for c in s):
39
+ return ''.join(w for w in words if '\u4e00' <= w[0] <= '\u9fff')[:6]
40
+ return words[0].lower() if words else s.lower()
41
+
42
+
43
+ def naming_similarity(name_a: str, name_b: str) -> float:
44
+ """两个名称的命名同源相似度 → [0, 1]。
45
+
46
+ 词干相同 → 0.8
47
+ 词干包含关系 → 0.6
48
+ 无关 → 0
49
+ """
50
+ stem_a = extract_stem(name_a)
51
+ stem_b = extract_stem(name_b)
52
+ if not stem_a or not stem_b:
53
+ return 0.0
54
+ if stem_a == stem_b:
55
+ return 0.8
56
+ if stem_a in stem_b or stem_b in stem_a:
57
+ return 0.6
58
+ # 词重叠 (多词情况)
59
+ words_a = set(stem_a.split())
60
+ words_b = set(stem_b.split())
61
+ if words_a and words_b:
62
+ overlap = len(words_a & words_b)
63
+ if overlap > 0:
64
+ return 0.5 * overlap / max(len(words_a), len(words_b))
65
+ return 0.0
66
+
67
+
68
+ def match_to_anchors(entity_canonical: str, anchors: list) -> list:
69
+ """把实体和 anchors 做命名匹配 → [(anchor_id, score)]。
70
+
71
+ anchors: [{'id':..., 'title':..., 'kind':...}]
72
+ Returns: 按相似度降序, score >= 0.6 的
73
+ """
74
+ out = []
75
+ for a in anchors:
76
+ score = naming_similarity(entity_canonical, a.get('title', '') or a.get('id', ''))
77
+ if score >= 0.6:
78
+ out.append((a['id'], score))
79
+ out.sort(key=lambda x: x[1], reverse=True)
80
+ return out