beacon-mfg-mcp 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -0
- package/RELEASE.md +60 -0
- package/bin/beacon-mfg-mcp.js +30 -0
- package/package.json +35 -0
- package/server.py +1851 -0
package/server.py
ADDED
|
@@ -0,0 +1,1851 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Beacon-MFG 只读 MCP 服务(stdio 传输,零第三方依赖)
|
|
5
|
+
|
|
6
|
+
设计边界(与用户 2026-09-18 约定一致):
|
|
7
|
+
- **只读**:只检索「已发布数据」—— Cloudflare Pages 公开端点(默认)或本地 git 仓库的
|
|
8
|
+
已提交快照。绝不写、绝不调用任何后端脚本(fetch_batch / postfetch / en_backfill …),
|
|
9
|
+
绝不持有 ZHIPU / CLOUDFLARE 密钥。
|
|
10
|
+
- **不污染仓库**:本地模式只通过 `git show HEAD:<path>` 读取已提交内容(不会触发
|
|
11
|
+
maskphone 的 smudge 过滤器,也不会碰 index 锁)。本服务**绝不**对仓库执行
|
|
12
|
+
`git checkout` / `git restore` / `git add` / `git commit` —— 那是历史「42578 个手机号
|
|
13
|
+
被无声掩码」事故的根源,必须规避。
|
|
14
|
+
- 手机号为隐私字段:git 已提交版本是掩码态(138****0000);CF 部署版可能是全号。
|
|
15
|
+
对隐私敏感场景,把 BEACON_REPO 指向本地仓库(默认即读取掩码态)即可,无需联网。
|
|
16
|
+
|
|
17
|
+
协议:JSON-RPC 2.0 over stdio(newline-delimited)。兼容 Claude Desktop / Cline /
|
|
18
|
+
Continue / WorkBuddy 等主流 MCP 客户端。
|
|
19
|
+
|
|
20
|
+
数据源解析:
|
|
21
|
+
BEACON_SOURCE HTTP 基址(默认 https://beacon-mfg.pages.dev)
|
|
22
|
+
BEACON_REPO 本地 git 仓库路径;设了就用 `git show HEAD:` 读(推荐:离线 + 隐私安全)
|
|
23
|
+
缺省时自动探测 cwd 所在 git 仓库(若含 data/gb 就用它)
|
|
24
|
+
|
|
25
|
+
暴露的 tool:
|
|
26
|
+
search_vendors 按 关键词 / 城市 / 国标码 检索(走 fp 指纹分片)
|
|
27
|
+
get_vendor 按 id(+ 国标码)取完整中文档案(走 zh 分片)
|
|
28
|
+
get_capability_card 按 id 取能力卡(走 skills/registry/capability,由 R2 提供)
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
import os
|
|
32
|
+
import re
|
|
33
|
+
import sys
|
|
34
|
+
import math
|
|
35
|
+
import json
|
|
36
|
+
import hashlib
|
|
37
|
+
import concurrent.futures
|
|
38
|
+
import subprocess
|
|
39
|
+
import tarfile
|
|
40
|
+
import tempfile
|
|
41
|
+
import io
|
|
42
|
+
import time
|
|
43
|
+
import hmac
|
|
44
|
+
import urllib.request
|
|
45
|
+
import urllib.error
|
|
46
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
47
|
+
import sys as _sys
|
|
48
|
+
import uuid as _uuid
|
|
49
|
+
|
|
50
|
+
# --------------------------------------------------------------------------- #
|
|
51
|
+
# rfq-kernel 桥接(G1/G2/G3:让 MCP 客户 agent 能跑多轮对话匹配)
|
|
52
|
+
# 路径相对本文件解析,与 cwd / BEACON_REPO 无关。桥接不可用时降级,不影响其余 tool。
|
|
53
|
+
# --------------------------------------------------------------------------- #
|
|
54
|
+
_RFK_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
55
|
+
"..", "skills", "rfq-kernel", "src")
|
|
56
|
+
if os.path.isdir(_RFK_SRC) and _RFK_SRC not in _sys.path:
|
|
57
|
+
_sys.path.insert(0, _RFK_SRC)
|
|
58
|
+
try:
|
|
59
|
+
import mcp_bridge as _bridge
|
|
60
|
+
except Exception: # 桥接缺失/异常 → 降级,MCP 其余 3 个只读 tool 照常
|
|
61
|
+
_bridge = None
|
|
62
|
+
|
|
63
|
+
_SESSIONS: Dict[str, Any] = {}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _new_session_id() -> str:
|
|
67
|
+
return _uuid.uuid4().hex
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
_gb_seed_cache: Dict[str, List[Dict[str, Any]]] = {}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _gb_seed_records(pack_id: str) -> List[Dict[str, Any]]:
|
|
74
|
+
"""GB 种子:检测到某行业 pack 时,把该 pack 国标码(GB/T 4754)映射的企业补进召回。
|
|
75
|
+
|
|
76
|
+
按 gb 分片精准取(O(命中分片),绝非全量扫描),进程内缓存复用。
|
|
77
|
+
语义依据:beacon-mfg 以国标码为唯一行业判别信号,检测到行业即应召回其国标分类下的企业,
|
|
78
|
+
即便其厂名不含召回词(如 conveyor 的 gb=3434 连续搬运设备厂「耐特斯传输设备」用『传输』
|
|
79
|
+
而非『输送』,而『传输』df=1 未入倒排索引,常规召回捞不到)。加法召回,绝不误删真实企业。
|
|
80
|
+
"""
|
|
81
|
+
cached = _gb_seed_cache.get(pack_id)
|
|
82
|
+
if cached is not None:
|
|
83
|
+
return cached
|
|
84
|
+
out: List[Dict[str, Any]] = []
|
|
85
|
+
try:
|
|
86
|
+
gbs = _bridge.gbs_of_pack(pack_id)
|
|
87
|
+
except Exception:
|
|
88
|
+
gbs = set()
|
|
89
|
+
if gbs:
|
|
90
|
+
for s in _shards_of_type("fp"):
|
|
91
|
+
c = str(s.get("c") or "")
|
|
92
|
+
if c in gbs or c[:2] in gbs:
|
|
93
|
+
try:
|
|
94
|
+
out.extend(_read_fp_shard(s))
|
|
95
|
+
except Exception:
|
|
96
|
+
continue
|
|
97
|
+
_gb_seed_cache[pack_id] = out
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _recall_for_sourcing(text: str, top_k: int = 200, pack_id: str | None = None) -> List[Dict[str, Any]]:
|
|
102
|
+
"""为匹配做宽召回:对『产品信号』(工艺/材料/认证/国标码)与『企业名』命中打分,取 top_k。
|
|
103
|
+
|
|
104
|
+
**并集召回**:需求词被切成 bigram 后,任意一个命中即算候选(OR),不做交集。
|
|
105
|
+
(旧的 `_candidate_shards` 对同一需求词内部的 bigram 求交,会把「钣金冲压」这类
|
|
106
|
+
连写词缩到只剩字面全含的极少数分片——实测 316/372 条缩到 1 个分片,属隐性漏召回。)
|
|
107
|
+
|
|
108
|
+
证据权重:
|
|
109
|
+
- 产品信号命中(proc/mat/cert/gb)权重 3x;企业名 `co` 命中权重 1x。
|
|
110
|
+
企业名是**合法证据**——大量长尾厂只把品类写在厂名里(如「中山市世通输送机械设备
|
|
111
|
+
有限公司」的 proc 只有 cnc_milling),只信 proc/mat 会让这些真实企业永远搜不到。
|
|
112
|
+
- 每个命中词按 IDF 加权:稀有词(「输送」df=6)权重大,泛词(城市名「上海」df≈1.3万、
|
|
113
|
+
通名「厂家」)权重小 —— 需求句里的地区/通名不会带偏排序。
|
|
114
|
+
- 传入 pack_id 时,国标码命中本行业的记录 +1000,确保本行业供应商排到 top_k 前列。
|
|
115
|
+
|
|
116
|
+
性能:优先走倒排索引的『区分词并集』快速路径 `_recall_candidates`(O(命中量));
|
|
117
|
+
索引不可用/无区分词/并集过大时回退全量扫描(进程内与盘上均缓存,可复用)。只读。
|
|
118
|
+
"""
|
|
119
|
+
toks = _collapse_prefixes({t for t in _grams(text, query_mode=True) if _usable_gram(t)})
|
|
120
|
+
if not toks:
|
|
121
|
+
return []
|
|
122
|
+
|
|
123
|
+
idf = _gram_idf(toks)
|
|
124
|
+
if not idf: # 索引不可用 -> 退化为均匀权重
|
|
125
|
+
idf = {t: 1.0 for t in toks}
|
|
126
|
+
recs = _recall_candidates(toks)
|
|
127
|
+
if recs is None:
|
|
128
|
+
recs = _build_fp_index() # 兜底:全量扫描(首次构建并落盘缓存,后续进程内复用)
|
|
129
|
+
|
|
130
|
+
# GB 种子:检测到的行业 pack,其国标码映射的企业(如 conveyor 的 gb=3434 连续搬运设备)
|
|
131
|
+
# 即便厂名不含召回词(如「耐特斯传输设备」用『传输』而非『输送』,而『传输』df=1 未入索引),
|
|
132
|
+
# 也作为语义对应补进召回(加法,绝不误删真实企业)。下方 +1000 国标命中加成自然覆盖。
|
|
133
|
+
if pack_id and _bridge is not None:
|
|
134
|
+
seed = _gb_seed_records(pack_id)
|
|
135
|
+
if seed:
|
|
136
|
+
seen_ids = {r.get("id") for r in recs if r.get("id")}
|
|
137
|
+
for s in seed:
|
|
138
|
+
sid = s.get("id")
|
|
139
|
+
if sid and sid not in seen_ids:
|
|
140
|
+
recs.append(s)
|
|
141
|
+
seen_ids.add(sid)
|
|
142
|
+
|
|
143
|
+
# 本 pack 的产品词面(小写):用于给「厂名含产品词但 gb 未映射本行业」的长尾真实企业加成
|
|
144
|
+
name_surfaces = [s.lower() for s in _bridge.pack_vocab_surfaces(pack_id)] if pack_id else []
|
|
145
|
+
|
|
146
|
+
scored: List[tuple] = []
|
|
147
|
+
for r in recs:
|
|
148
|
+
prod_hay = (" ".join(r.get("proc") or []) + " " +
|
|
149
|
+
" ".join(r.get("mat") or []) + " " +
|
|
150
|
+
" ".join(r.get("cert") or []) + " " +
|
|
151
|
+
(r.get("gb") or "")).lower()
|
|
152
|
+
name_hay = (r.get("co") or "").lower()
|
|
153
|
+
w = 0.0
|
|
154
|
+
for t in toks:
|
|
155
|
+
if not t or t not in idf: # 碎片/泛词不参与打分
|
|
156
|
+
continue
|
|
157
|
+
iw = idf[t]
|
|
158
|
+
if t in prod_hay:
|
|
159
|
+
w += 3.0 * iw
|
|
160
|
+
elif t in name_hay:
|
|
161
|
+
w += 1.0 * iw
|
|
162
|
+
# 国标码命中检测行业 / 厂名含本 pack 产品词:本行业强信号,即便需求词全为
|
|
163
|
+
# 稀有词(df<2,如「风送线」)导致 token 打分全为 0,也要保留——这些是
|
|
164
|
+
# 真实供应商(GB 种子加法补召的 3434 搬运设备厂、或厂名含产品词的长尾厂),
|
|
165
|
+
# 不能因为「没有高频区分词」就被 w<=0 一概滤除(beacon-mfg 红线:真实企业必须被看见)。
|
|
166
|
+
gb_hit = pack_id and _bridge is not None and _bridge.pack_of_gb(r.get("gb")) == pack_id
|
|
167
|
+
name_hit = pack_id and name_surfaces and any(s in name_hay for s in name_surfaces)
|
|
168
|
+
if w <= 0 and not (gb_hit or name_hit):
|
|
169
|
+
continue
|
|
170
|
+
# 国标码命中检测行业 -> 加权,确保本行业供应商排到 top_k 前列
|
|
171
|
+
if gb_hit:
|
|
172
|
+
w += 1000.0
|
|
173
|
+
# 厂名含本 pack 产品词、但 gb 未映射本行业的长尾真实企业(如输送厂 gb=3360/3451
|
|
174
|
+
# 而非 3434):同样视为本行业强信号给次高加成,避免被 GB 种子挤到池底
|
|
175
|
+
# (beacon-mfg 红线:真实企业必须被看见)。
|
|
176
|
+
elif name_hit:
|
|
177
|
+
w += 500.0
|
|
178
|
+
r2 = dict(r)
|
|
179
|
+
r2["_recall_relevance"] = round(w, 4)
|
|
180
|
+
scored.append((w, r2))
|
|
181
|
+
scored.sort(key=lambda x: -x[0])
|
|
182
|
+
return [r2 for _, r2 in scored[:top_k]]
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# --------------------------------------------------------------------------- #
|
|
186
|
+
# 配置
|
|
187
|
+
# --------------------------------------------------------------------------- #
|
|
188
|
+
DEFAULT_SOURCE = "https://beacon-mfg.pages.dev"
|
|
189
|
+
BEACON_SOURCE = os.environ.get("BEACON_SOURCE", DEFAULT_SOURCE).rstrip("/")
|
|
190
|
+
BEACON_REPO = os.environ.get("BEACON_REPO") # None -> 自动探测 / 退回 HTTP
|
|
191
|
+
|
|
192
|
+
CACHE_DIR = os.path.join(tempfile.gettempdir(), "beacon-mcp-cache")
|
|
193
|
+
UA = "BeaconMFG-MCP/1.0 (+https://beacon-mfg.pages.dev/)"
|
|
194
|
+
|
|
195
|
+
_manifest_cache: Optional[Dict[str, Any]] = None
|
|
196
|
+
# 国标别名表({word: entries}),懒加载一次;None 表示还没读过
|
|
197
|
+
_alias_cache: Optional[Dict[str, Any]] = None
|
|
198
|
+
# 国标码 → 中文名,懒加载一次
|
|
199
|
+
_gb_name_cache: Optional[Dict[str, str]] = None
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
# --------------------------------------------------------------------------- #
|
|
203
|
+
# 使用量埋点(设计见 docs/MCP_USAGE_AUDIT.md §3.4 / §4 / §14)
|
|
204
|
+
# --------------------------------------------------------------------------- #
|
|
205
|
+
# 三条原则,改这里前先读文档:
|
|
206
|
+
# 1. **默认不出网**。本地台账永远只写本地文件;随请求带出去的只有下面四个
|
|
207
|
+
# 请求头,且里面没有 IP、没有 UA 原文、没有完整 query。
|
|
208
|
+
# 2. **可关闭**。`BEACON_TELEMETRY=0` 时不发任何头;`BEACON_USAGE_LOG=0` 时不写台账。
|
|
209
|
+
# 3. **失败静默**。埋点出任何问题都不许影响检索结果 —— 这是只读服务,
|
|
210
|
+
# 记账不能变成新的故障源。
|
|
211
|
+
#
|
|
212
|
+
# 为什么要有本地台账:本地 git 模式(BEACON_REPO)**完全不触网**,服务端看不见,
|
|
213
|
+
# 只有本地这一份能记。对服务端而言它统计到的永远是下限,这是架构决定的。
|
|
214
|
+
|
|
215
|
+
BEACON_TELEMETRY = os.environ.get("BEACON_TELEMETRY", "1") != "0"
|
|
216
|
+
BEACON_USAGE_LOG = os.environ.get("BEACON_USAGE_LOG", "1") != "0"
|
|
217
|
+
USAGE_LOG_PATH = os.path.join(os.path.expanduser("~"), ".beacon-mfg", "usage.jsonl")
|
|
218
|
+
|
|
219
|
+
# 当前调用的上下文(tool / tokens / hits),由 _dispatch 设置、_http_get 读取。
|
|
220
|
+
# 用 ContextVar 而不是全局变量:MCP server 理论上可能并发处理请求。
|
|
221
|
+
_CUR: Any = None
|
|
222
|
+
try:
|
|
223
|
+
from contextvars import ContextVar
|
|
224
|
+
_CUR = ContextVar("beacon_current_call", default=None)
|
|
225
|
+
except Exception: # 极老的运行环境
|
|
226
|
+
_CUR = None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _cur_set(val: Any) -> Any:
|
|
230
|
+
"""返回 token 以便 finally 里 reset;无 ContextVar 时退化成全局变量。"""
|
|
231
|
+
if _CUR is not None:
|
|
232
|
+
return _CUR.set(val)
|
|
233
|
+
global _CUR_FALLBACK
|
|
234
|
+
_CUR_FALLBACK = val
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _cur_get() -> Any:
|
|
239
|
+
if _CUR is not None:
|
|
240
|
+
return _CUR.get()
|
|
241
|
+
return globals().get("_CUR_FALLBACK")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
_CUR_FALLBACK = None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _client_id() -> str:
|
|
248
|
+
"""稳定的本机 id(首次调用时生成),**只用于派生不可逆的 cid**。
|
|
249
|
+
|
|
250
|
+
文件里存的是随机 uuid;发到服务端的是 HMAC 结果,收不到原始 uuid。
|
|
251
|
+
"""
|
|
252
|
+
p = os.path.join(os.path.expanduser("~"), ".beacon-mfg", "client_id")
|
|
253
|
+
try:
|
|
254
|
+
if os.path.exists(p):
|
|
255
|
+
return open(p, "r", encoding="utf-8").read().strip()
|
|
256
|
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
257
|
+
v = str(_uuid.uuid4())
|
|
258
|
+
with open(p, "w", encoding="utf-8") as f:
|
|
259
|
+
f.write(v)
|
|
260
|
+
return v
|
|
261
|
+
except Exception:
|
|
262
|
+
return "unknown"
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
_CID_SALT = b"beacon-mfg-v1" # 公开盐:目的是让 cid 不可逆,不是防攻击者
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _cid() -> str:
|
|
269
|
+
return hmac.new(_CID_SALT, _client_id().encode("utf-8"), hashlib.sha256) \
|
|
270
|
+
.hexdigest()[:16]
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _beacon_headers() -> Dict[str, str]:
|
|
274
|
+
"""随请求带出的四个头。**不含 IP / UA 原文 / 完整 query**。"""
|
|
275
|
+
if not BEACON_TELEMETRY:
|
|
276
|
+
return {}
|
|
277
|
+
cur = _cur_get() or {}
|
|
278
|
+
h = {"X-Beacon-Tool": cur.get("tool") or "", "X-Beacon-Cid": _cid()}
|
|
279
|
+
# tokens = 分词后的**产品词**,不是整句(§14.2:记整句等于把用户输入落成明文台账)
|
|
280
|
+
if cur.get("tokens"):
|
|
281
|
+
h["X-Beacon-Tokens"] = ",".join(cur["tokens"])[:120]
|
|
282
|
+
if cur.get("hits") is not None:
|
|
283
|
+
h["X-Beacon-Hits"] = str(cur["hits"])
|
|
284
|
+
return {k: v for k, v in h.items() if v not in ("", None)}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _emit_usage(tool: str, ms: int, ok: bool, extra: Any = None) -> None:
|
|
288
|
+
"""写本地台账(jsonl)。默认开启,BEACON_USAGE_LOG=0 可关。"""
|
|
289
|
+
if not BEACON_USAGE_LOG:
|
|
290
|
+
return
|
|
291
|
+
try:
|
|
292
|
+
rec = {
|
|
293
|
+
"ts": int(time.time() * 1000),
|
|
294
|
+
"tool": tool,
|
|
295
|
+
"ms": ms,
|
|
296
|
+
"ok": ok,
|
|
297
|
+
# REPO 可能指向一个不存在的路径(此时实际走的是 HTTP,只是 git show 静默失败),
|
|
298
|
+
# 所以这里**必须判目录存在**再算 git,否则会把离线调用记成 http 或反之。
|
|
299
|
+
"src": ("git" if (REPO and os.path.isdir(REPO)) else "http"),
|
|
300
|
+
"cid": _cid(),
|
|
301
|
+
}
|
|
302
|
+
if extra:
|
|
303
|
+
rec.update(extra)
|
|
304
|
+
os.makedirs(os.path.dirname(USAGE_LOG_PATH), exist_ok=True)
|
|
305
|
+
with open(USAGE_LOG_PATH, "a", encoding="utf-8") as f:
|
|
306
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
307
|
+
except Exception:
|
|
308
|
+
pass # 记账失败绝不能影响检索
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# --------------------------------------------------------------------------- #
|
|
312
|
+
# 数据源抽象:只做只读取
|
|
313
|
+
# --------------------------------------------------------------------------- #
|
|
314
|
+
def _detect_repo() -> Optional[str]:
|
|
315
|
+
if BEACON_REPO:
|
|
316
|
+
return BEACON_REPO
|
|
317
|
+
try:
|
|
318
|
+
out = subprocess.run(
|
|
319
|
+
["git", "rev-parse", "--show-toplevel"],
|
|
320
|
+
capture_output=True, text=True, cwd=os.getcwd(),
|
|
321
|
+
)
|
|
322
|
+
if out.returncode == 0:
|
|
323
|
+
repo = out.stdout.strip()
|
|
324
|
+
# 只在该仓库确实是 beacon-mfg(含 data/gb)时才用,避免误读别的仓库
|
|
325
|
+
if os.path.isdir(os.path.join(repo, "data", "gb")):
|
|
326
|
+
return repo
|
|
327
|
+
except Exception:
|
|
328
|
+
pass
|
|
329
|
+
return None
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
REPO = _detect_repo()
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _cache_path(url: str) -> str:
|
|
336
|
+
os.makedirs(CACHE_DIR, exist_ok=True)
|
|
337
|
+
return os.path.join(CACHE_DIR, hashlib.sha1(url.encode("utf-8")).hexdigest() + ".json")
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _git_show(relpath: str) -> Optional[str]:
|
|
341
|
+
"""只读已提交内容。绝不 checkout —— 不触发 maskphone smudge、不碰 index 锁。"""
|
|
342
|
+
if not REPO:
|
|
343
|
+
return None
|
|
344
|
+
try:
|
|
345
|
+
r = subprocess.run(
|
|
346
|
+
["git", "-C", REPO, "show", "HEAD:" + relpath],
|
|
347
|
+
capture_output=True, text=True,
|
|
348
|
+
)
|
|
349
|
+
if r.returncode == 0 and r.stdout:
|
|
350
|
+
return r.stdout
|
|
351
|
+
except Exception:
|
|
352
|
+
pass
|
|
353
|
+
return None
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _http_get(relpath: str) -> Tuple[Optional[str], Optional[str]]:
|
|
357
|
+
url = BEACON_SOURCE + "/" + relpath
|
|
358
|
+
cap = _cache_path(url)
|
|
359
|
+
etag = None
|
|
360
|
+
if os.path.exists(cap):
|
|
361
|
+
try:
|
|
362
|
+
with open(cap, "r", encoding="utf-8") as f:
|
|
363
|
+
blob = json.load(f)
|
|
364
|
+
etag = blob.get("etag")
|
|
365
|
+
except Exception:
|
|
366
|
+
etag = None
|
|
367
|
+
headers = {"User-Agent": UA, "Accept": "*/*"}
|
|
368
|
+
# 埋点头:服务端据此填 tool / tokens / hits 列。收不到也没关系 ——
|
|
369
|
+
# worker 那边**照样计数**,只是这三列为空(文档 §3.4)。
|
|
370
|
+
try:
|
|
371
|
+
headers.update(_beacon_headers())
|
|
372
|
+
except Exception:
|
|
373
|
+
pass
|
|
374
|
+
req = urllib.request.Request(url, headers=headers)
|
|
375
|
+
if etag:
|
|
376
|
+
req.add_header("If-None-Match", etag)
|
|
377
|
+
try:
|
|
378
|
+
resp = urllib.request.urlopen(req, timeout=30)
|
|
379
|
+
body = resp.read().decode("utf-8")
|
|
380
|
+
new_etag = resp.headers.get("ETag")
|
|
381
|
+
try:
|
|
382
|
+
with open(cap, "w", encoding="utf-8") as f:
|
|
383
|
+
json.dump({"etag": new_etag, "body": body}, f)
|
|
384
|
+
except Exception:
|
|
385
|
+
pass
|
|
386
|
+
return body, new_etag
|
|
387
|
+
except urllib.error.HTTPError as e:
|
|
388
|
+
if e.code == 304 and os.path.exists(cap):
|
|
389
|
+
with open(cap, "r", encoding="utf-8") as f:
|
|
390
|
+
return json.load(f).get("body"), etag
|
|
391
|
+
return None, None
|
|
392
|
+
except Exception:
|
|
393
|
+
# 网络不通时若本地有缓存也返回,保证离线可用
|
|
394
|
+
if os.path.exists(cap):
|
|
395
|
+
with open(cap, "r", encoding="utf-8") as f:
|
|
396
|
+
return json.load(f).get("body"), etag
|
|
397
|
+
return None, None
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def fetch_text(relpath: str) -> Optional[str]:
|
|
401
|
+
"""统一的只读取入口:本地 git 优先(若可用),否则 HTTP。"""
|
|
402
|
+
if REPO:
|
|
403
|
+
g = _git_show(relpath)
|
|
404
|
+
if g is not None:
|
|
405
|
+
return g
|
|
406
|
+
# 本地没有(如能力卡不进 git)再退回 HTTP
|
|
407
|
+
return _http_get(relpath)[0]
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
# --------------------------------------------------------------------------- #
|
|
411
|
+
# manifest + 分片访问
|
|
412
|
+
# --------------------------------------------------------------------------- #
|
|
413
|
+
def load_manifest() -> Dict[str, Any]:
|
|
414
|
+
global _manifest_cache
|
|
415
|
+
if _manifest_cache is None:
|
|
416
|
+
txt = fetch_text("data/manifest.json")
|
|
417
|
+
if not txt:
|
|
418
|
+
raise RuntimeError("无法取得 manifest.json(检查 BEACON_SOURCE / BEACON_REPO / 网络)")
|
|
419
|
+
_manifest_cache = json.loads(txt)
|
|
420
|
+
return _manifest_cache
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def _shards_of_type(t: str) -> List[Dict[str, Any]]:
|
|
424
|
+
return [s for s in load_manifest().get("shards", []) if s.get("t") == t]
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _zh_paths_for_gb(gb: str) -> List[str]:
|
|
428
|
+
# 一个国标码可能拆成多个 zh 分片(如 3484.json + 3484-p2.json 续片),必须全扫
|
|
429
|
+
return [s.get("p") for s in _shards_of_type("zh") if s.get("c") == gb]
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
# --------------------------------------------------------------------------- #
|
|
433
|
+
# 国标行业别名表(与 App 的 AliasIndex 同源)
|
|
434
|
+
#
|
|
435
|
+
# 2026-09-24 之前 MCP **完全没接别名表**:「输送线」这种口语词在厂名/工艺/材料里
|
|
436
|
+
# 一个字都不出现,_hay 命中不了,于是返回 0 条 —— 而 App 同一句能出 3434。
|
|
437
|
+
# 别名表把「口语词 → 国标码」补上,检索时按码直接定向对应分片。
|
|
438
|
+
# --------------------------------------------------------------------------- #
|
|
439
|
+
def _load_alias_file(rel: str) -> Dict[str, Any]:
|
|
440
|
+
"""两种历史形态都兼容:[["metadata",..],["alias",{..}]] 和直接 {word: ...}。"""
|
|
441
|
+
txt = fetch_text(rel)
|
|
442
|
+
if not txt:
|
|
443
|
+
return {}
|
|
444
|
+
try:
|
|
445
|
+
raw = json.loads(txt)
|
|
446
|
+
except Exception:
|
|
447
|
+
return {}
|
|
448
|
+
if isinstance(raw, list):
|
|
449
|
+
for pair in raw:
|
|
450
|
+
if isinstance(pair, list) and len(pair) == 2 and pair[0] == "alias":
|
|
451
|
+
return pair[1] or {}
|
|
452
|
+
return {}
|
|
453
|
+
if isinstance(raw, dict):
|
|
454
|
+
return raw.get("alias", raw) or {}
|
|
455
|
+
return {}
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def load_gb_alias() -> Dict[str, Any]:
|
|
459
|
+
"""词 → 条目。两层表合并:gb-alias.json(数据推导)+ gb-alias-curated.json(人工策展)。"""
|
|
460
|
+
global _alias_cache
|
|
461
|
+
if _alias_cache is None:
|
|
462
|
+
merged: Dict[str, Any] = {}
|
|
463
|
+
for rel in ("data/gb-alias.json", "data/gb-alias-curated.json"):
|
|
464
|
+
for k, v in _load_alias_file(rel).items():
|
|
465
|
+
merged.setdefault(str(k).lower(), v)
|
|
466
|
+
_alias_cache = merged
|
|
467
|
+
return _alias_cache
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _alias_entries(v: Any) -> List[Dict[str, Any]]:
|
|
471
|
+
"""条目的两种写法归一成 [{code,name,hits}]:
|
|
472
|
+
gb-alias.json → [{"code","name","hits"}, ...]
|
|
473
|
+
gb-alias-curated → {"codes":[...], "note":...}
|
|
474
|
+
"""
|
|
475
|
+
out: List[Dict[str, Any]] = []
|
|
476
|
+
if isinstance(v, dict):
|
|
477
|
+
for c in (v.get("codes") or []):
|
|
478
|
+
out.append({"code": str(c), "name": v.get("name", ""), "hits": int(v.get("hits") or 0)})
|
|
479
|
+
if v.get("code"):
|
|
480
|
+
out.append({"code": str(v["code"]), "name": v.get("name", ""), "hits": int(v.get("hits") or 0)})
|
|
481
|
+
elif isinstance(v, list):
|
|
482
|
+
for e in v:
|
|
483
|
+
if isinstance(e, dict) and e.get("code"):
|
|
484
|
+
out.append({"code": str(e["code"]), "name": e.get("name", ""),
|
|
485
|
+
"hits": int(e.get("hits") or 0)})
|
|
486
|
+
elif isinstance(e, str):
|
|
487
|
+
out.append({"code": e, "name": "", "hits": 0})
|
|
488
|
+
return out
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _gb_names() -> Dict[str, str]:
|
|
492
|
+
"""国标码 → 中文名。取不到(离线/文件缺失)就返回空表,不影响主链路。"""
|
|
493
|
+
global _gb_name_cache
|
|
494
|
+
if _gb_name_cache is None:
|
|
495
|
+
names: Dict[str, str] = {}
|
|
496
|
+
txt = fetch_text("data/gb4754-full.json")
|
|
497
|
+
if txt:
|
|
498
|
+
try:
|
|
499
|
+
raw = json.loads(txt)
|
|
500
|
+
except Exception:
|
|
501
|
+
raw = None
|
|
502
|
+
# 两种形态都见过:{"classes":{...},"groups":{...}} 和 [["classes",{...}],...];
|
|
503
|
+
# 值本身也可能是 {"name":..,"desc":..},只取 name 段。
|
|
504
|
+
sections = raw.items() if isinstance(raw, dict) else (raw or [])
|
|
505
|
+
for key, body in sections:
|
|
506
|
+
if key in ("classes", "groups", "divisions") and isinstance(body, dict):
|
|
507
|
+
for k, v in body.items():
|
|
508
|
+
names[str(k)] = str(v.get("name", v)) if isinstance(v, dict) else str(v)
|
|
509
|
+
_gb_name_cache = names
|
|
510
|
+
return _gb_name_cache
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def alias_codes(q: str, max_codes: int = 6) -> List[Dict[str, Any]]:
|
|
514
|
+
"""采购口语词 → 国标码。返回 [{code,name,word,exact,hits}],按 精确>hits 排序。"""
|
|
515
|
+
alias = load_gb_alias()
|
|
516
|
+
q = (q or "").strip().lower()
|
|
517
|
+
if not alias or not q:
|
|
518
|
+
return []
|
|
519
|
+
probes = [q] + [t for t in q.split() if len(t) >= 2]
|
|
520
|
+
best: Dict[str, Dict[str, Any]] = {}
|
|
521
|
+
for probe in dict.fromkeys(probes):
|
|
522
|
+
if len(probe) < 2:
|
|
523
|
+
continue
|
|
524
|
+
for word, v in alias.items():
|
|
525
|
+
wl = word.lower()
|
|
526
|
+
if wl == probe:
|
|
527
|
+
exact = True
|
|
528
|
+
elif probe in wl or wl in probe:
|
|
529
|
+
exact = False
|
|
530
|
+
else:
|
|
531
|
+
continue
|
|
532
|
+
for e in _alias_entries(v):
|
|
533
|
+
code = e["code"]
|
|
534
|
+
if not code:
|
|
535
|
+
continue
|
|
536
|
+
prev = best.get(code)
|
|
537
|
+
rank = (1 if exact else 0, e["hits"])
|
|
538
|
+
if prev is None or rank > prev["_rank"]:
|
|
539
|
+
best[code] = {"code": code, "name": e["name"], "word": word,
|
|
540
|
+
"exact": exact, "hits": e["hits"], "_rank": rank}
|
|
541
|
+
out = [v for v in best.values()]
|
|
542
|
+
for v in out:
|
|
543
|
+
v.pop("_rank", None)
|
|
544
|
+
# curated 表只写 codes 不写 name,回填报一下 —— 返回体里带中文行业名,
|
|
545
|
+
# 调用方(agent / 人)不用再自己查一遍码表。
|
|
546
|
+
if not v["name"]:
|
|
547
|
+
v["name"] = _gb_names().get(v["code"], "")
|
|
548
|
+
out.sort(key=lambda x: (-int(x["exact"]), -x["hits"]))
|
|
549
|
+
return out[:max_codes]
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _fp_paths_for_gbs(codes: List[str]) -> List[str]:
|
|
553
|
+
return [s.get("p") for s in _shards_of_type("fp") if s.get("c") in set(codes)]
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
# --------------------------------------------------------------------------- #
|
|
557
|
+
# 能力(cap)别名召回:把「AI / 人工智能 / 机器学习 / 大模型 / 算法 …」映射到
|
|
558
|
+
# 能力键(如 tech_ai),按能力键定向召回。与 GB 别名不同,cap 是跨门类能力、
|
|
559
|
+
# 不挂在某个国标码下,故走 cap.json 的 shards 表找分片,而非 _fp_paths_for_gbs。
|
|
560
|
+
#
|
|
561
|
+
# 关键点:命中 cap 别名的 token 从「通用子串 AND 校验」里**消费掉**,只走 cap
|
|
562
|
+
# 召回 —— 否则「AI」会作为子串命中 algebraist / Ashore 等英文名咖啡店,造成噪声。
|
|
563
|
+
# 匹配用整词精确(不分词组子串),同样是为了避免「ai」误扩到别的词里。
|
|
564
|
+
# --------------------------------------------------------------------------- #
|
|
565
|
+
_cap_index_cache: Optional[Dict[str, Any]] = None
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _load_cap_index() -> Optional[Dict[str, Any]]:
|
|
569
|
+
"""skills/registry/index/cap.json —— 发布产物(与 city.json 同构):
|
|
570
|
+
{terms:{code:词面串}, shards:{code:{shard_name:count}}}。"""
|
|
571
|
+
global _cap_index_cache
|
|
572
|
+
if _cap_index_cache is None:
|
|
573
|
+
txt = _index_text("skills/registry/index/cap.json")
|
|
574
|
+
try:
|
|
575
|
+
_cap_index_cache = json.loads(txt) if txt else {}
|
|
576
|
+
except Exception:
|
|
577
|
+
_cap_index_cache = {}
|
|
578
|
+
return _cap_index_cache
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _cap_term_index() -> Dict[str, str]:
|
|
582
|
+
"""词面词 → 能力键(反向索引,缓存)。覆盖 cap.json terms 里的每个分词,
|
|
583
|
+
例如 「AI」「人工智能」「机器学习」「大模型」「算法」→ tech_ai。"""
|
|
584
|
+
idx: Dict[str, str] = {}
|
|
585
|
+
cap = _load_cap_index() or {}
|
|
586
|
+
for code, termstr in (cap.get("terms") or {}).items():
|
|
587
|
+
for w in str(termstr).lower().split():
|
|
588
|
+
idx.setdefault(w, code) # 首个出现的码优先
|
|
589
|
+
return idx
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def cap_alias_codes(q: str) -> List[Dict[str, Any]]:
|
|
593
|
+
"""能力口语词 → 能力键。整词精确匹配(不分词组子串)。返回 [{cap, word, name}]。"""
|
|
594
|
+
q = (q or "").strip().lower()
|
|
595
|
+
if not q:
|
|
596
|
+
return []
|
|
597
|
+
idx = _cap_term_index()
|
|
598
|
+
hits: Dict[str, Dict[str, Any]] = {}
|
|
599
|
+
for tok in dict.fromkeys(t for t in q.split() if t):
|
|
600
|
+
code = idx.get(tok)
|
|
601
|
+
if code:
|
|
602
|
+
hits[code] = {"cap": code, "word": tok, "name": _cap_name(code)}
|
|
603
|
+
return list(hits.values())
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _cap_name(code: str) -> str:
|
|
607
|
+
"""能力键 → 人类可读名(取 terms 词面串的首段)。"""
|
|
608
|
+
cap = _load_cap_index() or {}
|
|
609
|
+
t = (cap.get("terms") or {}).get(code, "")
|
|
610
|
+
return str(t).split()[0] if t else code
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _cap_shard_paths(cap_code: str) -> List[str]:
|
|
614
|
+
"""能力键 → 指纹分片路径。依据 cap.json shards[code] 的分片名,
|
|
615
|
+
优先用 manifest 的 fp 分片(按 c 对齐),否则按名直拼路径。"""
|
|
616
|
+
cap = _load_cap_index() or {}
|
|
617
|
+
names = (cap.get("shards") or {}).get(cap_code, {})
|
|
618
|
+
if not names:
|
|
619
|
+
return []
|
|
620
|
+
by_c = {str(s.get("c", "")): s.get("p") for s in _shards_of_type("fp")}
|
|
621
|
+
paths: List[str] = []
|
|
622
|
+
for nm in names:
|
|
623
|
+
p = by_c.get(str(nm))
|
|
624
|
+
if not p:
|
|
625
|
+
cand = f"skills/registry/fingerprint/gb/{nm}.jsonl"
|
|
626
|
+
if os.path.exists(cand):
|
|
627
|
+
p = cand
|
|
628
|
+
if p and p not in paths:
|
|
629
|
+
paths.append(p)
|
|
630
|
+
return paths
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
# --------------------------------------------------------------------------- #
|
|
635
|
+
# tool 实现
|
|
636
|
+
# --------------------------------------------------------------------------- #
|
|
637
|
+
# --------------------------------------------------------------------------- #
|
|
638
|
+
# 全文索引:首次构建后常驻进程内存,并按 HEAD sha 缓存到磁盘,
|
|
639
|
+
# 避免每次搜索都全扫 267 个 fp 分片(此前逐分片 git show 约 100s)。
|
|
640
|
+
# --------------------------------------------------------------------------- #
|
|
641
|
+
_fp_index: Optional[List[Dict[str, Any]]] = None
|
|
642
|
+
_fp_index_key: Optional[str] = None
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _head_sha() -> Optional[str]:
|
|
646
|
+
if not REPO:
|
|
647
|
+
return None
|
|
648
|
+
try:
|
|
649
|
+
r = subprocess.run(["git", "-C", REPO, "rev-parse", "HEAD"],
|
|
650
|
+
capture_output=True, text=True)
|
|
651
|
+
if r.returncode == 0 and r.stdout.strip():
|
|
652
|
+
return r.stdout.strip()
|
|
653
|
+
except Exception:
|
|
654
|
+
pass
|
|
655
|
+
return None
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _index_cache_path() -> Optional[str]:
|
|
659
|
+
if not REPO: # HTTP 模式不落盘缓存,避免读到陈旧的已发布快照
|
|
660
|
+
return None
|
|
661
|
+
key = (BEACON_REPO or BEACON_SOURCE) + "|" + (_head_sha() or "nosha")
|
|
662
|
+
h = hashlib.sha1(key.encode("utf-8")).hexdigest()
|
|
663
|
+
return os.path.join(CACHE_DIR, "fpindex-" + h + ".json")
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _read_fp_shard(s: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
667
|
+
txt = fetch_text(s["p"])
|
|
668
|
+
out: List[Dict[str, Any]] = []
|
|
669
|
+
if not txt:
|
|
670
|
+
return out
|
|
671
|
+
for line in txt.splitlines():
|
|
672
|
+
line = line.strip()
|
|
673
|
+
if not line:
|
|
674
|
+
continue
|
|
675
|
+
try:
|
|
676
|
+
out.append(json.loads(line))
|
|
677
|
+
except Exception:
|
|
678
|
+
continue
|
|
679
|
+
return out
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def _build_fp_index_via_archive() -> Optional[List[Dict[str, Any]]]:
|
|
683
|
+
"""git 模式:用 `git archive HEAD -- <dir>` 一次性批量取出所有 fp 分片。
|
|
684
|
+
|
|
685
|
+
只读已提交 object(本就是 maskphone 掩码态),不碰工作树、不触发 smudge、
|
|
686
|
+
不碰 index 锁 —— 安全边界与逐分片 `git show` 一致,但把 267 次 subprocess
|
|
687
|
+
降到 1 次,冷启动从 ~48s 降至数秒。任一环节失败都返回 None,由调用方退回
|
|
688
|
+
逐分片 fallback。
|
|
689
|
+
"""
|
|
690
|
+
try:
|
|
691
|
+
d = os.path.join("skills", "registry", "fingerprint")
|
|
692
|
+
r = subprocess.run(
|
|
693
|
+
["git", "-C", REPO, "archive", "HEAD", "--", d],
|
|
694
|
+
capture_output=True,
|
|
695
|
+
)
|
|
696
|
+
if r.returncode != 0 or not r.stdout:
|
|
697
|
+
return None
|
|
698
|
+
recs: List[Dict[str, Any]] = []
|
|
699
|
+
with tarfile.open(fileobj=io.BytesIO(r.stdout), mode="r:*") as tf:
|
|
700
|
+
for m in tf.getmembers():
|
|
701
|
+
if not m.isfile():
|
|
702
|
+
continue
|
|
703
|
+
try:
|
|
704
|
+
data = tf.extractfile(m).read().decode("utf-8", "replace")
|
|
705
|
+
except Exception:
|
|
706
|
+
continue
|
|
707
|
+
for line in data.splitlines():
|
|
708
|
+
line = line.strip()
|
|
709
|
+
if not line:
|
|
710
|
+
continue
|
|
711
|
+
try:
|
|
712
|
+
recs.append(json.loads(line))
|
|
713
|
+
except Exception:
|
|
714
|
+
continue
|
|
715
|
+
return recs if recs else None
|
|
716
|
+
except Exception:
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _build_fp_index() -> List[Dict[str, Any]]:
|
|
721
|
+
global _fp_index, _fp_index_key
|
|
722
|
+
cp = _index_cache_path()
|
|
723
|
+
key = cp or "mem"
|
|
724
|
+
if _fp_index is not None and _fp_index_key == key:
|
|
725
|
+
return _fp_index
|
|
726
|
+
if cp and os.path.exists(cp):
|
|
727
|
+
try:
|
|
728
|
+
with open(cp, "r", encoding="utf-8") as f:
|
|
729
|
+
_fp_index = json.load(f)
|
|
730
|
+
_fp_index_key = key
|
|
731
|
+
return _fp_index
|
|
732
|
+
except Exception:
|
|
733
|
+
pass
|
|
734
|
+
fp_shards = _shards_of_type("fp")
|
|
735
|
+
# git 模式优先用 `git archive` 一次性批量读(1 次 subprocess,远快于逐分片 git show)
|
|
736
|
+
if REPO:
|
|
737
|
+
recs = _build_fp_index_via_archive()
|
|
738
|
+
if recs is not None:
|
|
739
|
+
_fp_index = recs
|
|
740
|
+
_fp_index_key = key
|
|
741
|
+
if cp:
|
|
742
|
+
try:
|
|
743
|
+
with open(cp, "w", encoding="utf-8") as f:
|
|
744
|
+
json.dump(recs, f, ensure_ascii=False)
|
|
745
|
+
for old in os.listdir(CACHE_DIR):
|
|
746
|
+
if old.startswith("fpindex-") and old != os.path.basename(cp):
|
|
747
|
+
try:
|
|
748
|
+
os.remove(os.path.join(CACHE_DIR, old))
|
|
749
|
+
except Exception:
|
|
750
|
+
pass
|
|
751
|
+
except Exception:
|
|
752
|
+
pass
|
|
753
|
+
return recs
|
|
754
|
+
# fallback:逐分片读取(HTTP 模式,或 git archive 异常时)
|
|
755
|
+
recs = []
|
|
756
|
+
# git show / HTTP GET 均为 I/O 密集,线程池并发拉取可大幅缩短首次构建耗时
|
|
757
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
|
|
758
|
+
for part in ex.map(_read_fp_shard, fp_shards):
|
|
759
|
+
recs.extend(part)
|
|
760
|
+
_fp_index = recs
|
|
761
|
+
_fp_index_key = key
|
|
762
|
+
if cp:
|
|
763
|
+
try:
|
|
764
|
+
with open(cp, "w", encoding="utf-8") as f:
|
|
765
|
+
json.dump(recs, f, ensure_ascii=False)
|
|
766
|
+
for old in os.listdir(CACHE_DIR):
|
|
767
|
+
if old.startswith("fpindex-") and old != os.path.basename(cp):
|
|
768
|
+
try:
|
|
769
|
+
os.remove(os.path.join(CACHE_DIR, old))
|
|
770
|
+
except Exception:
|
|
771
|
+
pass
|
|
772
|
+
except Exception:
|
|
773
|
+
pass
|
|
774
|
+
return recs
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
_CAP_TERMS_CACHE: Dict[str, Any] = {}
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def _cap_term_of(code: str) -> str:
|
|
781
|
+
"""能力键 → 中文词面(`tech_ai` → `人工智能与算法 人工智能 AI 算法 …`)。
|
|
782
|
+
|
|
783
|
+
词面表来自 `skills/registry/index/cap.json`(发布产物,与 city.json 同构)——
|
|
784
|
+
刻意**不**把词面写进指纹:那一层按字节计费(约 139k 条),而词面表只有几十 KB,
|
|
785
|
+
且改词面(加别名)不需要重建指纹。
|
|
786
|
+
|
|
787
|
+
取不到时退回键本身:`caps_index` 允许出现码表之外的裸值(材料「不锈钢」、
|
|
788
|
+
品类「正餐」),它们本身就是可检索词,丢掉等于让这类查询瞎掉。
|
|
789
|
+
"""
|
|
790
|
+
if "v" not in _CAP_TERMS_CACHE:
|
|
791
|
+
terms: Dict[str, Any] = {}
|
|
792
|
+
try:
|
|
793
|
+
txt = _index_text("skills/registry/index/cap.json")
|
|
794
|
+
if txt:
|
|
795
|
+
terms = (json.loads(txt).get("terms") or {})
|
|
796
|
+
except Exception:
|
|
797
|
+
terms = {}
|
|
798
|
+
_CAP_TERMS_CACHE["v"] = terms
|
|
799
|
+
return str(_CAP_TERMS_CACHE["v"].get(code) or code or "")
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _hay(rec: Dict[str, Any]) -> str:
|
|
803
|
+
"""检索面:query 的每个词都必须出现在这里(AND 全命中)。
|
|
804
|
+
|
|
805
|
+
2026-09-23 加入 `cap`(跨门类能力键)—— 在此之前的六个字段
|
|
806
|
+
(co/city/dist/gb/proc/mat/cert/products)全都表达不了「能力」:
|
|
807
|
+
赤兔智能的键是 `tech_ai`,而它的厂名/工艺/材料里一个「人工智能」都没有,
|
|
808
|
+
客户问「苏州做人工智能的企业」就永远 0 条。键必须摊成中文词面才能被中文命中。
|
|
809
|
+
"""
|
|
810
|
+
parts = [
|
|
811
|
+
# `or ""` 不是多余:gb 为 None(未归类)时 str() 会产出字面量 "None",
|
|
812
|
+
# 2271 条 gb=null 的记录于是每条都带一个 "None" 词面 —— 查 "none" 能命中
|
|
813
|
+
# 整个未归类批(假阳性),而且白占检索面字节。空值就是空值。
|
|
814
|
+
" ".join(str(rec.get(k) or "") for k in ("co", "city", "dist", "gb")),
|
|
815
|
+
" ".join(rec.get("proc", []) or []),
|
|
816
|
+
" ".join(rec.get("mat", []) or []),
|
|
817
|
+
" ".join(rec.get("cert", []) or []),
|
|
818
|
+
" ".join(rec.get("products", []) or []),
|
|
819
|
+
" ".join(_cap_term_of(c) for c in (rec.get("cap") or [])),
|
|
820
|
+
]
|
|
821
|
+
return " ".join(p for p in parts if p)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _rec_summary(rec: Dict[str, Any]) -> Dict[str, Any]:
|
|
825
|
+
return {
|
|
826
|
+
"id": rec.get("id"),
|
|
827
|
+
"company": rec.get("co"),
|
|
828
|
+
"city": rec.get("city"),
|
|
829
|
+
"district": rec.get("dist"),
|
|
830
|
+
"gb": rec.get("gb"),
|
|
831
|
+
"badge": rec.get("cl"),
|
|
832
|
+
"score": rec.get("sc"),
|
|
833
|
+
"has_phone": bool(rec.get("tel")),
|
|
834
|
+
"process": rec.get("proc", []),
|
|
835
|
+
"material": rec.get("mat", []),
|
|
836
|
+
"cert": rec.get("cert", []),
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
INDEX_DIR = "skills/registry/index"
|
|
841
|
+
INDEX_VERSION = 3 # 倒排 key 为分片路径;v2 用国标码会漏掉 gb=null 的记录
|
|
842
|
+
_index_meta_cache: Optional[Dict[str, Any]] = None
|
|
843
|
+
_bucket_cache: Dict[str, Dict[str, Any]] = {}
|
|
844
|
+
_shard_rec_cache: Dict[str, List[Dict[str, Any]]] = {}
|
|
845
|
+
|
|
846
|
+
_RE_CJK = re.compile("[\u4e00-\u9fff\u3400-\u4dbf]+")
|
|
847
|
+
_RE_WORD = re.compile(r"[a-z0-9]+")
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _tokenize(text: str) -> List[str]:
|
|
851
|
+
"""切成可索引的词。必须与 scripts/gen_search_index.py 的 tokenize 保持一致。
|
|
852
|
+
|
|
853
|
+
CJK 取 1-gram + 2-gram;拉丁/数字取整词 + 长度 >= 2 的前缀。
|
|
854
|
+
"""
|
|
855
|
+
return _grams(text, query_mode=False)
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _grams(text: str, query_mode: bool = False) -> List[str]:
|
|
859
|
+
"""query_mode=True 时,长度 >= 2 的 CJK 串只用 2-gram。
|
|
860
|
+
|
|
861
|
+
原因:查询侧若同时用 1-gram 求交,含「酒」和「店」但不含「酒店」的分片
|
|
862
|
+
也会被算成候选,白白多拉分片(上海+酒店实测 20 片)。2-gram 已足以保证
|
|
863
|
+
召回(含「酒店」的记录必然含 bigram「酒店」),单字查询仍走 1-gram。
|
|
864
|
+
"""
|
|
865
|
+
t = (text or "").lower()
|
|
866
|
+
out = set()
|
|
867
|
+
for run in _RE_CJK.findall(t):
|
|
868
|
+
long_run = len(run) >= 2
|
|
869
|
+
for i, ch in enumerate(run):
|
|
870
|
+
if not (query_mode and long_run):
|
|
871
|
+
out.add(ch)
|
|
872
|
+
if i + 2 <= len(run):
|
|
873
|
+
out.add(run[i:i + 2])
|
|
874
|
+
for w in _RE_WORD.findall(t):
|
|
875
|
+
if len(w) >= 2:
|
|
876
|
+
out.add(w)
|
|
877
|
+
for n in range(2, min(len(w), 12)):
|
|
878
|
+
out.add(w[:n])
|
|
879
|
+
elif w:
|
|
880
|
+
out.add(w)
|
|
881
|
+
return sorted(out)
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def _index_text(relpath: str) -> Optional[str]:
|
|
885
|
+
"""读索引文件。git 模式按 HEAD sha 落盘缓存(桶均 11KB),
|
|
886
|
+
缓存后跨进程重复查询几乎零成本。
|
|
887
|
+
"""
|
|
888
|
+
cp = None
|
|
889
|
+
if REPO:
|
|
890
|
+
h = _head_sha() or "nosha"
|
|
891
|
+
cp = os.path.join(CACHE_DIR, "idx-" +
|
|
892
|
+
hashlib.sha1((h + "|" + relpath).encode("utf-8")).hexdigest() + ".json")
|
|
893
|
+
if os.path.exists(cp):
|
|
894
|
+
try:
|
|
895
|
+
with open(cp, "r", encoding="utf-8") as f:
|
|
896
|
+
return f.read()
|
|
897
|
+
except Exception:
|
|
898
|
+
pass
|
|
899
|
+
txt = fetch_text(relpath)
|
|
900
|
+
if txt and cp:
|
|
901
|
+
try:
|
|
902
|
+
os.makedirs(CACHE_DIR, exist_ok=True)
|
|
903
|
+
with open(cp, "w", encoding="utf-8") as f:
|
|
904
|
+
f.write(txt)
|
|
905
|
+
except Exception:
|
|
906
|
+
pass
|
|
907
|
+
return txt
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def _git_read_many(paths: List[str]) -> Dict[str, str]:
|
|
911
|
+
"""git 模式:一次 `git archive` 批量取出多个文件。
|
|
912
|
+
|
|
913
|
+
「每个文件一次 git show」是冷启动的主要开销(18 个分片 ≈ 5s),
|
|
914
|
+
批量取把 N 次 subprocess 降到 1 次。只读已提交 object,安全边界不变。
|
|
915
|
+
"""
|
|
916
|
+
if not REPO or not paths:
|
|
917
|
+
return {}
|
|
918
|
+
try:
|
|
919
|
+
r = subprocess.run(["git", "-C", REPO, "archive", "HEAD", "--", *paths],
|
|
920
|
+
capture_output=True)
|
|
921
|
+
if r.returncode != 0 or not r.stdout:
|
|
922
|
+
return {}
|
|
923
|
+
out: Dict[str, str] = {}
|
|
924
|
+
with tarfile.open(fileobj=io.BytesIO(r.stdout), mode="r:*") as tf:
|
|
925
|
+
for m in tf.getmembers():
|
|
926
|
+
if not m.isfile():
|
|
927
|
+
continue
|
|
928
|
+
try:
|
|
929
|
+
out[m.name.replace("\\", "/").lstrip("/")] = \
|
|
930
|
+
tf.extractfile(m).read().decode("utf-8", "replace")
|
|
931
|
+
except Exception:
|
|
932
|
+
continue
|
|
933
|
+
return out
|
|
934
|
+
except Exception:
|
|
935
|
+
return {}
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
def _read_many_text(paths: List[str]) -> Dict[str, str]:
|
|
939
|
+
"""批量取文本:git 模式 1 次 archive;HTTP 模式线程池并发。"""
|
|
940
|
+
if not paths:
|
|
941
|
+
return {}
|
|
942
|
+
if REPO:
|
|
943
|
+
got = _git_read_many(paths)
|
|
944
|
+
if len(got) >= max(1, len(paths) // 2): # archive 正常覆盖
|
|
945
|
+
return got
|
|
946
|
+
out: Dict[str, str] = {}
|
|
947
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
|
|
948
|
+
for path, txt in zip(paths, ex.map(fetch_text, paths)):
|
|
949
|
+
if txt:
|
|
950
|
+
out[path] = txt
|
|
951
|
+
return out
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def _index_meta() -> Optional[Dict[str, Any]]:
|
|
955
|
+
global _index_meta_cache
|
|
956
|
+
if _index_meta_cache is not None:
|
|
957
|
+
return _index_meta_cache
|
|
958
|
+
txt = _index_text(INDEX_DIR + "/meta.json")
|
|
959
|
+
if not txt:
|
|
960
|
+
return None
|
|
961
|
+
try:
|
|
962
|
+
_index_meta_cache = json.loads(txt)
|
|
963
|
+
except Exception:
|
|
964
|
+
return None
|
|
965
|
+
return _index_meta_cache
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _index_fresh() -> bool:
|
|
969
|
+
"""索引可用且版本匹配、不落后于数据 → True;否则回退全量扫描。
|
|
970
|
+
|
|
971
|
+
索引用等号判「记录数一致」会永远失败:它是提交前从工作树构建的,
|
|
972
|
+
天然比已提交快照新。因此判「索引不落后于数据」即可 —— 索引更新只会让
|
|
973
|
+
候选分片更全,真正的精确过滤仍在分片侧做,结果等价。
|
|
974
|
+
"""
|
|
975
|
+
meta = _index_meta()
|
|
976
|
+
if not meta:
|
|
977
|
+
return False
|
|
978
|
+
try:
|
|
979
|
+
if int(meta.get("version", 0)) != INDEX_VERSION:
|
|
980
|
+
return False
|
|
981
|
+
tot = sum(s.get("k", 0) for s in _shards_of_type("fp"))
|
|
982
|
+
rec = int(meta.get("records", -1))
|
|
983
|
+
return rec >= tot > 0
|
|
984
|
+
except Exception:
|
|
985
|
+
return False
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _bucket_rel(term: str, buckets: int) -> str:
|
|
989
|
+
b = int(hashlib.sha1(term.encode("utf-8")).hexdigest(), 16) % buckets
|
|
990
|
+
return "%s/terms/b%04d.json" % (INDEX_DIR, b)
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
# --- 自由文本需求 -> 可靠产品词:过滤 + 加权 -------------------------------- #
|
|
994
|
+
# df < MIN 的 bigram 几乎必是分词碎片(「送线」df=0),不作证据。
|
|
995
|
+
# 泛词(城市名/企业通名)已由 `_usable_gram` 拦掉,故不另设 df 上限。
|
|
996
|
+
_RECALL_DF_MIN = 2
|
|
997
|
+
# 以功能字开头/结尾、或以企业通名结尾的 bigram 不是产品词(「的输」「线厂」「厂家」)。
|
|
998
|
+
_FUNC_CHARS = set("的了要找做想帮我你在和与或及是有能可会请给把被用让需个些这那们吧呢吗来去上下就到从对为")
|
|
999
|
+
_ORG_SUFFIX = set("厂家司部行店商社")
|
|
1000
|
+
_city_names_cache: Optional[set] = None
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def _city_names() -> set:
|
|
1004
|
+
"""倒排索引里出现过的城市名集合:地区词不作为相关性证据(只作筛选偏好)。"""
|
|
1005
|
+
global _city_names_cache
|
|
1006
|
+
if _city_names_cache is not None:
|
|
1007
|
+
return _city_names_cache
|
|
1008
|
+
names: set = set()
|
|
1009
|
+
txt = _index_text(INDEX_DIR + "/city.json")
|
|
1010
|
+
if txt:
|
|
1011
|
+
try:
|
|
1012
|
+
names = set(json.loads(txt).keys())
|
|
1013
|
+
except Exception:
|
|
1014
|
+
names = set()
|
|
1015
|
+
_city_names_cache = names
|
|
1016
|
+
return names
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def _is_cjk(s: str) -> bool:
|
|
1020
|
+
return bool(s) and all("\u4e00" <= ch <= "\u9fff" or "\u3400" <= ch <= "\u4dbf" for ch in s)
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _usable_gram(t: str) -> bool:
|
|
1024
|
+
"""该查询 gram 是否为可采信的产品词:滤掉单字功能词、分词碎片、地区词与企业通名。"""
|
|
1025
|
+
if not t:
|
|
1026
|
+
return False
|
|
1027
|
+
if len(t) == 1:
|
|
1028
|
+
return t not in _FUNC_CHARS # 单字功能词(的/要/找…)不作证据
|
|
1029
|
+
if _is_cjk(t):
|
|
1030
|
+
if t[0] in _FUNC_CHARS or t[-1] in _FUNC_CHARS or t[-1] in _ORG_SUFFIX:
|
|
1031
|
+
return False
|
|
1032
|
+
if t in _city_names():
|
|
1033
|
+
return False
|
|
1034
|
+
return True
|
|
1035
|
+
|
|
1036
|
+
|
|
1037
|
+
def _collapse_prefixes(toks: set) -> set:
|
|
1038
|
+
"""英文前缀 token 折叠:`_grams('iso9001')` 会派生 is/iso/iso9/…/iso9001。
|
|
1039
|
+
|
|
1040
|
+
若全留着,一个「ISO9001」会被算成 6 次重复命中(分数被放大),还会误命中
|
|
1041
|
+
『BLU ISOLA cafe』这类把 is/iso 当子串的名字。这里只保留最长的那个前缀。
|
|
1042
|
+
"""
|
|
1043
|
+
latin = sorted([t for t in toks if t.isascii() and t.isalnum()], key=len, reverse=True)
|
|
1044
|
+
keep: List[str] = []
|
|
1045
|
+
for t in latin:
|
|
1046
|
+
if not any(k.startswith(t) for k in keep):
|
|
1047
|
+
keep.append(t)
|
|
1048
|
+
keep_set = set(keep)
|
|
1049
|
+
return {t for t in toks if not (t.isascii() and t.isalnum()) or t in keep_set}
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _bucket_postings(gram: str, buckets: int) -> Dict[str, Any]:
|
|
1053
|
+
"""取倒排索引里某词的 postings {分片路径: 命中条数};缺失/异常返回空 dict。"""
|
|
1054
|
+
rel = _bucket_rel(gram, buckets)
|
|
1055
|
+
b = _bucket_cache.get(rel)
|
|
1056
|
+
if b is None:
|
|
1057
|
+
txt = _index_text(rel)
|
|
1058
|
+
try:
|
|
1059
|
+
b = json.loads(txt) if txt else {}
|
|
1060
|
+
except Exception:
|
|
1061
|
+
b = {}
|
|
1062
|
+
_bucket_cache[rel] = b
|
|
1063
|
+
post = b.get(gram) if isinstance(b, dict) else None
|
|
1064
|
+
return post if isinstance(post, dict) else {}
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _bucket_df(gram: str, buckets: int) -> int:
|
|
1068
|
+
"""某词的文档频次 df = postings 各分片命中数之和。索引缺失返回 0。"""
|
|
1069
|
+
post = _bucket_postings(gram, buckets)
|
|
1070
|
+
if not post:
|
|
1071
|
+
return 0
|
|
1072
|
+
try:
|
|
1073
|
+
return sum(int(v) for v in post.values())
|
|
1074
|
+
except Exception:
|
|
1075
|
+
return len(post)
|
|
1076
|
+
|
|
1077
|
+
|
|
1078
|
+
def _gram_idf(grams) -> Dict[str, float]:
|
|
1079
|
+
"""按倒排索引的 df 给产品词估 IDF 权重 = 1/(1+ln(df))。
|
|
1080
|
+
|
|
1081
|
+
稀有的真产品词(「输送」df=6 -> 0.36)权重高;泛词权重低、自然让位,
|
|
1082
|
+
碎片(df<MIN,如「送线」)直接剔除。索引不可用时返回空 dict,
|
|
1083
|
+
调用方退化为均匀权重 1.0。
|
|
1084
|
+
"""
|
|
1085
|
+
meta = _index_meta()
|
|
1086
|
+
if not meta:
|
|
1087
|
+
return {}
|
|
1088
|
+
try:
|
|
1089
|
+
buckets = int(meta["buckets"])
|
|
1090
|
+
except Exception:
|
|
1091
|
+
return {}
|
|
1092
|
+
out: Dict[str, float] = {}
|
|
1093
|
+
for g in grams:
|
|
1094
|
+
if not g:
|
|
1095
|
+
continue
|
|
1096
|
+
d = _bucket_df(g, buckets)
|
|
1097
|
+
if d < _RECALL_DF_MIN:
|
|
1098
|
+
continue
|
|
1099
|
+
out[g] = 1.0 / (1.0 + math.log(d))
|
|
1100
|
+
return out
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
def _recall_candidates(toks) -> Optional[List[Dict[str, Any]]]:
|
|
1104
|
+
"""倒排索引『区分词并集』快速召回:取并集(OR)而非交集。
|
|
1105
|
+
|
|
1106
|
+
锚定所有采信的产品词(df >= _RECALL_DF_MIN)。碎片/地区词/通名已在 `_usable_gram`
|
|
1107
|
+
与 `_gram_idf` 阶段剔除,故这里只做并集。索引不可用、无采信词、或并集覆盖过大
|
|
1108
|
+
(≥60% 分片)时返回 None,交调用方全量扫描兜底(结果一致)。
|
|
1109
|
+
"""
|
|
1110
|
+
if not _index_fresh():
|
|
1111
|
+
return None
|
|
1112
|
+
meta = _index_meta() or {}
|
|
1113
|
+
try:
|
|
1114
|
+
buckets = int(meta["buckets"])
|
|
1115
|
+
except Exception:
|
|
1116
|
+
return None
|
|
1117
|
+
|
|
1118
|
+
known = {s.get("p") for s in _shards_of_type("fp")}
|
|
1119
|
+
picked: set = set()
|
|
1120
|
+
for g in toks:
|
|
1121
|
+
if not g:
|
|
1122
|
+
continue
|
|
1123
|
+
if _bucket_df(g, buckets) < _RECALL_DF_MIN: # 碎片 -> 不作锚
|
|
1124
|
+
continue
|
|
1125
|
+
picked.update(p for p in _bucket_postings(g, buckets) if p in known)
|
|
1126
|
+
if not picked:
|
|
1127
|
+
return None
|
|
1128
|
+
if len(picked) >= max(1, int(len(known) * 0.6)): # 并集过大,全量扫描更划算
|
|
1129
|
+
return None
|
|
1130
|
+
return _records_from_paths(sorted(picked))
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
def _candidate_shards(tokens: List[str], city: str) -> Optional[List[str]]:
|
|
1134
|
+
"""用预构建索引求候选分片路径;索引不可用/版本不符/陈旧时返回 None 交回退。
|
|
1135
|
+
|
|
1136
|
+
返回空列表表示「索引明确判定无命中」,无需拉任何分片。
|
|
1137
|
+
"""
|
|
1138
|
+
if not _index_fresh():
|
|
1139
|
+
return None
|
|
1140
|
+
meta = _index_meta() or {}
|
|
1141
|
+
try:
|
|
1142
|
+
buckets = int(meta["buckets"])
|
|
1143
|
+
except Exception:
|
|
1144
|
+
return None
|
|
1145
|
+
|
|
1146
|
+
gram_groups: List[List[str]] = []
|
|
1147
|
+
for tok in tokens:
|
|
1148
|
+
gs = _grams(tok, query_mode=True)
|
|
1149
|
+
if not gs:
|
|
1150
|
+
return None
|
|
1151
|
+
gram_groups.append(gs)
|
|
1152
|
+
|
|
1153
|
+
need: List[str] = []
|
|
1154
|
+
if city:
|
|
1155
|
+
need.append(INDEX_DIR + "/city.json")
|
|
1156
|
+
for gs in gram_groups:
|
|
1157
|
+
for g in gs:
|
|
1158
|
+
need.append(_bucket_rel(g, buckets))
|
|
1159
|
+
need = sorted(set(need))
|
|
1160
|
+
|
|
1161
|
+
raw = _read_many_text(need)
|
|
1162
|
+
docs: Dict[str, Any] = {}
|
|
1163
|
+
for k in need:
|
|
1164
|
+
t = raw.get(k)
|
|
1165
|
+
if not t:
|
|
1166
|
+
continue
|
|
1167
|
+
try:
|
|
1168
|
+
docs[k] = json.loads(t)
|
|
1169
|
+
except Exception:
|
|
1170
|
+
return None
|
|
1171
|
+
|
|
1172
|
+
cands: Optional[set] = None
|
|
1173
|
+
if city:
|
|
1174
|
+
cd = docs.get(INDEX_DIR + "/city.json")
|
|
1175
|
+
if cd is None:
|
|
1176
|
+
return None
|
|
1177
|
+
m = cd.get(city)
|
|
1178
|
+
if not m: # 索引里没这个城市 → 确无命中
|
|
1179
|
+
return []
|
|
1180
|
+
cands = set(m.keys())
|
|
1181
|
+
|
|
1182
|
+
for gs in gram_groups:
|
|
1183
|
+
tset: Optional[set] = None
|
|
1184
|
+
for g in gs:
|
|
1185
|
+
d = docs.get(_bucket_rel(g, buckets))
|
|
1186
|
+
m = d.get(g) if isinstance(d, dict) else None
|
|
1187
|
+
s = set(m.keys()) if m else set() # 桶里没这个词 → 确无命中
|
|
1188
|
+
tset = s if tset is None else (tset & s)
|
|
1189
|
+
if not tset:
|
|
1190
|
+
break
|
|
1191
|
+
if tset is None:
|
|
1192
|
+
return None
|
|
1193
|
+
cands = tset if cands is None else (cands & tset)
|
|
1194
|
+
if not cands:
|
|
1195
|
+
return []
|
|
1196
|
+
|
|
1197
|
+
if cands is None:
|
|
1198
|
+
return None
|
|
1199
|
+
# 与 manifest 求交:防止陈旧索引指向已删除的分片
|
|
1200
|
+
known = {s.get("p") for s in _shards_of_type("fp")}
|
|
1201
|
+
return sorted(p for p in cands if p in known)
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _records_from_paths(paths: List[str]) -> List[Dict[str, Any]]:
|
|
1205
|
+
"""批量读取若干 fp 分片的记录,并在进程内按路径缓存(重复查询近乎零成本)。"""
|
|
1206
|
+
out: List[Dict[str, Any]] = []
|
|
1207
|
+
if not paths:
|
|
1208
|
+
return out
|
|
1209
|
+
todo = [p for p in paths if p not in _shard_rec_cache]
|
|
1210
|
+
for p in todo:
|
|
1211
|
+
_shard_rec_cache[p] = []
|
|
1212
|
+
for _path, txt in _read_many_text(todo).items():
|
|
1213
|
+
recs: List[Dict[str, Any]] = []
|
|
1214
|
+
for line in txt.splitlines():
|
|
1215
|
+
line = line.strip()
|
|
1216
|
+
if not line:
|
|
1217
|
+
continue
|
|
1218
|
+
try:
|
|
1219
|
+
recs.append(json.loads(line))
|
|
1220
|
+
except Exception:
|
|
1221
|
+
continue
|
|
1222
|
+
_shard_rec_cache[_path] = recs
|
|
1223
|
+
for p in paths:
|
|
1224
|
+
out.extend(_shard_rec_cache.get(p, []))
|
|
1225
|
+
return out
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _city_ok(rec: Dict[str, Any], city: str) -> bool:
|
|
1229
|
+
"""city 匹配「地级市 或 区县」:县级市(昆山/海盐…)在高德里归到地级市名下,
|
|
1230
|
+
记录里 city=苏州、dist=昆山。只比 city 的话查「昆山」永远 0 条。
|
|
1231
|
+
"""
|
|
1232
|
+
if not city:
|
|
1233
|
+
return True
|
|
1234
|
+
return city in (rec.get("city", ""), rec.get("dist", ""))
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def search_vendors(query: str = "", city: str = "", gb: str = "",
|
|
1238
|
+
limit: int = 20, offset: int = 0) -> Dict[str, Any]:
|
|
1239
|
+
limit = max(1, min(int(limit), 200))
|
|
1240
|
+
offset = max(0, int(offset))
|
|
1241
|
+
q = (query or "").strip().lower()
|
|
1242
|
+
# 多词按空格分词,要求全部命中(AND)——这样「上海 酒店」也能正确匹配,
|
|
1243
|
+
# 而非必须作为连续子串出现。单关键词时退化为原行为。
|
|
1244
|
+
tokens = [t for t in q.split() if t]
|
|
1245
|
+
|
|
1246
|
+
# 能力(cap)别名:把口语词(AI / 人工智能 / 机器学习 / 大模型 / 算法 …)映射到
|
|
1247
|
+
# 能力键(如 tech_ai),并**消费**掉该 token(不再做通用子串 AND 校验),只走 cap
|
|
1248
|
+
# 定向召回 —— 否则「AI」会作为子串命中 algebraist / Ashore 等英文名咖啡店造成噪声。
|
|
1249
|
+
cap_hits = cap_alias_codes(q) if q else []
|
|
1250
|
+
# 只消费「会造成子串噪声的短 ASCII token」(如 ai 会子串命中 algebraist 等英文名),
|
|
1251
|
+
# 中文/较长 token 不消费 —— 保留其文本匹配(厂名含「人工智能」的企业不被误丢),
|
|
1252
|
+
# 同时下方 cap 召回仍会叠加,最终是「文本 ∪ 能力」的并集。
|
|
1253
|
+
cap_tokens = {h["word"].lower() for h in cap_hits
|
|
1254
|
+
if len(h["word"]) <= 2 and h["word"].isascii()}
|
|
1255
|
+
gen_tokens = [t for t in tokens if t not in cap_tokens]
|
|
1256
|
+
|
|
1257
|
+
# 口语词 → 国标码(2026-09-24 接入别名表)。
|
|
1258
|
+
# 「输送线/流水线/PCB」这类词在任何记录的厂名/工艺/材料里都不出现,纯子串匹配
|
|
1259
|
+
# 必然 0 条。别名命中的记录按码定向召回,**豁免 AND token 校验** —— 不豁免的话
|
|
1260
|
+
# 刚拉进来就又被 _hay 判定「不含输送线」而滤掉,等于白接。
|
|
1261
|
+
alias_hits = alias_codes(q) if q else []
|
|
1262
|
+
alias_by_code = {a["code"]: a for a in alias_hits}
|
|
1263
|
+
alias_paths = _fp_paths_for_gbs(list(alias_by_code)) if alias_by_code else []
|
|
1264
|
+
|
|
1265
|
+
# 给了国标码:只扫对应分片(最快路径,不构建全量索引)
|
|
1266
|
+
if gb:
|
|
1267
|
+
fp_shards = [s for s in _shards_of_type("fp") if s.get("c") == gb]
|
|
1268
|
+
scanned = 0
|
|
1269
|
+
matches: List[Dict[str, Any]] = []
|
|
1270
|
+
for s in fp_shards:
|
|
1271
|
+
scanned += 1
|
|
1272
|
+
for rec in _read_fp_shard(s):
|
|
1273
|
+
if not _city_ok(rec, city):
|
|
1274
|
+
continue
|
|
1275
|
+
if gen_tokens and not all(tok in _hay(rec).lower() for tok in gen_tokens):
|
|
1276
|
+
continue
|
|
1277
|
+
matches.append(_rec_summary(rec))
|
|
1278
|
+
return {
|
|
1279
|
+
"total_matched": len(matches),
|
|
1280
|
+
"returned": len(matches[offset:offset + limit]),
|
|
1281
|
+
"shards_scanned": scanned,
|
|
1282
|
+
"results": matches[offset:offset + limit],
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
# 未给国标码:优先走「预构建倒排索引 → 只拉命中分片」(O(命中量))。
|
|
1286
|
+
# 索引缺失或陈旧时回退进程内全量索引(O(总量),慢但结果等价)。
|
|
1287
|
+
cands = _candidate_shards(gen_tokens, city) if (gen_tokens or city) else None
|
|
1288
|
+
via_index = cands is not None
|
|
1289
|
+
|
|
1290
|
+
if via_index and not cands and not alias_paths and not cap_hits:
|
|
1291
|
+
# 索引明确判定无命中、别名也没给方向 —— 无需拉任何分片
|
|
1292
|
+
return {
|
|
1293
|
+
"total_matched": 0,
|
|
1294
|
+
"returned": 0,
|
|
1295
|
+
"shards_scanned": 0,
|
|
1296
|
+
"via_index": True,
|
|
1297
|
+
"results": [],
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
matches: List[Dict[str, Any]] = []
|
|
1301
|
+
seen: set = set()
|
|
1302
|
+
|
|
1303
|
+
def _collect(recs, relax_tokens: bool = False) -> None:
|
|
1304
|
+
for rec in recs:
|
|
1305
|
+
if not _city_ok(rec, city):
|
|
1306
|
+
continue
|
|
1307
|
+
if not relax_tokens and gen_tokens and not all(tok in _hay(rec).lower() for tok in gen_tokens):
|
|
1308
|
+
continue
|
|
1309
|
+
rid = rec.get("id")
|
|
1310
|
+
if rid in seen: # 别名召回与文本召回的并集要去重
|
|
1311
|
+
continue
|
|
1312
|
+
seen.add(rid)
|
|
1313
|
+
s = _rec_summary(rec)
|
|
1314
|
+
if relax_tokens:
|
|
1315
|
+
a = alias_by_code.get(rec.get("gb") or "")
|
|
1316
|
+
if a:
|
|
1317
|
+
s["alias_match"] = {"word": a["word"], "gb": a["code"], "name": a["name"]}
|
|
1318
|
+
matches.append(s)
|
|
1319
|
+
|
|
1320
|
+
if via_index:
|
|
1321
|
+
# 只拉候选分片(git 模式一次 archive 批量取,通常 1~N 个)。
|
|
1322
|
+
# 若全部 token 都被 cap 别名消费掉(gen_tokens 空),且本就有 cap 命中,
|
|
1323
|
+
# 则跳过文本召回(否则 city 命中会拉回整座城市的全部记录),只走下方 cap 召回。
|
|
1324
|
+
if gen_tokens or not cap_hits:
|
|
1325
|
+
scanned = len(cands or [])
|
|
1326
|
+
_collect(_records_from_paths(cands or []))
|
|
1327
|
+
else:
|
|
1328
|
+
scanned = 0
|
|
1329
|
+
else:
|
|
1330
|
+
if gen_tokens or not cap_hits:
|
|
1331
|
+
scanned = len(_shards_of_type("fp"))
|
|
1332
|
+
_collect(_build_fp_index())
|
|
1333
|
+
else:
|
|
1334
|
+
scanned = 0
|
|
1335
|
+
|
|
1336
|
+
# 别名定向召回(后追加,纯度次之)
|
|
1337
|
+
alias_scanned = 0
|
|
1338
|
+
for p in alias_paths:
|
|
1339
|
+
for rec in _records_from_paths([p]):
|
|
1340
|
+
alias_scanned += 1
|
|
1341
|
+
if not _city_ok(rec, city):
|
|
1342
|
+
continue
|
|
1343
|
+
rid = rec.get("id")
|
|
1344
|
+
if rid in seen:
|
|
1345
|
+
continue
|
|
1346
|
+
seen.add(rid)
|
|
1347
|
+
s = _rec_summary(rec)
|
|
1348
|
+
a = alias_by_code.get(rec.get("gb") or "")
|
|
1349
|
+
if a:
|
|
1350
|
+
s["alias_match"] = {"word": a["word"], "gb": a["code"], "name": a["name"]}
|
|
1351
|
+
matches.append(s)
|
|
1352
|
+
|
|
1353
|
+
# 能力(cap)别名定向召回(消费 token,纯度最高,最后追加并去重)。
|
|
1354
|
+
# 按能力键扫对应指纹分片,只收 cap 含该键且城市命中的记录;其余通用 token
|
|
1355
|
+
# (gen_tokens)仍做 AND 校验,保证「AI 喷涂」这类组合查询不跑偏。
|
|
1356
|
+
cap_scanned = 0
|
|
1357
|
+
for h in cap_hits:
|
|
1358
|
+
for p in _cap_shard_paths(h["cap"]):
|
|
1359
|
+
for rec in _records_from_paths([p]):
|
|
1360
|
+
cap_scanned += 1
|
|
1361
|
+
# 该分片可能含多条记录,只收真正带此能力键的(cap.json 的 shards
|
|
1362
|
+
# 表只是「哪些分片含此 cap」,分片内还需按 cap 成员过滤)。
|
|
1363
|
+
if h["cap"] not in (rec.get("cap") or []):
|
|
1364
|
+
continue
|
|
1365
|
+
if not _city_ok(rec, city):
|
|
1366
|
+
continue
|
|
1367
|
+
if gen_tokens and not all(tok in _hay(rec).lower() for tok in gen_tokens):
|
|
1368
|
+
continue
|
|
1369
|
+
rid = rec.get("id")
|
|
1370
|
+
if rid in seen:
|
|
1371
|
+
continue
|
|
1372
|
+
seen.add(rid)
|
|
1373
|
+
s = _rec_summary(rec)
|
|
1374
|
+
s["alias_match"] = {"word": h["word"], "cap": h["cap"],
|
|
1375
|
+
"name": h["name"], "type": "cap"}
|
|
1376
|
+
matches.append(s)
|
|
1377
|
+
|
|
1378
|
+
out = {
|
|
1379
|
+
"total_matched": len(matches),
|
|
1380
|
+
"returned": len(matches[offset:offset + limit]),
|
|
1381
|
+
"shards_scanned": scanned,
|
|
1382
|
+
"via_index": via_index,
|
|
1383
|
+
"results": matches[offset:offset + limit],
|
|
1384
|
+
}
|
|
1385
|
+
if alias_hits:
|
|
1386
|
+
out["alias_expanded"] = [{"word": a["word"], "gb": a["code"], "name": a["name"]}
|
|
1387
|
+
for a in alias_hits]
|
|
1388
|
+
out["alias_shards_scanned"] = len(alias_paths)
|
|
1389
|
+
out["alias_records_seen"] = alias_scanned
|
|
1390
|
+
if cap_hits:
|
|
1391
|
+
out["cap_expanded"] = [{"word": h["word"], "cap": h["cap"], "name": h["name"]}
|
|
1392
|
+
for h in cap_hits]
|
|
1393
|
+
out["cap_records_seen"] = cap_scanned
|
|
1394
|
+
return out
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _pick(*vals: Any) -> Any:
|
|
1398
|
+
"""取第一个「非空」值。None/""/[]/{} 都算空 —— setdefault 会把 None 当已填,
|
|
1399
|
+
旧结构的 `city: null` 于是永远盖住新结构里的 region.city。
|
|
1400
|
+
"""
|
|
1401
|
+
for v in vals:
|
|
1402
|
+
if v not in (None, "", [], {}):
|
|
1403
|
+
return v
|
|
1404
|
+
return None
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
def normalize_vendor(rec: Dict[str, Any]) -> Dict[str, Any]:
|
|
1408
|
+
"""zh 分片存在两套历史结构,下游 agent 只认一种,在这里展平(只增不删)。
|
|
1409
|
+
|
|
1410
|
+
已归类:co / city / dist / gb ... (扁平)
|
|
1411
|
+
未归类:company / region.{province,city} / category / keywords ...
|
|
1412
|
+
manufacturer_id / industry / tel 两种写法都有,一并归一。
|
|
1413
|
+
"""
|
|
1414
|
+
out = dict(rec)
|
|
1415
|
+
region = rec.get("region") or {}
|
|
1416
|
+
out["co"] = _pick(rec.get("co"), rec.get("company"))
|
|
1417
|
+
out["company"] = _pick(rec.get("company"), rec.get("co"))
|
|
1418
|
+
out["city"] = _pick(rec.get("city"), region.get("city"))
|
|
1419
|
+
out["dist"] = _pick(rec.get("dist"), region.get("district"), region.get("dist"))
|
|
1420
|
+
out["province"] = _pick(rec.get("province"), region.get("province"))
|
|
1421
|
+
out["gb"] = rec.get("gb") if rec.get("gb") not in (None, "") else None
|
|
1422
|
+
out["keywords"] = _pick(rec.get("keywords"), rec.get("products"))
|
|
1423
|
+
if not out.get("gb"):
|
|
1424
|
+
out["gb_unclassified"] = True
|
|
1425
|
+
return out
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def get_vendor(vid: str, gb: str = "") -> Dict[str, Any]:
|
|
1429
|
+
vid = (vid or "").strip()
|
|
1430
|
+
if not vid:
|
|
1431
|
+
return {"error": "缺少必填参数 id"}
|
|
1432
|
+
gb = (gb or "").strip()
|
|
1433
|
+
# 没给国标码就先建 id->gb 索引(扫 fp 分片,HTTP 缓存加速)
|
|
1434
|
+
# 找到了就立刻停:gb 为 null(未归类)继续扫剩下的分片是纯浪费(275 次 IO)。
|
|
1435
|
+
fp_hit = False
|
|
1436
|
+
if not gb:
|
|
1437
|
+
for s in _shards_of_type("fp"):
|
|
1438
|
+
txt = fetch_text(s["p"])
|
|
1439
|
+
if not txt:
|
|
1440
|
+
continue
|
|
1441
|
+
for line in txt.splitlines():
|
|
1442
|
+
line = line.strip()
|
|
1443
|
+
if not line:
|
|
1444
|
+
continue
|
|
1445
|
+
try:
|
|
1446
|
+
rec = json.loads(line)
|
|
1447
|
+
except Exception:
|
|
1448
|
+
continue
|
|
1449
|
+
if rec.get("id") == vid:
|
|
1450
|
+
gb = rec.get("gb") or ""
|
|
1451
|
+
fp_hit = True
|
|
1452
|
+
break
|
|
1453
|
+
if fp_hit:
|
|
1454
|
+
break
|
|
1455
|
+
|
|
1456
|
+
if gb:
|
|
1457
|
+
zh_paths = _zh_paths_for_gb(gb)
|
|
1458
|
+
if not zh_paths:
|
|
1459
|
+
return {"error": f"国标码 {gb} 没有对应的 zh 分片"}
|
|
1460
|
+
for zh_path in zh_paths:
|
|
1461
|
+
txt = fetch_text(zh_path)
|
|
1462
|
+
if not txt:
|
|
1463
|
+
continue
|
|
1464
|
+
try:
|
|
1465
|
+
arr = json.loads(txt)
|
|
1466
|
+
except Exception as e:
|
|
1467
|
+
return {"error": f"分片 {zh_path} 解析失败: {e}"}
|
|
1468
|
+
for rec in arr:
|
|
1469
|
+
if rec.get("id") == vid:
|
|
1470
|
+
return {"vendor": normalize_vendor(rec)}
|
|
1471
|
+
return {"error": f"国标码 {gb} 的全部 zh 分片(共{len(zh_paths)}个)中均未找到 id={vid}"}
|
|
1472
|
+
|
|
1473
|
+
# 兜底:gb 为 null(未归类)—— 这些记录躺在 c=='' 的 zh 分片里
|
|
1474
|
+
# (主要是 _unclassified.json,外加几个 xxx/_partial.json)。
|
|
1475
|
+
# 2026-09-24 之前这条路径直接报「找不到国标码」,2271 条未归类企业等于查无此人。
|
|
1476
|
+
loose = [s.get("p") for s in _shards_of_type("zh") if not s.get("c")]
|
|
1477
|
+
for p in loose:
|
|
1478
|
+
txt = fetch_text(p)
|
|
1479
|
+
if not txt:
|
|
1480
|
+
continue
|
|
1481
|
+
try:
|
|
1482
|
+
arr = json.loads(txt)
|
|
1483
|
+
except Exception:
|
|
1484
|
+
continue
|
|
1485
|
+
for rec in arr:
|
|
1486
|
+
if rec.get("id") == vid:
|
|
1487
|
+
v = normalize_vendor(rec)
|
|
1488
|
+
v["_from"] = p
|
|
1489
|
+
return {"vendor": v}
|
|
1490
|
+
return {"error": f"找不到 id={vid} 对应的国标码,可能该记录尚未发布"}
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def get_capability_card(vid: str) -> Dict[str, Any]:
|
|
1494
|
+
vid = (vid or "").strip()
|
|
1495
|
+
if not vid:
|
|
1496
|
+
return {"error": "缺少必填参数 id"}
|
|
1497
|
+
rel = f"skills/registry/capability/{vid}.json"
|
|
1498
|
+
txt = fetch_text(rel)
|
|
1499
|
+
if not txt:
|
|
1500
|
+
return {
|
|
1501
|
+
"id": vid,
|
|
1502
|
+
"has_card": False,
|
|
1503
|
+
"note": "无已发布能力卡:该供应商为 L0 未认证,或能力卡尚未生成/尚未发布到云端。"
|
|
1504
|
+
"能力卡不进 git,仅经 Cloudflare R2 按需提供。",
|
|
1505
|
+
}
|
|
1506
|
+
try:
|
|
1507
|
+
card = json.loads(txt)
|
|
1508
|
+
except Exception as e:
|
|
1509
|
+
return {"error": f"能力卡解析失败: {e}"}
|
|
1510
|
+
return {"id": vid, "has_card": True, "card": card}
|
|
1511
|
+
|
|
1512
|
+
|
|
1513
|
+
# --------------------------------------------------------------------------- #
|
|
1514
|
+
# rfq-kernel 桥接 tool:多轮对话式供应商匹配(G1/G2/G3)
|
|
1515
|
+
# 入口只有一个:客户 agent 检测到采购/代工/找厂意图时自动调 start_sourcing。
|
|
1516
|
+
# --------------------------------------------------------------------------- #
|
|
1517
|
+
def start_sourcing(demand_text: str, audience_id: str = "domestic_downstream") -> Dict[str, Any]:
|
|
1518
|
+
"""【客户 agent 自动触发】当对话中出现找厂/代工/采购意图时调用。
|
|
1519
|
+
|
|
1520
|
+
内部:品类检测 → 指纹宽召回 → 解析需求 → 生成 1~2 轮澄清问题,把需求归一到
|
|
1521
|
+
beacon-mfg 供应商数据结构做初级筛选。返回 session_id 供后续轮次续接。
|
|
1522
|
+
"""
|
|
1523
|
+
if _bridge is None:
|
|
1524
|
+
return {"error": "rfq-kernel 桥接未就绪(skills/rfq-kernel 缺失)"}
|
|
1525
|
+
demand_text = (demand_text or "").strip()
|
|
1526
|
+
if not demand_text:
|
|
1527
|
+
return {"error": "缺少必填参数 demand_text"}
|
|
1528
|
+
pack_id = _bridge.detect_industry(demand_text)
|
|
1529
|
+
# 召回 query 并入检测行业的「精准」召回扩词(recall_terms),使检索只拉相关行业供应商
|
|
1530
|
+
# (收敛跨行业噪声),同时避免宽泛 2 字词把 OR 召回池冲爆而挤出长尾真实企业。
|
|
1531
|
+
# 未声明 recall_terms 的 pack 回退到全量 vocab(旧行为)。
|
|
1532
|
+
recall_query = demand_text
|
|
1533
|
+
if pack_id:
|
|
1534
|
+
vocab = _bridge.pack_recall_terms(pack_id)
|
|
1535
|
+
if vocab:
|
|
1536
|
+
recall_query = demand_text + " " + " ".join(vocab)
|
|
1537
|
+
recs = _recall_for_sourcing(recall_query, top_k=200, pack_id=pack_id)
|
|
1538
|
+
if not recs:
|
|
1539
|
+
return {"stage": "clarifying", "session_id": None, "candidates_found": 0,
|
|
1540
|
+
"clarifying_questions": [],
|
|
1541
|
+
"note": "未从名录中召回相关工厂,请换更具体的产品/工艺描述。"}
|
|
1542
|
+
state, resp = _bridge.build_session(demand_text, recs, pack_id, audience_id)
|
|
1543
|
+
sid = _new_session_id()
|
|
1544
|
+
_SESSIONS[sid] = state
|
|
1545
|
+
resp["session_id"] = sid
|
|
1546
|
+
return resp
|
|
1547
|
+
|
|
1548
|
+
|
|
1549
|
+
def answer_sourcing(session_id: str, answers: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
1550
|
+
"""续接澄清轮次:把客户回答写回,返回下一轮澄清问题或直接给出初选推荐。"""
|
|
1551
|
+
if _bridge is None:
|
|
1552
|
+
return {"error": "rfq-kernel 桥接未就绪"}
|
|
1553
|
+
state = _SESSIONS.get(session_id or "")
|
|
1554
|
+
if state is None:
|
|
1555
|
+
return {"error": f"session {session_id} 不存在或已过期(请重新 start_sourcing)"}
|
|
1556
|
+
state, resp = _bridge.answer_session(state, answers or {})
|
|
1557
|
+
_SESSIONS[session_id] = state
|
|
1558
|
+
resp["session_id"] = session_id
|
|
1559
|
+
return resp
|
|
1560
|
+
|
|
1561
|
+
|
|
1562
|
+
def refine_sourcing(session_id: str, action: str = "", value: Optional[str] = None) -> Dict[str, Any]:
|
|
1563
|
+
"""推荐轮次的交互:details(带 supplier_id 看详情/RFQ 入口) / more(带 N) / best。"""
|
|
1564
|
+
if _bridge is None:
|
|
1565
|
+
return {"error": "rfq-kernel 桥接未就绪"}
|
|
1566
|
+
state = _SESSIONS.get(session_id or "")
|
|
1567
|
+
if state is None:
|
|
1568
|
+
return {"error": f"session {session_id} 不存在或已过期(请重新 start_sourcing)"}
|
|
1569
|
+
resp = _bridge.refine_session(state, action or "", value)
|
|
1570
|
+
resp["session_id"] = session_id
|
|
1571
|
+
return resp
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
# --------------------------------------------------------------------------- #
|
|
1575
|
+
# MCP 协议层(JSON-RPC 2.0 over stdio)
|
|
1576
|
+
# --------------------------------------------------------------------------- #
|
|
1577
|
+
TOOLS = [
|
|
1578
|
+
{
|
|
1579
|
+
"name": "search_vendors",
|
|
1580
|
+
"description": "检索灯塔工厂供应商名录。可按关键词(企业名/工艺/材料/认证)、城市、国标码(GB/T 4754)过滤。"
|
|
1581
|
+
"返回精简档案(id/企业名/城市/国标码/认证等级/工艺/材料/认证/是否含电话)。",
|
|
1582
|
+
"inputSchema": {
|
|
1583
|
+
"type": "object",
|
|
1584
|
+
"properties": {
|
|
1585
|
+
"query": {"type": "string", "description": "关键词:企业名/工艺/材料/认证子串"},
|
|
1586
|
+
"city": {"type": "string",
|
|
1587
|
+
"description": "城市名精确匹配,如 深圳 / 东莞;"
|
|
1588
|
+
"也接受区县或县级市(如 昆山、海盐),"
|
|
1589
|
+
"这些地方在高德里归地级市名下,靠 district 字段命中"},
|
|
1590
|
+
"gb": {"type": "string", "description": "国标码,如 3484(机械零部件加工)。给了就只扫对应分片"},
|
|
1591
|
+
"limit": {"type": "integer", "default": 20, "description": "返回条数上限(1-200)"},
|
|
1592
|
+
"offset": {"type": "integer", "default": 0, "description": "分页偏移"},
|
|
1593
|
+
},
|
|
1594
|
+
},
|
|
1595
|
+
},
|
|
1596
|
+
{
|
|
1597
|
+
"name": "get_vendor",
|
|
1598
|
+
"description": "按供应商 id(+可选国标码)取完整中文档案:企业名/地址/经纬度/电话/行业路径/认证/关键词等。",
|
|
1599
|
+
"inputSchema": {
|
|
1600
|
+
"type": "object",
|
|
1601
|
+
"properties": {
|
|
1602
|
+
"id": {"type": "string", "description": "供应商 id,如 CN-MFG-0000163(必填)"},
|
|
1603
|
+
"gb": {"type": "string", "description": "国标码;不填会自动从指纹分片反查(稍慢)"},
|
|
1604
|
+
},
|
|
1605
|
+
"required": ["id"],
|
|
1606
|
+
},
|
|
1607
|
+
},
|
|
1608
|
+
{
|
|
1609
|
+
"name": "get_capability_card",
|
|
1610
|
+
"description": "按供应商 id 取能力卡(工艺位/设备/产能/认证/起订量等)。"
|
|
1611
|
+
"无已发布卡片时返回 has_card=false 及原因说明。",
|
|
1612
|
+
"inputSchema": {
|
|
1613
|
+
"type": "object",
|
|
1614
|
+
"properties": {
|
|
1615
|
+
"id": {"type": "string", "description": "供应商 id(必填)"},
|
|
1616
|
+
},
|
|
1617
|
+
"required": ["id"],
|
|
1618
|
+
},
|
|
1619
|
+
},
|
|
1620
|
+
{
|
|
1621
|
+
"name": "start_sourcing",
|
|
1622
|
+
"description": "【客户 agent 自动触发】当对话中出现找厂/代工/采购/询价意图时调用,"
|
|
1623
|
+
"例如『帮我找个能做不锈钢保温杯的厂』『哪家能做铝合金压铸』。"
|
|
1624
|
+
"内部做品类识别→指纹宽召回→解析需求→生成 1~2 轮澄清问题,"
|
|
1625
|
+
"把需求归一到 beacon-mfg 供应商数据结构做初级筛选,返回 session_id。"
|
|
1626
|
+
"后续用 answer_sourcing 续接澄清、refine_sourcing 看推荐详情。",
|
|
1627
|
+
"inputSchema": {
|
|
1628
|
+
"type": "object",
|
|
1629
|
+
"properties": {
|
|
1630
|
+
"demand_text": {"type": "string",
|
|
1631
|
+
"description": "客户的原始需求描述(必填),如『想找东莞做ISO9001的钣金厂』"},
|
|
1632
|
+
"audience_id": {"type": "string", "description": "客户视图:domestic_downstream(国内下游)/intl_buyer(国际采购商),默认国内下游"},
|
|
1633
|
+
},
|
|
1634
|
+
"required": ["demand_text"],
|
|
1635
|
+
},
|
|
1636
|
+
},
|
|
1637
|
+
{
|
|
1638
|
+
"name": "answer_sourcing",
|
|
1639
|
+
"description": "续接 start_sourcing 的澄清轮次:把客户对澄清问题的回答写回,"
|
|
1640
|
+
"返回下一轮澄清问题,或(1~2 轮后)直接给出按需求匹配度初选的供应商列表。",
|
|
1641
|
+
"inputSchema": {
|
|
1642
|
+
"type": "object",
|
|
1643
|
+
"properties": {
|
|
1644
|
+
"session_id": {"type": "string", "description": "start_sourcing 返回的会话 id(必填)"},
|
|
1645
|
+
"answers": {"type": "object",
|
|
1646
|
+
"description": "澄清答案,键为问题里的 field(如 certifications_required/material/process/region),值为选项"},
|
|
1647
|
+
},
|
|
1648
|
+
"required": ["session_id"],
|
|
1649
|
+
},
|
|
1650
|
+
},
|
|
1651
|
+
{
|
|
1652
|
+
"name": "refine_sourcing",
|
|
1653
|
+
"description": "推荐轮次的交互:details(带 supplier_id 看详情与 RFQ 在线入口) / more(带数字 N 看更多) / best(看最匹配一家)。",
|
|
1654
|
+
"inputSchema": {
|
|
1655
|
+
"type": "object",
|
|
1656
|
+
"properties": {
|
|
1657
|
+
"session_id": {"type": "string", "description": "会话 id(必填)"},
|
|
1658
|
+
"action": {"type": "string", "description": "details / more / best"},
|
|
1659
|
+
"value": {"type": "string", "description": "details 时为 supplier_id;more 时为数量 N"},
|
|
1660
|
+
},
|
|
1661
|
+
"required": ["session_id"],
|
|
1662
|
+
},
|
|
1663
|
+
},
|
|
1664
|
+
]
|
|
1665
|
+
|
|
1666
|
+
|
|
1667
|
+
def _chain_grams(grams: List[str], max_len: int = 8) -> List[str]:
|
|
1668
|
+
"""把重叠的 2-gram 链回原词:`流水` + `水线` → `流水线`。
|
|
1669
|
+
|
|
1670
|
+
为什么需要这一步:`_tokenize` 出于召回考虑会同时产出 1-gram 和 2-gram,
|
|
1671
|
+
直接取前 N 个得到的是「厂 / 家 / 水 / 流」这种碎片 —— 拿来当需求信号毫无意义
|
|
1672
|
+
(2026-09-23 实测:「上海的流水线厂家」抽出来是 `['厂','家','水','水线','流','流水']`)。
|
|
1673
|
+
链回原词之后才是「流水线」这种能直接翻译成抓取矩阵的词。
|
|
1674
|
+
"""
|
|
1675
|
+
gs = sorted(set(g for g in grams if len(g) == 2))
|
|
1676
|
+
if not gs:
|
|
1677
|
+
return []
|
|
1678
|
+
succ: Dict[str, List[str]] = {}
|
|
1679
|
+
for g in gs:
|
|
1680
|
+
succ.setdefault(g[0], []).append(g)
|
|
1681
|
+
has_pred = {g[1] for g in gs}
|
|
1682
|
+
out: List[str] = []
|
|
1683
|
+
for h in [g for g in gs if g[0] not in has_pred]:
|
|
1684
|
+
best = h
|
|
1685
|
+
stack = [(h, h)]
|
|
1686
|
+
while stack:
|
|
1687
|
+
cur, acc = stack.pop()
|
|
1688
|
+
if len(acc) > len(best):
|
|
1689
|
+
best = acc
|
|
1690
|
+
if len(acc) >= max_len:
|
|
1691
|
+
continue
|
|
1692
|
+
for nxt in succ.get(cur[-1], []):
|
|
1693
|
+
stack.append((nxt, acc + nxt[1:]))
|
|
1694
|
+
out.append(best)
|
|
1695
|
+
return out
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def _product_tokens(name: str, args: Dict[str, Any]) -> List[str]:
|
|
1699
|
+
"""从入参里抽出**产品词**(供 X-Beacon-Tokens)。
|
|
1700
|
+
|
|
1701
|
+
刻意不记录整句 query(§14.2):用户什么都可能输入,原样记等于落成明文台账。
|
|
1702
|
+
流程:分词 → 只留 `_usable_gram` 认可的(滤掉城市、企业通名、单字功能词)
|
|
1703
|
+
→ **只取长度 ≥ 2**(这一条同时干掉了单字碎片)→ 2-gram 链回原词。
|
|
1704
|
+
"""
|
|
1705
|
+
text = ""
|
|
1706
|
+
if name == "search_vendors":
|
|
1707
|
+
text = args.get("query") or ""
|
|
1708
|
+
elif name == "start_sourcing":
|
|
1709
|
+
text = args.get("demand_text") or ""
|
|
1710
|
+
if not text:
|
|
1711
|
+
return []
|
|
1712
|
+
try:
|
|
1713
|
+
toks = [t for t in _tokenize(str(text)) if _usable_gram(t)]
|
|
1714
|
+
except Exception:
|
|
1715
|
+
toks = [t for t in str(text).split() if t]
|
|
1716
|
+
|
|
1717
|
+
city = (args.get("city") or "").strip()
|
|
1718
|
+
cjk: List[str] = []
|
|
1719
|
+
latin: List[str] = []
|
|
1720
|
+
for t in toks:
|
|
1721
|
+
if not t or len(t) < 2 or t == city:
|
|
1722
|
+
continue
|
|
1723
|
+
(cjk if _is_cjk(t) else latin).append(t)
|
|
1724
|
+
|
|
1725
|
+
# 英文/数字取整词(不链);中文走 2-gram 链还原
|
|
1726
|
+
out: List[str] = []
|
|
1727
|
+
for t in sorted(set(latin), key=len, reverse=True):
|
|
1728
|
+
if t not in out:
|
|
1729
|
+
out.append(t)
|
|
1730
|
+
for t in _chain_grams(cjk):
|
|
1731
|
+
if t not in out:
|
|
1732
|
+
out.append(t)
|
|
1733
|
+
return out[:6]
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def _dispatch(name: str, args: Dict[str, Any]) -> Dict[str, Any]:
|
|
1737
|
+
"""tool 调用的唯一入口 —— 埋点就挂在这里(文档 §4)。
|
|
1738
|
+
|
|
1739
|
+
顺序有讲究:**先**把 tokens 放进上下文,再调用。因为 HTTP 请求发生在
|
|
1740
|
+
被调用函数内部,tokens 必须在请求发出前就位;而 hits 只能等结果出来后
|
|
1741
|
+
回填,供**后续**请求带上(一次检索的头几个请求因此没有 hits 列,属正常)。
|
|
1742
|
+
"""
|
|
1743
|
+
args = args or {}
|
|
1744
|
+
tok = _cur_set({"tool": name, "tokens": _product_tokens(name, args)})
|
|
1745
|
+
t0 = time.time()
|
|
1746
|
+
err: Optional[BaseException] = None
|
|
1747
|
+
r: Any = None
|
|
1748
|
+
try:
|
|
1749
|
+
r = _dispatch_inner(name, args)
|
|
1750
|
+
if isinstance(r, dict) and "total_matched" in r:
|
|
1751
|
+
cur = _cur_get() or {}
|
|
1752
|
+
cur["hits"] = r.get("total_matched")
|
|
1753
|
+
_cur_set(cur)
|
|
1754
|
+
return r
|
|
1755
|
+
except BaseException as e: # noqa: BLE001 —— 埋点不能吞掉协议层行为
|
|
1756
|
+
err = e
|
|
1757
|
+
raise
|
|
1758
|
+
finally:
|
|
1759
|
+
cur = _cur_get() or {}
|
|
1760
|
+
# _dispatch_inner 把业务异常包成了 {"error": ...},那也算失败
|
|
1761
|
+
ok = err is None and not (isinstance(r, dict) and "error" in r)
|
|
1762
|
+
_emit_usage(name, int((time.time() - t0) * 1000), ok,
|
|
1763
|
+
{"tokens": cur.get("tokens"), "hits": cur.get("hits")})
|
|
1764
|
+
try:
|
|
1765
|
+
if tok is not None and _CUR is not None:
|
|
1766
|
+
_CUR.reset(tok)
|
|
1767
|
+
except Exception:
|
|
1768
|
+
pass
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
def _dispatch_inner(name: str, args: Dict[str, Any]) -> Dict[str, Any]:
|
|
1772
|
+
try:
|
|
1773
|
+
if name == "search_vendors":
|
|
1774
|
+
return search_vendors(
|
|
1775
|
+
query=args.get("query", ""),
|
|
1776
|
+
city=args.get("city", ""),
|
|
1777
|
+
gb=args.get("gb", ""),
|
|
1778
|
+
limit=args.get("limit", 20),
|
|
1779
|
+
offset=args.get("offset", 0),
|
|
1780
|
+
)
|
|
1781
|
+
if name == "get_vendor":
|
|
1782
|
+
return get_vendor(args.get("id", ""), args.get("gb", ""))
|
|
1783
|
+
if name == "get_capability_card":
|
|
1784
|
+
return get_capability_card(args.get("id", ""))
|
|
1785
|
+
if name == "start_sourcing":
|
|
1786
|
+
return start_sourcing(args.get("demand_text", ""), args.get("audience_id", "domestic_downstream"))
|
|
1787
|
+
if name == "answer_sourcing":
|
|
1788
|
+
return answer_sourcing(args.get("session_id", ""), args.get("answers"))
|
|
1789
|
+
if name == "refine_sourcing":
|
|
1790
|
+
return refine_sourcing(args.get("session_id", ""), args.get("action", ""), args.get("value"))
|
|
1791
|
+
except Exception as e: # 任何异常都包成文本,避免协议崩
|
|
1792
|
+
return {"error": f"{name} 执行异常: {e}"}
|
|
1793
|
+
return {"error": f"未知 tool: {name}"}
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
def _send(obj: Dict[str, Any]) -> None:
|
|
1797
|
+
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
|
|
1798
|
+
sys.stdout.flush()
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def _log(msg: str) -> None:
|
|
1802
|
+
sys.stderr.write("[beacon-mcp] " + msg + "\n")
|
|
1803
|
+
sys.stderr.flush()
|
|
1804
|
+
|
|
1805
|
+
|
|
1806
|
+
def main() -> None:
|
|
1807
|
+
_log(f"start; source={BEACON_SOURCE} repo={REPO or '(http)'}")
|
|
1808
|
+
for raw in sys.stdin:
|
|
1809
|
+
raw = raw.strip()
|
|
1810
|
+
if not raw:
|
|
1811
|
+
continue
|
|
1812
|
+
try:
|
|
1813
|
+
msg = json.loads(raw)
|
|
1814
|
+
except Exception:
|
|
1815
|
+
continue
|
|
1816
|
+
method = msg.get("method")
|
|
1817
|
+
mid = msg.get("id")
|
|
1818
|
+
if method == "initialize":
|
|
1819
|
+
_send({
|
|
1820
|
+
"jsonrpc": "2.0", "id": mid,
|
|
1821
|
+
"result": {
|
|
1822
|
+
"protocolVersion": "2024-11-05",
|
|
1823
|
+
"capabilities": {"tools": {}},
|
|
1824
|
+
"serverInfo": {"name": "beacon-mfg-readonly", "version": "1.3.0"},
|
|
1825
|
+
},
|
|
1826
|
+
})
|
|
1827
|
+
elif method == "notifications/initialized":
|
|
1828
|
+
continue # 通知无需回复
|
|
1829
|
+
elif method == "ping":
|
|
1830
|
+
_send({"jsonrpc": "2.0", "id": mid, "result": {}})
|
|
1831
|
+
elif method == "tools/list":
|
|
1832
|
+
_send({"jsonrpc": "2.0", "id": mid, "result": {"tools": TOOLS}})
|
|
1833
|
+
elif method == "tools/call":
|
|
1834
|
+
params = msg.get("params", {})
|
|
1835
|
+
name = params.get("name", "")
|
|
1836
|
+
res = _dispatch(name, params.get("arguments", {}))
|
|
1837
|
+
_send({
|
|
1838
|
+
"jsonrpc": "2.0", "id": mid,
|
|
1839
|
+
"result": {
|
|
1840
|
+
"content": [{"type": "text", "text": json.dumps(res, ensure_ascii=False)}],
|
|
1841
|
+
"isError": "error" in res,
|
|
1842
|
+
},
|
|
1843
|
+
})
|
|
1844
|
+
else:
|
|
1845
|
+
# 未知方法:若有 id 则回空结果,通知则忽略
|
|
1846
|
+
if mid is not None:
|
|
1847
|
+
_send({"jsonrpc": "2.0", "id": mid, "result": {}})
|
|
1848
|
+
|
|
1849
|
+
|
|
1850
|
+
if __name__ == "__main__":
|
|
1851
|
+
main()
|