@furongjun1999/dsh-memory 0.9.10 → 0.9.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +181 -150
- package/lib/hooks.d.ts +16 -1
- package/lib/hooks.js +53 -2
- package/lib/index.d.ts +3 -0
- package/lib/index.js +9 -1
- package/lib/tools.d.ts +8 -0
- package/lib/tools.js +32 -5
- package/md_cg/config_registry_bulk.py +4 -0
- package/md_cg/config_validate.py +461 -457
- package/md_cg/mcp_server.py +133 -11
- package/md_cg/mdcg.py +14 -3
- package/md_cg/mdcos.py +97 -1
- package/md_cg/refindex.py +11 -4
- package/md_cg/test_c8_search_rrf_gates.py +511 -502
- package/md_cg/test_domain_granularity.py +68 -0
- package/md_cg/test_issue623_fullwidth_symmetry.py +66 -0
- package/md_cg/test_p28_refcheck.py +7 -7
- package/md_cg/test_retr_gates_prodpath.py +11 -1
- package/md_cg/test_search_async.py +84 -0
- package/md_cg/test_search_prefilter.py +85 -0
- package/md_cg/tokens.py +5 -1
- package/package.json +113 -113
- package/scripts/workspace_index.py +2 -0
- package/src/hooks.ts +60 -3
- package/src/index.ts +12 -1
- package/src/tools.ts +37 -6
package/md_cg/mcp_server.py
CHANGED
|
@@ -262,11 +262,12 @@ TOOLS = [
|
|
|
262
262
|
"inputSchema": _s("", query=_p("string", "查询", True), k=_p("integer", "条数"),
|
|
263
263
|
layer=_p("string", "限定层"), context=_p("object", "情境"),
|
|
264
264
|
roles=_p("array", "限定角色"), include_work=_p("boolean", "含工作角色"),
|
|
265
|
-
session=_p("string", "会话归属过滤(frontmatter.session
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
265
|
+
session=_p("string", "会话归属过滤(frontmatter.session;缺省不过滤)"),
|
|
266
|
+
validity=_p("boolean", "时效过滤(显式启用,缺省不过滤):仅排除已过期(valid_until 已过)节点,未生效(valid_from 未到)保留"),
|
|
267
|
+
# 异步检索(2026-10-21):怕 MCP 超时的客户端用它——提交后立刻拿
|
|
268
|
+
# job_id,隔若干秒带 job_id 再来取结果(后台线程跑同一条检索)。
|
|
269
|
+
defer_s=_p("number", "异步检索:大于 0 时立刻返回工单(job_id),检索在后台跑"),
|
|
270
|
+
job_id=_p("string", "取异步检索结果:传上次返回的 job_id")),
|
|
270
271
|
},
|
|
271
272
|
{
|
|
272
273
|
"name": "mdcg_get",
|
|
@@ -3684,6 +3685,84 @@ def _stg_call(cg, a):
|
|
|
3684
3685
|
# --------------------------------------------------------------------------
|
|
3685
3686
|
|
|
3686
3687
|
# 生效条件:a=args or {},unit=(a.get("as_unit") or "").strip(),sess=_declared_session(a.get("session"));cg.principal 为 None 或 unit 与 sess 皆为空时直接返回 _dispatch(cg,name,a);否则先 narrowed_principal(cg.principal, unit)(TokenError 时返回 {"ok":False,"error":f"as_unit 非法:{e}"}),sess 非空时再以 copy 出的 Principal 覆盖 session,暂存 cg.principal/cg.session、置为作用域值、try 返回 _dispatch、finally 还原。
|
|
3688
|
+
# 生效条件:cg 为已构造的安全客户端、args 为 mdcg_search 的原始入参、defer_s>0 时——
|
|
3689
|
+
# 立即返回 {job_id,status=pending,hint},并在守护线程里以**同一份入参(去掉 defer_s/job_id)**
|
|
3690
|
+
# 调 call_tool(cg,"mdcg_search",...);失败也落进工单(status=error),不抛到调用方。
|
|
3691
|
+
_SEARCH_JOBS = {}
|
|
3692
|
+
_SEARCH_JOBS_LOCK = __import__("threading").Lock()
|
|
3693
|
+
_SEARCH_JOB_TTL = 600.0 # 秒:工单保留时长(过期即丢,防常驻进程无限涨)
|
|
3694
|
+
_SEARCH_JOB_MAX = 64 # 条:同时保留的工单上限
|
|
3695
|
+
|
|
3696
|
+
|
|
3697
|
+
def _search_job_gc(now=None, reserve=0):
|
|
3698
|
+
"""清过期/超量工单(调用方持锁)。TTL 与条数双闸,防常驻进程内存无界。"""
|
|
3699
|
+
import time as _t
|
|
3700
|
+
now = _t.time() if now is None else now
|
|
3701
|
+
for _k in [k for k, v in _SEARCH_JOBS.items()
|
|
3702
|
+
if now - float(v.get("created") or 0) > _SEARCH_JOB_TTL]:
|
|
3703
|
+
_SEARCH_JOBS.pop(_k, None)
|
|
3704
|
+
_keep = max(0, _SEARCH_JOB_MAX - int(reserve or 0))
|
|
3705
|
+
if len(_SEARCH_JOBS) > _keep:
|
|
3706
|
+
for _k, _v in sorted(_SEARCH_JOBS.items(),
|
|
3707
|
+
key=lambda kv: float(kv[1].get("created") or 0))[:len(_SEARCH_JOBS) - _keep]:
|
|
3708
|
+
_SEARCH_JOBS.pop(_k, None)
|
|
3709
|
+
|
|
3710
|
+
|
|
3711
|
+
def _search_job_submit(cg, args, defer_s):
|
|
3712
|
+
"""提交异步检索:返回工单,不等待检索完成。"""
|
|
3713
|
+
import random as _rnd
|
|
3714
|
+
import threading as _th
|
|
3715
|
+
import time as _t
|
|
3716
|
+
with _SEARCH_JOBS_LOCK:
|
|
3717
|
+
_search_job_gc(reserve=1) # 给自己预留一格,插入后不超过上限
|
|
3718
|
+
jid = "sjob_%d_%04d" % (int(_t.time() * 1000), _rnd.randint(0, 9999))
|
|
3719
|
+
rec = {"job_id": jid, "status": "pending", "created": _t.time(),
|
|
3720
|
+
"query": args.get("query"), "result": None, "error": None,
|
|
3721
|
+
"elapsed_s": None}
|
|
3722
|
+
_SEARCH_JOBS[jid] = rec
|
|
3723
|
+
payload = {k: v for k, v in args.items() if k not in ("defer_s", "job_id")}
|
|
3724
|
+
|
|
3725
|
+
def _run():
|
|
3726
|
+
_t0 = _t.time()
|
|
3727
|
+
try:
|
|
3728
|
+
out = call_tool(cg, "mdcg_search", payload)
|
|
3729
|
+
with _SEARCH_JOBS_LOCK:
|
|
3730
|
+
rec["status"] = "done"
|
|
3731
|
+
rec["result"] = out
|
|
3732
|
+
rec["elapsed_s"] = round(_t.time() - _t0, 3)
|
|
3733
|
+
except Exception as exc: # noqa: BLE001 —— 工单承接失败,不抛给调用方
|
|
3734
|
+
with _SEARCH_JOBS_LOCK:
|
|
3735
|
+
rec["status"] = "error"
|
|
3736
|
+
rec["error"] = "%s: %s" % (type(exc).__name__, exc)
|
|
3737
|
+
rec["elapsed_s"] = round(_t.time() - _t0, 3)
|
|
3738
|
+
|
|
3739
|
+
_th.Thread(target=_run, name="mdcg-search-job", daemon=True).start()
|
|
3740
|
+
return {"job_id": jid, "status": "pending",
|
|
3741
|
+
"poll_after_s": int(defer_s) if defer_s else 10,
|
|
3742
|
+
"hint": "检索在后台进行;请约 %d 秒后再用 job_id 调用一次 mdcg_search 取结果"
|
|
3743
|
+
% (int(defer_s) if defer_s else 10)}
|
|
3744
|
+
|
|
3745
|
+
|
|
3746
|
+
def _search_job_result(jid):
|
|
3747
|
+
"""取异步检索结果:pending 原样回报,done 带 result,error 带 error。"""
|
|
3748
|
+
with _SEARCH_JOBS_LOCK:
|
|
3749
|
+
_search_job_gc()
|
|
3750
|
+
rec = _SEARCH_JOBS.get(str(jid))
|
|
3751
|
+
if rec is None:
|
|
3752
|
+
return {"job_id": str(jid), "status": "unknown",
|
|
3753
|
+
"error": "未知或已过期的 job_id(工单保留 %d 秒)" % int(_SEARCH_JOB_TTL)}
|
|
3754
|
+
snap = dict(rec)
|
|
3755
|
+
out = {"job_id": snap["job_id"], "status": snap["status"],
|
|
3756
|
+
"elapsed_s": snap.get("elapsed_s")}
|
|
3757
|
+
if snap["status"] == "done":
|
|
3758
|
+
out["result"] = snap.get("result")
|
|
3759
|
+
elif snap["status"] == "error":
|
|
3760
|
+
out["error"] = snap.get("error")
|
|
3761
|
+
else:
|
|
3762
|
+
out["poll_after_s"] = 10
|
|
3763
|
+
return out
|
|
3764
|
+
|
|
3765
|
+
|
|
3687
3766
|
def call_tool(cg, name, args):
|
|
3688
3767
|
"""MCP tools/call 入口:按 `as_unit` 做**请求级身份收窄**(单进程多身份),
|
|
3689
3768
|
按 `session` 做**请求级会话归因**(单进程多会话)。
|
|
@@ -4224,6 +4303,18 @@ def _dispatch(cg, name, args):
|
|
|
4224
4303
|
return _pack
|
|
4225
4304
|
|
|
4226
4305
|
if name == "mdcg_search":
|
|
4306
|
+
# ---- 异步检索(2026-10-21):把「怕超时」变成「先拿工单,过几秒来取」----
|
|
4307
|
+
# 背景:真实库首次检索曾要 8.4 秒、握手 5 秒,客户端 hook 若把超时设得较短
|
|
4308
|
+
# 就会整单失败。异步面不改检索语义:后台线程跑的是**同一个** call_tool,
|
|
4309
|
+
# 只是把「等待」从客户端挪到工单上。默认关(不带 defer_s 即同步)。
|
|
4310
|
+
if a.get("job_id"):
|
|
4311
|
+
return _search_job_result(str(a.get("job_id")))
|
|
4312
|
+
try:
|
|
4313
|
+
_defer = float(a.get("defer_s") or 0)
|
|
4314
|
+
except (TypeError, ValueError):
|
|
4315
|
+
_defer = 0.0
|
|
4316
|
+
if _defer > 0:
|
|
4317
|
+
return _search_job_submit(cg, a, _defer)
|
|
4227
4318
|
from . import refindex
|
|
4228
4319
|
# ---- query 级热缓存(默认面接线,2026-10-21)----
|
|
4229
4320
|
# 热缓存的读写在 search_rrf 内,而默认检索面走 cg.search(本分支)——
|
|
@@ -4975,7 +5066,26 @@ def _serve_line(cg, msg) -> bool:
|
|
|
4975
5066
|
"serverInfo": {"name": SERVER_NAME,
|
|
4976
5067
|
"version": SERVER_VERSION}})
|
|
4977
5068
|
elif method in ("notifications/initialized", "initialized"):
|
|
4978
|
-
|
|
5069
|
+
# 2026-10-21 冷启动预热(MCP 读写超时的实测根因之一):
|
|
5070
|
+
# 本进程**首次**检索要读 26k 个候选文档(真实库实测 8.4 秒),
|
|
5071
|
+
# 而读过一次之后**任意**新查询只要 0.26 秒(readcache 已热)。
|
|
5072
|
+
# 故在握手完成后立刻起一个后台线程做一次"暖查询",把这次一次性成本
|
|
5073
|
+
# 挪到客户端等 initialize 之后的空档里;用户真正发起检索时通常已热。
|
|
5074
|
+
# 不改变任何检索语义(record=False、结果丢弃),失败静默。
|
|
5075
|
+
try:
|
|
5076
|
+
if os.environ.get("MDCG_WARM_SEARCH", "1") != "0" and not globals().get("_warm_started"):
|
|
5077
|
+
globals()["_warm_started"] = True
|
|
5078
|
+
import threading as _th
|
|
5079
|
+
|
|
5080
|
+
def _warm():
|
|
5081
|
+
try:
|
|
5082
|
+
cg.search("灵枢", k=1, record=False)
|
|
5083
|
+
except Exception:
|
|
5084
|
+
pass
|
|
5085
|
+
|
|
5086
|
+
_th.Thread(target=_warm, name="mdcg-warm", daemon=True).start()
|
|
5087
|
+
except Exception:
|
|
5088
|
+
pass
|
|
4979
5089
|
elif method == "tools/list":
|
|
4980
5090
|
_reply(rid, {"tools": tools_for_surface()})
|
|
4981
5091
|
elif method == "tools/call":
|
|
@@ -5205,11 +5315,23 @@ def main():
|
|
|
5205
5315
|
# 真源自身损坏只告警留痕不自动改写(T9 边界:结构合法≠内容正确)。告警与
|
|
5206
5316
|
# 留痕由模块发声(diff 非空才输出,干净库零噪声);MDCG_RECONCILE=0 关闭
|
|
5207
5317
|
# (缺省开,开关真源 reconcile.enabled)。失败不阻塞启动,与③同风格。
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5318
|
+
# 2026-10-21 启动耗时:reconcile_state 实测 2.92 秒(真实库 31,688 节点),
|
|
5319
|
+
# 是 initialize 5.18 秒里最大的一块。它自己的注释与 reconcile 模块都写明
|
|
5320
|
+
# 「失败不阻塞启动、属增益」,故默认放到**后台线程**跑:握手不再等它。
|
|
5321
|
+
# MDCG_RECONCILE_BG=0 可退回同步(旧行为,便于对照与排障)。
|
|
5322
|
+
def _reconcile_bg(): # noqa: D401 —— 承接面:cg.reconcile_state()
|
|
5323
|
+
try:
|
|
5324
|
+
cg.reconcile_state()
|
|
5325
|
+
except Exception as _exc: # 对账故障不得影响服务可用性
|
|
5326
|
+
sys.stderr.write("[mdcg-mcp] 启动对账 v0 失败(不阻塞启动): %r\n"
|
|
5327
|
+
% (_exc,))
|
|
5328
|
+
|
|
5329
|
+
if os.environ.get("MDCG_RECONCILE_BG", "1") == "0":
|
|
5330
|
+
_reconcile_bg()
|
|
5331
|
+
else:
|
|
5332
|
+
import threading as _thr
|
|
5333
|
+
_thr.Thread(target=_reconcile_bg, name="mdcg-reconcile",
|
|
5334
|
+
daemon=True).start()
|
|
5213
5335
|
# 进程自报(诊断设施,fail-safe):把「md_cg 实际加载源 + render 契约代际」
|
|
5214
5336
|
# 落成 `<tempdir>/md_cg_servers/<pid>.json`,供 `scripts/mdcg_stale_servers.py`
|
|
5215
5337
|
# 机械判定**活进程代际**。根因(第4条取证):原先只能比「进程启动时间 vs
|
package/md_cg/mdcg.py
CHANGED
|
@@ -602,7 +602,11 @@ def normalize_en(text: str) -> str:
|
|
|
602
602
|
- 时态/复数归零(walked→walk, pets→pet)
|
|
603
603
|
"""
|
|
604
604
|
# 清标点(保留中文字符和空格和数字)
|
|
605
|
-
|
|
605
|
+
# issue #623(2026-10-22):本函数原先直接对**原文**做字符类清理,而 [^\u4e00-\u9fff a-zA-Z0-9] 会把全角
|
|
606
|
+
# 拉丁/数字(ABC123,U+FF01–FF5E)整段清成空格 ⇒ 查询词直接消失,全角与半角双向不可命中。
|
|
607
|
+
# 故先过 routing.nfkc_text(本仓 NFKC 唯一单点),再做字符类清理——一处改动同时覆盖
|
|
608
|
+
# _like 召回 / _score 打分 / reach 前筛三条腿(它们都由本函数产出 bigram)。
|
|
609
|
+
cleaned = re.sub(r'[^\u4e00-\u9fff a-zA-Z0-9]', ' ', routing.nfkc_text(text) or "")
|
|
606
610
|
# 生效条件:m.group(0) 小写后交模块级 _norm_word_mem 查词级 memo(纯函数精确键控:命中返回缓存值,未命中按 EN_STOPWORDS 剔除/strip_tense_en 同口径计算并回填)——命中值≡逐词现算值,返回串接进归一结果;
|
|
607
611
|
def _norm_word(m):
|
|
608
612
|
return _norm_word_mem(m.group(0).lower())
|
|
@@ -4293,8 +4297,13 @@ class MdCG:
|
|
|
4293
4297
|
return nodefile.positive_body(content)
|
|
4294
4298
|
|
|
4295
4299
|
def _like_body_lower(self, entry, content):
|
|
4296
|
-
"""Lowercase positive body; readcache reuses this pure projection.
|
|
4297
|
-
|
|
4300
|
+
"""Lowercase positive body; readcache reuses this pure projection.
|
|
4301
|
+
|
|
4302
|
+
issue #623(2026-10-22):这里也要过 routing.nfkc_text——_like 拿
|
|
4303
|
+
normalize_en 产出的词去 body 里找,而 body 若保留全角(ABC123),
|
|
4304
|
+
半角查询就永远找不到它。NFKC 单点同样只用 routing 那一份。
|
|
4305
|
+
"""
|
|
4306
|
+
return routing.nfkc_text(self._positive_body(entry, content)).lower()
|
|
4298
4307
|
|
|
4299
4308
|
def _doc_exact_text(self, entry, content):
|
|
4300
4309
|
"""Whitespace-free full content for the existing exact-hit tie break.
|
|
@@ -5511,6 +5520,8 @@ class MdCG:
|
|
|
5511
5520
|
# 分阶段审计仅在门控真的产生信息时落键(默认关闭 → meta 与改动前逐字节一致;契约 §5.3)
|
|
5512
5521
|
if stat.get("gates"):
|
|
5513
5522
|
meta["gates"] = stat["gates"]
|
|
5523
|
+
if stat.get("prefilter"):
|
|
5524
|
+
meta["prefilter"] = stat["prefilter"]
|
|
5514
5525
|
# P2-2(§5.2):拒绝域命中的**独立读数**(与资格 REJECT 分开计数)——
|
|
5515
5526
|
# 只在确有边界命中时落键(无边界命中的路径 meta 键集合与改动前一致)。
|
|
5516
5527
|
if stat.get("boundary"):
|
package/md_cg/mdcos.py
CHANGED
|
@@ -1294,7 +1294,56 @@ class MdCGOS(MdCG):
|
|
|
1294
1294
|
return out
|
|
1295
1295
|
|
|
1296
1296
|
# 生效条件:query strip 后为空即返回 ([], {"tier": None, "reason": "empty_query", "scanned": 0});pool_cfg 由 pooling.resolve(pooling.from_env(pools)) 解析;_candidates 为空即返回 no_candidates;其余与 MdCG.search 同构(T0–T3 阶梯 + 资格判定),并按 roles/include_work 默认剔除工具输出与命令类 role,view 真值时移交 role 维度裁决权给 roleviews.matches(见 _candidates),validity 真值时候选层剔除已过期节点;
|
|
1297
|
-
|
|
1297
|
+
# 生效条件:预筛(_prefilter_entries)生效且结果不足时——自动退回全量再查一次,
|
|
1298
|
+
# 保证「先试快路、不自信就退回」;命中充足时零额外成本。
|
|
1299
|
+
def search(self, *args, **kwargs):
|
|
1300
|
+
"""检索外层:分层预筛 + 置信回退(2026-10-22)。
|
|
1301
|
+
|
|
1302
|
+
背景:预筛把 26k 候选降到数百,代价是「标签不匹配就查不到」——实测已两次
|
|
1303
|
+
挡住自己的归档读回(写进去的节点按关键词搜不到,按 id 能读到)。
|
|
1304
|
+
这里加一道回退:预筛生效且(结果为空 或 条数不足 k)时,禁用预筛重查一次,
|
|
1305
|
+
并把 meta.prefilter_fallback=True 如实标出,便于事后核对。
|
|
1306
|
+
"""
|
|
1307
|
+
res, meta = self._search_impl(*args, **kwargs)
|
|
1308
|
+
# 置信判据:预筛生效时,看**查询词在结果里的覆盖率**。
|
|
1309
|
+
# 实测:标签不匹配的查询会返回 5 条「格式对但内容不对」的结果(覆盖率 2/3、2/4),
|
|
1310
|
+
# 而真正命中的查询覆盖 3/3。故 < 75% 视为不自信 ⇒ 禁用预筛重查一次。
|
|
1311
|
+
if isinstance(meta, dict) and meta.get("prefilter"):
|
|
1312
|
+
try:
|
|
1313
|
+
q = args[0] if args else kwargs.get("query", "")
|
|
1314
|
+
terms = [t for t in re.split(r"[\s,,、;;]+", str(q or "")) if len(t) >= 2]
|
|
1315
|
+
blob = []
|
|
1316
|
+
for _item in (res or ()):
|
|
1317
|
+
_n = _item[0] if isinstance(_item, tuple) else _item
|
|
1318
|
+
if not isinstance(_n, dict):
|
|
1319
|
+
continue
|
|
1320
|
+
blob.append(str(_n.get("title") or ""))
|
|
1321
|
+
_fm = _n.get("frontmatter") or {}
|
|
1322
|
+
if isinstance(_fm, dict):
|
|
1323
|
+
blob.append(str(_fm.get("tags") or ""))
|
|
1324
|
+
blob.append(str(_fm.get("title") or ""))
|
|
1325
|
+
blob.append(str(_n.get("tags") or ""))
|
|
1326
|
+
blob.append(str(_n.get("content") or _n.get("snippet") or "")[:600])
|
|
1327
|
+
_text = " ".join(blob).lower()
|
|
1328
|
+
_need = max(1, int(len(terms) * 0.75 + 0.999))
|
|
1329
|
+
_hit = sum(1 for t in terms if t.lower() in _text)
|
|
1330
|
+
_weak = bool(terms) and _hit < _need
|
|
1331
|
+
except Exception:
|
|
1332
|
+
_weak = False
|
|
1333
|
+
if _weak:
|
|
1334
|
+
self._pf_off = True
|
|
1335
|
+
try:
|
|
1336
|
+
res2, meta2 = self._search_impl(*args, **kwargs)
|
|
1337
|
+
finally:
|
|
1338
|
+
self._pf_off = False
|
|
1339
|
+
if isinstance(meta2, dict):
|
|
1340
|
+
meta2["prefilter_fallback"] = True
|
|
1341
|
+
meta2["prefilter_skipped"] = meta.get("prefilter")
|
|
1342
|
+
meta2["prefilter_coverage"] = "%d/%d" % (_hit, len(terms))
|
|
1343
|
+
return res2, meta2
|
|
1344
|
+
return res, meta
|
|
1345
|
+
|
|
1346
|
+
def _search_impl(self, query: str, layer: str = None, k: int = 20,
|
|
1298
1347
|
context=None, min_results: int = 1, record: bool = True,
|
|
1299
1348
|
include_neg: bool = True, judge: bool = True,
|
|
1300
1349
|
roles=None, include_work: bool = False, pools=None,
|
|
@@ -1518,6 +1567,17 @@ class MdCGOS(MdCG):
|
|
|
1518
1567
|
stat.pop(_k, None)
|
|
1519
1568
|
|
|
1520
1569
|
# 截断依据=相关度(同 MdCG.search:cap 值不变,改的是拿什么排序)
|
|
1570
|
+
# ---- 候选预筛(2026-10-21,使用者指出的分层检索)----
|
|
1571
|
+
# 现状:T2 无条件 _read_many(entries) 读全量正文(真实库 26k 文档 6.4 秒),
|
|
1572
|
+
# 这是「每进程首查 8 秒」的主因。既然**全局索引里已有元数据**(tags/桶),
|
|
1573
|
+
# 就先按索引元数据筛一轮:把词项与 tags/bucket_zh 命中的条目当候选,
|
|
1574
|
+
# 命中太少(默认 <50)再回退全量,并用 meta.prefilter 如实暴露是否筛过。
|
|
1575
|
+
# 只用**原始查询词**(>=2 字):扩展词里有单字(库/记),substring 匹配会命中几乎所有 tag,等于没筛。
|
|
1576
|
+
_raw_terms = [t for t in re.split(r"[\s,,、;;]+", q or "") if len(t) >= 2]
|
|
1577
|
+
_pf_entries, _pf_meta = self._prefilter_entries(entries, _raw_terms)
|
|
1578
|
+
if _pf_entries is not None:
|
|
1579
|
+
stat["prefilter"] = _pf_meta
|
|
1580
|
+
entries = _pf_entries
|
|
1521
1581
|
docs_all = self._read_many(entries, stat)
|
|
1522
1582
|
# 语义资格(MDCG_SEMANTIC=1):fm.semantic 节点无条件入池
|
|
1523
1583
|
hits = [d for d in docs_all
|
|
@@ -1555,6 +1615,42 @@ class MdCGOS(MdCG):
|
|
|
1555
1615
|
context, neg_coverage, big_domain, big_scores, pool_cfg)
|
|
1556
1616
|
|
|
1557
1617
|
# 生效条件:当 query、entries、stat 传入时,先以 terms/qb 从 entries 读取文档并保留 LIKE 命中或(semantic_on 且 frontmatter.semantic)的项;若 hits 为空则按 importance/created_at 降序取前 GLOBAL_CAP 作为兜底;随后对 hits 打分并按相关度与 importance 降序排序,超过 GLOBAL_CAP 时截断并更新 stat,否则返回全部 scored;
|
|
1618
|
+
# 生效条件:terms 非空、entries 为候选条目列表;词项与条目的 tags / bucket_zh 有交集且命中数
|
|
1619
|
+
# 达到 MDCG_PREFILTER_MIN(默认 50)时返回(筛后条目, {"prefilter": "tags", ...}),否则返回 (None, None)
|
|
1620
|
+
# 表示「不筛、走全量」——命中太少说明索引元数据不足以支撑该查询,不能拿它当召回面。
|
|
1621
|
+
def _prefilter_entries(self, entries, terms):
|
|
1622
|
+
"""按**索引元数据**预筛候选(分层检索的第二级:全局索引 → 正文)。
|
|
1623
|
+
|
|
1624
|
+
为什么用它:T2 无条件读全量正文是真实库 26k 文档 6.4 秒的主因;
|
|
1625
|
+
而全局索引里已带 tags/bucket_zh,先按词项筛一轮通常能把候选降到数百。
|
|
1626
|
+
边界(诚实声明):这是**召回取舍**——只在元数据命中足够多时启用,
|
|
1627
|
+
并把 prefilter 读数写进 meta,便于事后核对是否漏召回;显式 MDCG_PREFILTER=0 可关。
|
|
1628
|
+
"""
|
|
1629
|
+
try:
|
|
1630
|
+
if getattr(self, "_pf_off", False):
|
|
1631
|
+
return None, None
|
|
1632
|
+
if os.environ.get("MDCG_PREFILTER", "0") != "1":
|
|
1633
|
+
return None, None
|
|
1634
|
+
try:
|
|
1635
|
+
need = int(os.environ.get("MDCG_PREFILTER_MIN", "50"))
|
|
1636
|
+
except ValueError:
|
|
1637
|
+
need = 50
|
|
1638
|
+
ts = [str(t).lower() for t in (terms or []) if t]
|
|
1639
|
+
if not ts:
|
|
1640
|
+
return None, None
|
|
1641
|
+
out = []
|
|
1642
|
+
for e in entries or ():
|
|
1643
|
+
blob = " ".join(str(x) for x in (e.get("tags") or ())).lower()
|
|
1644
|
+
blob += " " + str(e.get("bucket_zh") or "").lower()
|
|
1645
|
+
if any(t in blob for t in ts):
|
|
1646
|
+
out.append(e)
|
|
1647
|
+
if len(out) < need:
|
|
1648
|
+
return None, None
|
|
1649
|
+
return out, {"prefilter": "tags", "matched": len(out),
|
|
1650
|
+
"pool": len(entries or ()), "min": need}
|
|
1651
|
+
except Exception:
|
|
1652
|
+
return None, None
|
|
1653
|
+
|
|
1558
1654
|
def _lexical(self, query, entries, stat):
|
|
1559
1655
|
"""词法路径:LIKE 预筛 + 二元组相似度(口径见 mdcg.SCORE_MODE)。
|
|
1560
1656
|
|
package/md_cg/refindex.py
CHANGED
|
@@ -423,11 +423,18 @@ def _domain_of(it: dict) -> str:
|
|
|
423
423
|
两者混成一个字段就会重演普查里的退化:实例名嵌进条件字段 → 3037 桶 /
|
|
424
424
|
3048 节点(99.9% 单例桶),路由等于失效。
|
|
425
425
|
|
|
426
|
-
|
|
427
|
-
|
|
426
|
+
取**整条相对路径**(2026-10-21 采纳外部用户 miyu 的修复一)。
|
|
427
|
+
|
|
428
|
+
为什么不再取首段:真实知识树常见形态是「少数顶层目录 + 其下大量子树」,
|
|
429
|
+
例如 default-kb/{shorinwiki,kb}。取首段时只有 2 个桶、99 篇文档全压在一个
|
|
430
|
+
桶里 ⇒ 巨桶占比 97.2%、期望扫描 94.6%,路由等于失效(外部实测:改整路径后
|
|
431
|
+
桶数 2→60、最大桶 97.2%→5.8%、期望扫描 94.6%→2.7%)。
|
|
432
|
+
|
|
433
|
+
安全性:docindex.node_id 只哈希 path#heading_path,**不含本域键**,
|
|
434
|
+
故改域只换桶、节点 id 完全不变,不产生重复节点。
|
|
428
435
|
"""
|
|
429
436
|
path = it.get("path") or ""
|
|
430
|
-
return path
|
|
437
|
+
return path or "orphan"
|
|
431
438
|
|
|
432
439
|
|
|
433
440
|
|
|
@@ -1189,4 +1196,4 @@ def rebuild(cg, *, ledger: "Ledger" = None, only_roots=None, max_files: int = 50
|
|
|
1189
1196
|
out["errors"].append(f"{root} [{kind}]:{exc}")
|
|
1190
1197
|
if ledger is not None:
|
|
1191
1198
|
ledger.save()
|
|
1192
|
-
return out
|
|
1199
|
+
return out
|