@furongjun1999/dsh-memory 0.9.23 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/CHANGELOG.en.md +450 -404
  2. package/CHANGELOG.md +67 -0
  3. package/README.en.md +244 -240
  4. package/README.md +272 -262
  5. package/codebuddy/CODEBUDDY.md +1 -1
  6. package/docs/discipline/templates/full.md.tmpl +1 -1
  7. package/docs/discipline/templates/rules.mdc.tmpl +1 -1
  8. package/docs/en/mdcg/world-model-endpoint.md +1 -1
  9. package/docs/mdcg/cogmap_ops.md +16 -15
  10. package/lib/client.js +29 -3
  11. package/lib/hooks.d.ts +12 -0
  12. package/lib/hooks.js +43 -1
  13. package/md_cg/bucketindex.py +231 -0
  14. package/md_cg/bucketscope.py +339 -0
  15. package/md_cg/config_registry_bulk.py +19 -10
  16. package/md_cg/config_validate.py +14 -2
  17. package/md_cg/detrerank.py +137 -0
  18. package/md_cg/fsutil.py +135 -0
  19. package/md_cg/govern/__init__.py +23 -0
  20. package/md_cg/iface/__init__.py +46 -0
  21. package/md_cg/iface/_verify.py +609 -0
  22. package/md_cg/iface/contracts.py +337 -0
  23. package/md_cg/judgment_manifest.py +14 -0
  24. package/md_cg/kernel/__init__.py +27 -0
  25. package/md_cg/mcp_server.py +5991 -5761
  26. package/md_cg/mdcg.py +474 -16
  27. package/md_cg/mdcos.py +228 -11
  28. package/md_cg/metacognition.py +66 -13
  29. package/md_cg/mutate_extract_shipped.py +126 -0
  30. package/md_cg/mutate_ledger_e2e.py +100 -0
  31. package/md_cg/op_table.py +8 -0
  32. package/md_cg/probe_c_ledger_integrity.py +258 -0
  33. package/md_cg/probe_d_retirement_discipline.py +263 -0
  34. package/md_cg/review_cli.py +6 -1
  35. package/md_cg/run_tests.py +30 -1
  36. package/md_cg/scopeselect.py +214 -0
  37. package/md_cg/scrub.py +66 -9
  38. package/md_cg/state_extract.py +149 -0
  39. package/md_cg/state_slots.py +58 -10
  40. package/md_cg/structboost.py +179 -0
  41. package/md_cg/test_alternatives_historical.py +162 -0
  42. package/md_cg/test_b2_sensitivity_landing.py +226 -223
  43. package/md_cg/test_b3_merge_keeps_content.py +608 -583
  44. package/md_cg/test_bucket_cards.py +85 -0
  45. package/md_cg/test_bucket_day_order.py +74 -0
  46. package/md_cg/test_bucket_quota.py +79 -0
  47. package/md_cg/test_bucket_recency_guards.py +75 -0
  48. package/md_cg/test_bucket_three_causes.py +83 -0
  49. package/md_cg/test_bucketindex.py +130 -0
  50. package/md_cg/test_bucketscope_early.py +71 -0
  51. package/md_cg/test_cond_term_fold_gate.py +1210 -0
  52. package/md_cg/test_cond_term_neg_scope.py +147 -0
  53. package/md_cg/test_detrerank.py +157 -0
  54. package/md_cg/test_gate_four_kpi.py +136 -0
  55. package/md_cg/test_generation_guard.py +14 -2
  56. package/md_cg/test_issue192_move_layer_indices_guard.py +48 -0
  57. package/md_cg/test_issue39_utf8_stdio.py +572 -572
  58. package/md_cg/test_issue413_preimage_failclosed.py +147 -9
  59. package/md_cg/test_issue768_overwrite_time_window_anchor.py +135 -0
  60. package/md_cg/test_issue789_log_plaintext_probe.py +100 -0
  61. package/md_cg/test_issue790_verb_negation.py +77 -0
  62. package/md_cg/test_issue810_root_guard.py +71 -0
  63. package/md_cg/test_issue812_info_index_stale.py +78 -0
  64. package/md_cg/test_issue827_weaken_dedup.py +361 -0
  65. package/md_cg/test_issue829_calibration_positive_gate.py +473 -0
  66. package/md_cg/test_kernel_iface_contracts.py +18 -0
  67. package/md_cg/test_md_cg_no_hive_dep.py +238 -17
  68. package/md_cg/test_probe_cd_shipped.py +385 -0
  69. package/md_cg/test_recent_tail.py +93 -0
  70. package/md_cg/test_run_tests_entry_gates.py +110 -0
  71. package/md_cg/test_scopeselect.py +70 -0
  72. package/md_cg/test_scopeselect_clock.py +70 -0
  73. package/md_cg/test_session_attribution_consistency.py +835 -813
  74. package/md_cg/test_state_extract_shipped.py +162 -0
  75. package/md_cg/test_state_ledger_chain.py +207 -0
  76. package/md_cg/test_state_ledger_e2e.py +145 -0
  77. package/md_cg/test_structboost.py +102 -0
  78. package/md_cg/test_timeboost.py +83 -0
  79. package/md_cg/test_timeboost_relativedate.py +71 -0
  80. package/md_cg/test_topicagg.py +129 -0
  81. package/md_cg/timeboost.py +223 -0
  82. package/md_cg/topicagg.py +142 -0
  83. package/md_cg/whitebox_kb/wisdom/code_compose.py +33 -5
  84. package/md_cg/whitebox_kb/wisdom/compose_engine.py +5 -1
  85. package/md_cg/whitebox_kb/wisdom/role_compose.py +271 -268
  86. package/package.json +137 -137
  87. package/src/hooks.ts +43 -1
  88. package/ui/minimal/payload/build-ui.mjs +41 -2
  89. package/ui/minimal/payload/lib/client.js +29 -3
  90. package/ui/minimal/payload/web/app.js +3 -1
  91. package/ui/minimal/payload/web/client.js +30 -3
  92. package/web/app.js +3 -1
  93. package/web/client.js +30 -3
  94. package/zcode/AGENTS.md +1 -1
@@ -0,0 +1,231 @@
1
+ # -*- coding: utf-8 -*-
2
+ """日桶 / 会话桶索引(bucketindex)——把「时间」与「会话身份」真正用起来。
3
+
4
+ 实测现状(主库 34,218 节点,2026-11-09):
5
+ · `created_at` 覆盖 **100%**,按天聚集(09-29 有 2733 条、11-08 有 2516 条);
6
+ · `session` 覆盖 **99.4%**,但其中 **48% 是 `unattributed`**(无法归属);
7
+ · `bucket` 字段**不是**日桶/会话桶,而是**条件桶**(`cond_<topic>_<hash>`)——别混用。
8
+
9
+ 所以两件事必须显式建出来:
10
+ · **日桶** `day:YYYY-MM-DD`:由 `created_at` 确定性派生(不新增写入代价);
11
+ · **会话桶** `session:<sid>`:由 `frontmatter.session` 派生,`unattributed` 单独成桶不与他人合并。
12
+
13
+ 索引文件(落库根目录,按需重建):
14
+ · `_days.json` {day: [node_id, ...]}
15
+ · `_sessions.json` {session: [node_id, ...]}
16
+
17
+ 为什么用索引:查询期扫 34k 节点按日期过滤实测 **2–17 秒**(见 timeboost.scoped_recall 注释),
18
+ 索引后是字典查找 + O(命中数) 取节点。
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import os
24
+ import time
25
+
26
+ from .bm25 import tokenize as _bm25_tokenize
27
+
28
+ DAYS_FILE = "_days.json"
29
+ SESSIONS_FILE = "_sessions.json"
30
+ TERMS_FILE = "_bucketterms.json"
31
+ SKEL_FILE = "_bucketskel.json"
32
+ UNATTRIBUTED = "unattributed"
33
+ #: 桶骨架:每节点/每桶保留的字符数(供子串共现,bench 口径)
34
+ SKEL_CHARS_PER_NODE = 600
35
+ SKEL_CHARS_PER_BUCKET = 20000
36
+
37
+
38
+ def day_of(ts):
39
+ """epoch 秒 → 本地日期字符串;不可解析返回 None。"""
40
+ try:
41
+ return time.strftime("%Y-%m-%d", time.localtime(float(ts)))
42
+ except (TypeError, ValueError, OSError):
43
+ return None
44
+
45
+
46
+ def _fm(rec):
47
+ if not isinstance(rec, dict):
48
+ return {}
49
+ fm = rec.get("frontmatter")
50
+ return fm if isinstance(fm, dict) and fm else rec
51
+
52
+
53
+ def build(store, write=True, with_content=False, content_limit=400):
54
+ """从索引面重建两个桶。返回 {days, sessions, nodes, unattributed}。"""
55
+ idx = getattr(store, "index", {}) or {}
56
+ content_terms = 0
57
+ nodes = idx.get("nodes") or {}
58
+ days = {}
59
+ sessions = {}
60
+ unattributed = 0
61
+ for nid, rec in nodes.items():
62
+ fm = _fm(rec)
63
+ d = day_of(fm.get("created_at"))
64
+ if d:
65
+ days.setdefault(d, []).append(nid)
66
+ sid = str(fm.get("session") or "") or UNATTRIBUTED
67
+ if sid == UNATTRIBUTED:
68
+ unattributed += 1
69
+ sessions.setdefault(sid, []).append(nid)
70
+ # 桶词表:{bucket_key: {term: 次数}} —— 供 scopeselect 打分(查询期只做字典运算)
71
+ terms = {}
72
+ skel = {}
73
+ for nid, rec in nodes.items():
74
+ fm = _fm(rec)
75
+ d0 = day_of(fm.get("created_at"))
76
+ sid0 = str(fm.get("session") or "") or UNATTRIBUTED
77
+ names = []
78
+ if d0:
79
+ names.append("day:" + d0)
80
+ if sid0 != UNATTRIBUTED:
81
+ names.append("session:" + sid0)
82
+ if not names:
83
+ continue
84
+ blob = []
85
+ for key in ("tags", "bucket_zh", "bucket", "role", "content_kind"):
86
+ val = fm.get(key)
87
+ if isinstance(val, (list, tuple)):
88
+ blob.extend(str(x) for x in val)
89
+ elif val:
90
+ blob.append(str(val))
91
+ # 与检索面**同口径**分词(bm25.tokenize:汉字单字 + 相邻二元)。
92
+ # 早期版本按标点切整句,查询里的短词(「商人」「蒙德城广场」)整词匹配不上,
93
+ # 生产共现实测只有 34%;换成 n-gram 粒度后才可能与 bench 的「子串语义」对齐。
94
+ text = " ".join(blob).lower()
95
+ if with_content:
96
+ node = store.get(nid)
97
+ body = str((node or {}).get("content") or "")[:content_limit * 8].lower()
98
+ if body:
99
+ text = text + " " + body
100
+ content_terms += 1
101
+ toks = list(_bm25_tokenize(text))
102
+ skel_part = (body if with_content else text)[:SKEL_CHARS_PER_NODE]
103
+ for bucket in names:
104
+ skel[bucket] = (skel.get(bucket, "") + " " + skel_part)[:SKEL_CHARS_PER_BUCKET]
105
+ bag = terms.setdefault(bucket, {})
106
+ for t in toks[:content_limit]:
107
+ bag[t] = bag.get(t, 0) + 1
108
+ out = {"content_enriched": content_terms, "days": len(days), "sessions": len(sessions),
109
+ "nodes": len(nodes), "unattributed": unattributed}
110
+ if write:
111
+ root = getattr(store, "root", None)
112
+ if root:
113
+ for name, data in ((DAYS_FILE, days), (SESSIONS_FILE, sessions),
114
+ (TERMS_FILE, terms), (SKEL_FILE, skel)):
115
+ path = os.path.join(root, name)
116
+ tmp = path + ".tmp"
117
+ with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
118
+ json.dump(data, fh, ensure_ascii=False)
119
+ os.replace(tmp, path)
120
+ return out
121
+
122
+
123
+ def _load(store, name):
124
+ root = getattr(store, "root", None)
125
+ if not root:
126
+ return {}
127
+ path = os.path.join(root, name)
128
+ if not os.path.exists(path):
129
+ return {}
130
+ try:
131
+ with open(path, encoding="utf-8") as fh:
132
+ return json.load(fh)
133
+ except (OSError, json.JSONDecodeError):
134
+ return {}
135
+
136
+
137
+ def day_ids(store, day):
138
+ return list(_load(store, DAYS_FILE).get(day) or ())
139
+
140
+
141
+ def session_ids(store, session):
142
+ return list(_load(store, SESSIONS_FILE).get(str(session)) or ())
143
+
144
+
145
+ def all_days(store):
146
+ """全部日桶键(升序未排序,调用方自行排序)。"""
147
+ return list(_load(store, DAYS_FILE).keys())
148
+
149
+
150
+ #: 自动重建:元数据文件(记录上次重建时间与规模)。
151
+ META_FILE = "_bucketindex_meta.json"
152
+ #: 缺省重建间隔(小时):每日收尾一次 ⇒ 20 小时���给抖动余量)。
153
+ DEFAULT_INTERVAL_H = 20.0
154
+ #: 自动重建开关的环境变量(缺省开;0=关)。
155
+ AUTO_ENV = "MDCG_BUCKET_AUTO"
156
+ INTERVAL_ENV = "MDCG_BUCKET_INTERVAL_H"
157
+
158
+ _RUNNING = {"on": False}
159
+
160
+
161
+ def _meta_path(store):
162
+ root = getattr(store, "root", None)
163
+ return os.path.join(root, META_FILE) if root else None
164
+
165
+
166
+ def last_build_ts(store):
167
+ p = _meta_path(store)
168
+ if not p or not os.path.exists(p):
169
+ return 0.0
170
+ try:
171
+ with open(p, encoding="utf-8") as fh:
172
+ return float(json.load(fh).get("ts") or 0.0)
173
+ except (OSError, json.JSONDecodeError, TypeError, ValueError):
174
+ return 0.0
175
+
176
+
177
+ def _write_meta(store, stats, ts):
178
+ p = _meta_path(store)
179
+ if not p:
180
+ return
181
+ try:
182
+ payload = dict(stats)
183
+ payload["ts"] = ts
184
+ tmp = p + ".tmp"
185
+ with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
186
+ json.dump(payload, fh, ensure_ascii=False)
187
+ os.replace(tmp, p)
188
+ except OSError:
189
+ pass
190
+
191
+
192
+ def maybe_auto_rebuild(store, interval_h=None, background=True):
193
+ """每日收尾式自动重建:距上次重建超过 interval 小时才重建。
194
+
195
+ 为什么这样设计:主库重建 3–5 秒,**不能**卡在写入或查询路径上,
196
+ 故(a)只在「陈旧」时触发,(b)缺省放后台线程跑;
197
+ 并有模块级 `_RUNNING` 防重入(同一进程内不会并发重建)。
198
+
199
+ 返回 {"rebuilt": bool, "reason": str, "age_h": float, ...},便于调用方审计。
200
+ """
201
+ if os.environ.get("MDCG_BUCKET_AUTO", "1") == "0":
202
+ return {"rebuilt": False, "reason": "disabled"}
203
+ if interval_h is not None: # 显式参数优先(0 = 强制重建)
204
+ interval = float(interval_h)
205
+ else:
206
+ try:
207
+ interval = float(os.environ.get("MDCG_BUCKET_INTERVAL_H", "20"))
208
+ except ValueError:
209
+ interval = DEFAULT_INTERVAL_H
210
+ age_h = (time.time() - last_build_ts(store)) / 3600.0
211
+ if last_build_ts(store) > 0 and age_h < interval:
212
+ return {"rebuilt": False, "reason": "fresh", "age_h": round(age_h, 2)}
213
+ if _RUNNING["on"]:
214
+ return {"rebuilt": False, "reason": "running", "age_h": round(age_h, 2)}
215
+
216
+ def _job():
217
+ _RUNNING["on"] = True
218
+ try:
219
+ stats = build(store, write=True)
220
+ _write_meta(store, stats, time.time())
221
+ except Exception: # noqa: BLE001 —— 后台失败不得影响主流程
222
+ pass
223
+ finally:
224
+ _RUNNING["on"] = False
225
+
226
+ if background:
227
+ import threading
228
+ threading.Thread(target=_job, daemon=True).start()
229
+ return {"rebuilt": True, "reason": "stale", "age_h": round(age_h, 2), "async": True}
230
+ _job()
231
+ return {"rebuilt": True, "reason": "stale", "age_h": round(age_h, 2), "async": False}
@@ -0,0 +1,339 @@
1
+ # -*- coding: utf-8 -*-
2
+ """桶作用域召回(bucketscope)——日桶 / 会话桶的**召回侧**落地。
3
+
4
+ 两类桶的定死口径(实测主库 34,218 节点,2026-11-09):
5
+ · **日桶** = `created_at` 的本地日期,`day:YYYY-MM-DD`。83 个桶;查询 5ms(索引)vs 2–17s(全扫)。
6
+ · **会话桶** = `frontmatter.session`,`session:<sid>`。631 个桶;**72% 是 `unattributed`**——
7
+ 这类节点的会话身份不可知,**只按日桶走**,绝不与他人合并(合并会把无关内容塞进同一上下文)。
8
+
9
+ 两条边界(都是实测教训):
10
+ 1. **日桶不能当「那天只写了这些」**:主库 2026-11-08 有 14,217 条(多日批量写入挤到同一天),
11
+ 所以日桶召回必须**再按查询词过滤**,不能整桶直接塞给模型。
12
+ 2. 会话桶同理:最大真实会话 3,026 条,整桶塞不进去。
13
+
14
+ 故本模块只做**受控召回**:桶内按查询词面筛选 → 取前 cap 条 → 以 0 分追加(不抢名次)。
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import os
19
+ import re
20
+
21
+ from . import bucketindex as _bi
22
+
23
+ #: 单次桶召回上限(与 recent_tail 同一量级,避免撑爆预算)。
24
+ CAP = 8
25
+
26
+
27
+ def _terms(q):
28
+ """查询切词:先按标点/空白切段,再对**中文长段补 2 元**。
29
+
30
+ 为什么要补二元:中文查询通常没有空格(「江北回声那回到了哪里」是**一个**段),
31
+ 整段做子串匹配几乎不可能命中任何节点 ⇒ 桶内 `_pick` 恒返回空 ⇒ 上层回落
32
+ 「最近 14 天」。补 2 元后「江北」「回声」都能匹配;排序仍按命中数降序,
33
+ 长段本身也算一个词,故整段命中者权重更高。
34
+ """
35
+ out, seen = [], set()
36
+ for seg in re.split(r"[\s,,、;;。!?::()()\[\]【】]+", q or ""):
37
+ seg = seg.lower()
38
+ if len(seg) < 2:
39
+ continue
40
+ if seg not in seen:
41
+ seen.add(seg)
42
+ out.append(seg)
43
+ if len(seg) >= 4 and re.search(r"[\u4e00-\u9fff]", seg):
44
+ for i in range(len(seg) - 1):
45
+ bg = seg[i:i + 2]
46
+ if bg not in seen:
47
+ seen.add(bg)
48
+ out.append(bg)
49
+ return out
50
+
51
+
52
+ #: 桶内扫描预算:桶可能上万条(主库 2026-11-08 有 14,217 条),逐条读盘会拖到秒级;
53
+ #: 实测 14k 桶全扫 2.9s、预算 800 条约 0.2s。故按**id 序**取前 MAX_SCAN 条(确定性、可复算),
54
+ #: 代价是桶内覆盖不全——这是「快」与「全」的显式取舍,不是静默降级。
55
+ MAX_SCAN = 800
56
+
57
+
58
+ def _pick(store, ids, terms, cap, validity=False, now=None, fallback_recency=False,
59
+ fill_rest=False):
60
+ """桶内按词面命中筛选(确定性):命中词数降序 → **时间降序** → id;取前 cap。
61
+
62
+ 【#842】排序键必须**在截断之前**含时间:早先只按 (-hits, nid) 排再截 cap,
63
+ 9 条同等命中 + cap=8 时被 id 序丢掉的那条可能正是最新的(后置的时间排序
64
+ 已经拿不回来)。现在把 created_at 提到截断前,截断即保留「同分里最新的」。
65
+
66
+ 【#843】validity=True 时必须在此处就过滤过期节点:桶补召回是**在候选检索
67
+ 之后**追加的,若不在源头过滤,调用方显式要求的「排除过期」会被补召回推翻。
68
+ """
69
+ scored = []
70
+ for nid in list(ids)[:MAX_SCAN]:
71
+ node = store.get(nid)
72
+ if not node:
73
+ continue
74
+ if validity and _is_expired(node, now):
75
+ continue
76
+ text = str(node.get("content") or "").lower()
77
+ hits = sum(1 for t in terms if t in text)
78
+ if hits:
79
+ scored.append((-hits, _recency_key(node), nid, node))
80
+ scored.sort()
81
+ out = [(node, 0.0, {"reason": "bucket_scope"}, [])
82
+ for _h, _r, _nid, node in scored[:cap]]
83
+ # 【外测 2026-11-11 卡点 1】命中抑制:只要有一条命中,早先就把**同桶其余节点全丢**,
84
+ # 于是「锚点在桶里但没命中 query 词」的条目直接消失(P16/d03:桶 [114,115,116],
85
+ # 只命中 114 ⇒ 116 被丢)。现改为:命中优先,**按时间序补足到 cap**。
86
+ if fill_rest and len(out) < cap:
87
+ chosen = {n.get("id") for n, _s, _q, _p in out}
88
+ rest = [n for n in (store.get(i) for i in list(ids)[:MAX_SCAN])
89
+ if n and n.get("id") not in chosen]
90
+ rest.sort(key=lambda n: (_recency_key(n), str(n.get("id"))))
91
+ for n in rest[:cap - len(out)]:
92
+ out.append((n, 0.0, {"reason": "bucket_scope",
93
+ "fallback": ("recency" if not scored else "fill")}, []))
94
+ return out
95
+
96
+
97
+ def day_scope(store, query, day, cap=CAP, validity=False, now=None):
98
+ """日桶召回:day 桶内按查询词筛选。索引缺失返回 None(调用方据此区分「无桶」与「空桶」)。"""
99
+ ids = _bi.day_ids(store, day)
100
+ if not ids:
101
+ return None
102
+ return _pick(store, ids, _terms(query), cap, validity=validity, now=now)
103
+
104
+
105
+ def session_scope(store, query, session, cap=CAP, validity=False, now=None):
106
+ """会话桶召回;`unattributed` 明确拒绝(会话身份不可知,不冒充)。"""
107
+ sid = str(session or "")
108
+ if not sid or sid == _bi.UNATTRIBUTED:
109
+ return None
110
+ ids = _bi.session_ids(store, sid)
111
+ if not ids:
112
+ return None
113
+ return _pick(store, ids, _terms(query), cap, validity=validity, now=now)
114
+
115
+
116
+ #: 通用默认:桶作用域**默认开启**(日期 + 会话是记忆的基本坐标,不是可选优化)。
117
+ DEFAULT_ON = True
118
+ #: 桶召回默认条数(会话桶 + 日桶合计)。
119
+ DEFAULT_CAP = 8
120
+
121
+ #: **会话桶独占条数**(2026-11-10 外测 v2.1):早先会话桶与日桶共享 cap=8,
122
+ #: 会话桶先取满 ⇒ 日桶一条都挤不进去(P17 的锚点日已被选中却 0 条进 pack)。
123
+ #: 现按会话 4 + 日 4 分配:日桶保证拿到 DEFAULT_CAP - SESSION_QUOTA 个名额。
124
+ SESSION_QUOTA = 4
125
+ #: 早段意图下的会话配额(把名额让给日桶)
126
+ SESSION_QUOTA_EARLY = 2
127
+ #: 日桶回溯天数上限(从最近一天往回数,够覆盖「前几天说的」)。
128
+ DAY_LOOKBACK = 14
129
+
130
+ #: 「早段意图」标记:问的是很早以前的事,回落「最近优先」必然错过。
131
+ #: 依据(2026-11-10 外测):10 个稳定失败全是这类——「江北回声那回」「我最早说…」「我最初说…」。
132
+ _EARLY_RE = re.compile(r"最早|最初|第一次|头一次|头回|初次|当初|以前|从前|很久以前|"
133
+ r"上次|上一回|上回|那回|那次|当年|那时候")
134
+
135
+ #: 早段意图下向前扫多少个日桶(成本上界:每桶 ≤ MAX_SCAN 条读盘)。
136
+ EARLY_SCAN_DAYS = 16
137
+
138
+ #: 早段意图的日桶集合:**最早几天 + 全时段均匀采样**。
139
+ #: 教训(2026-11-10 外测 v2.0):只扫「最早 12 天」会漏掉「库内最后一天」这种锚点
140
+ #: (P16 锚点 2026-05-03 恰是最后一天、P20 锚点在第 18 天),故改为全时段覆盖。
141
+ def _early_days(all_days, limit=EARLY_SCAN_DAYS, head=4, tail=4):
142
+ """两端交替有序池(兼容入口;新代码请用 `_early_candidates`,它把证据也纳入池生成)。"""
143
+ n = len(all_days)
144
+ if n <= limit:
145
+ return list(all_days)
146
+ ends = _two_ended(all_days, head=head, tail=tail)
147
+ chosen = set(ends)
148
+ rest = max(0, limit - len(ends))
149
+ step = max(1, n // max(1, rest)) if rest else n
150
+ uniform = [d for d in all_days[::step] if d not in chosen][:rest]
151
+ return ends + uniform
152
+
153
+
154
+ def _evidence_scores(store, terms):
155
+ """全库每个日桶的**证据分**(= 查询词在该桶词表里出现的个数),毫秒级不读正文。"""
156
+ table = _bi._load(store, _bi.TERMS_FILE) or {}
157
+ scores = {}
158
+ for bucket, bag in table.items():
159
+ key = str(bucket)
160
+ if not key.startswith("day:"):
161
+ continue
162
+ d = key[4:]
163
+ scores[d] = sum(1 for t in terms if t in bag)
164
+ return scores
165
+
166
+
167
+ def _order_by_evidence(store, days, terms):
168
+ """把给定候选日按证据强弱排序(同分保持入参顺序)。"""
169
+ scores = _evidence_scores(store, terms)
170
+ pos = {d: i for i, d in enumerate(days)}
171
+ return sorted(days, key=lambda d: (-scores.get(d, 0), pos[d]))
172
+
173
+
174
+ def _early_candidates(store, all_days, terms, limit=EARLY_SCAN_DAYS, head=4, tail=4):
175
+ """早段意图的**候选池生成**:证据优先,两端兜底,均匀补齐。
176
+
177
+ 三条并列病因(外测 2026-11-11 #858 判别):
178
+ ① 名额:锚点日在池内排第 5,而日名额只有 4;
179
+ ② 桶内闸:该日词表与 query 无交集 ⇒ `_pick` 不补足(该桶是两端交替选进来的);
180
+ ③ 池生成:均匀采样漏掉**有强证据**的那天(P20:04-30)⇒ 证据排序在池外无从发力。
181
+ 本函数治第 ③ 条:**先在全库按证据挑**,再补两端,最后均匀补齐。
182
+ """
183
+ n = len(all_days)
184
+ if n <= limit:
185
+ return list(all_days)
186
+ scores = _evidence_scores(store, terms)
187
+ ends = _two_ended(all_days, head=head, tail=tail)
188
+ end_rank = {d: i for i, d in enumerate(ends)}
189
+ # 证据优先;同分时按「两端交替」序(末端与首端先占位),其余按日期
190
+ ordered = sorted(all_days, key=lambda d: (-scores.get(d, 0),
191
+ end_rank.get(d, 10 ** 6), d))
192
+ out = ordered[:limit]
193
+ # 两端必须仍在池里(证据为 0 的末端日靠这条兜底)
194
+ for d in ends:
195
+ if d not in out:
196
+ out.append(d)
197
+ return out[:limit + len(ends)]
198
+
199
+
200
+ def _two_ended(all_days, head=4, tail=4):
201
+ """两端交替有序列表:[末1,首1,末2,首2,…,末k,首k]。"""
202
+ heads = list(all_days[:head])
203
+ tails = list(all_days[-tail:])
204
+ order = []
205
+ for i in range(max(len(heads), len(tails))):
206
+ if i < len(tails):
207
+ order.append(tails[-1 - i])
208
+ if i < len(heads):
209
+ order.append(heads[i])
210
+ return order
211
+
212
+
213
+ def _early_intent(query):
214
+ """query 是否表达了「早段意图」(要很早的记忆,而不是最近的)。"""
215
+ return bool(_EARLY_RE.search(str(query or "")))
216
+
217
+
218
+ def _is_expired(node, now=None):
219
+ """节点是否已过期(valid_until 早于 now)。
220
+
221
+ 判据与检索面一致:只认 frontmatter.valid_until(ISO 日期/时间),解析失败不算过期
222
+ (宁放过不错杀——过期是显式字段,不是推断)。
223
+ """
224
+ fm = node.get("frontmatter") or {}
225
+ until = fm.get("valid_until") or fm.get("validUntil")
226
+ if not until:
227
+ return False
228
+ t = now if now is not None else time.time()
229
+ raw = str(until).strip()
230
+ for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
231
+ try:
232
+ return time.mktime(time.strptime(raw[:19], fmt)) < float(t)
233
+ except ValueError:
234
+ continue
235
+ return False
236
+
237
+
238
+ def _recency_key(node):
239
+ """桶内排序键:created_at 降序(最近的在前)。"""
240
+ fm = node.get("frontmatter") or {}
241
+ try:
242
+ return -float(fm.get("created_at") or fm.get("valid_from") or 0)
243
+ except (TypeError, ValueError):
244
+ return 0.0
245
+
246
+
247
+ def scope_recall(store, query, session, now, cap=DEFAULT_CAP, validity=False):
248
+ """**默认口径**的桶作用域召回:会话桶优先 → 日桶按日期倒排 → 都按时间降序。
249
+
250
+ 使用者口径(2026-11-09):
251
+ 「日期、会话是默认就应该开启的;会话优先使用会话桶,然后按日期倒排,
252
+ 优先取自己最近的。条件才是筛选方式。」
253
+
254
+ 故本函数与「重排」无关——它只**补召回**:返回的每条 score=0.0,
255
+ 不抢正排名次,但一定进入候选(调用方可按预算追加)。
256
+ """
257
+ terms = _terms(query)
258
+ # 【外测 v2.0 建议 3】把解析出的**绝对日期**回吐给调用方(metadata.resolved_dates):
259
+ # 作答侧据此直接引用日期,不必自己把「上周五」换算成具体日子(B 类 7 题的根因之一)。
260
+ resolved_dates = []
261
+ try:
262
+ from . import timeboost as _tbd # noqa: PLC0415
263
+ resolved_dates = sorted(_tbd.query_dates(query, now))
264
+ except Exception: # noqa: BLE001 —— 解析失败不影响召回
265
+ resolved_dates = []
266
+ out = []
267
+ seen = set()
268
+ # ① 会话桶优先(自己的最近记录)
269
+ sid = str(session or "")
270
+ if sid and sid != _bi.UNATTRIBUTED:
271
+ ids = _bi.session_ids(store, sid)
272
+ # 【外测 #858 病因①】早段意图问的是「很久以前」,本会话桶的相关性天然更低:
273
+ # 会话配额从 4 降到 2,把 6 个名额留给日桶(P16 的锚点日在池内排第 5,4 个名额挤不进)。
274
+ _sess_quota = SESSION_QUOTA_EARLY if _early_intent(query) else SESSION_QUOTA
275
+ picked = _pick(store, ids, terms, min(cap, _sess_quota), validity=validity, now=now)
276
+ for node, score, qual, prov in picked:
277
+ qual = dict(qual)
278
+ qual["bucket"] = "session"
279
+ out.append((node, score, qual, prov))
280
+ seen.add(node.get("id"))
281
+ # ② 桶顺序:**范围选择优先**(状态追踪式共现推断),否则按日期倒排。
282
+ # MDCG_SCOPE_SELECT=1(缺省)→ 用 scopeselect 排序;=0 回落「最近优先」。
283
+ # 依据:bench 实测共现推断时间桶 94.20%(字面无关),词面消融仅 90.76%。
284
+ _ordered_days = None
285
+ _declared_days = False # 这些日桶是否来自「声明日期 / 共现推断」
286
+ _early_hit = False
287
+ try:
288
+ if os.environ.get("MDCG_SCOPE_SELECT", "1") != "0":
289
+ from . import scopeselect as _ss # noqa: PLC0415
290
+ _picked = _ss.pick_scopes(store, query, session=session,
291
+ limit=DAY_LOOKBACK, now=now)
292
+ _ordered_days = [key for kind, key, _s in _picked if kind == "day"]
293
+ _declared_days = bool(_ordered_days)
294
+ except Exception: # noqa: BLE001 —— 推断失败回落最近优先
295
+ _ordered_days = None
296
+ if not _ordered_days:
297
+ # 【2026-11-10 外测反馈】「最早 / 最初 / 上次 / 那回」这类**时间副词**查询,
298
+ # query 里没有可解析的绝对日期 ⇒ 选桶为空 ⇒ 若一路回落「最近 14 天」,
299
+ # 3–5 个月前的锚点必然被排除(实测 10/17 稳定失败属此类)。
300
+ # 故:早段意图优先扫**最早的日子**,其余仍回落最近优先。
301
+ _all_days = sorted(_bi.all_days(store))
302
+ _early_hit = bool(_early_intent(query) and _all_days)
303
+ if _early_hit:
304
+ _ordered_days = _early_candidates(store, _all_days, terms)
305
+ else:
306
+ _ordered_days = _all_days[::-1][:DAY_LOOKBACK]
307
+ # ② 日桶按上述顺序取
308
+ if len(out) < cap:
309
+ days = list(_ordered_days or [])
310
+ # 【外测 2026-11-11】跨日桶**轮转分配**:早先每天拿走「剩余全部名额」,
311
+ # 第一个桶条目够多就把 cap 吃光(实测 8 桶 cap=4 只覆盖到 2 桶,末日永不入选)。
312
+ # 现每轮每天最多取 per_day 条,保证「任何一天拿第 2 条之前,所有天都先拿到第 1 条」。
313
+ per_day = max(1, cap // max(1, len(days))) if days else cap
314
+ # 判据:桶是**按理由选进来**的(声明日期 / 早段意图 / 共现)⇒ 桶内补足;
315
+ # 若只是「最近 N 天」兜底,桶本身不携带查询证据,仍严格按词面筛选(避免塞无关条目)。
316
+ _fill = bool(_declared_days or _early_hit)
317
+ for day in days:
318
+ if len(out) >= cap:
319
+ break
320
+ picked = _pick(store, _bi.day_ids(store, day), terms,
321
+ min(per_day, cap - len(out)),
322
+ validity=validity, now=now, fallback_recency=True,
323
+ fill_rest=_fill)
324
+ for node, score, qual, prov in picked:
325
+ if node.get("id") in seen:
326
+ continue
327
+ qual = dict(qual)
328
+ qual["bucket"] = "day:%s" % day
329
+ if resolved_dates:
330
+ qual["resolved_dates"] = resolved_dates
331
+ out.append((node, score, qual, prov))
332
+ seen.add(node.get("id"))
333
+ # ③ 排序:**会话桶整体优先**,再按时间降序;日桶同样按时间降序。
334
+ # 使用者口径「会话优先使用会话桶,然后按日期倒排」——只按时间排会让日桶插到会话前面(实测发生过)。
335
+ def _key(it):
336
+ bucket = ((it[2] or {}).get("bucket") or "")
337
+ return (0 if bucket == "session" else 1, _recency_key(it[0]))
338
+ out.sort(key=_key)
339
+ return out[:cap]