@furongjun1999/dsh-memory 0.9.23 → 0.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +515 -404
- package/CHANGELOG.md +642 -449
- package/README.en.md +244 -240
- package/README.md +272 -262
- package/codebuddy/CODEBUDDY.md +1 -1
- package/docs/discipline/templates/full.md.tmpl +1 -1
- package/docs/discipline/templates/rules.mdc.tmpl +1 -1
- package/docs/en/mdcg/world-model-endpoint.md +1 -1
- package/docs/mdcg/cogmap_ops.md +19 -16
- package/lib/client.js +29 -3
- package/lib/hooks.d.ts +12 -0
- package/lib/hooks.js +43 -1
- package/lib/tools.d.ts +2 -2
- package/md_cg/bucketindex.py +231 -0
- package/md_cg/bucketscope.py +357 -0
- package/md_cg/cohesion.py +337 -0
- package/md_cg/config_registry_bulk.py +24 -10
- package/md_cg/config_validate.py +14 -2
- package/md_cg/constraint_audit.py +231 -0
- package/md_cg/constraint_watch.py +206 -0
- package/md_cg/context_audit.py +240 -0
- package/md_cg/context_plan.py +142 -0
- package/md_cg/detrerank.py +137 -0
- package/md_cg/distill.py +247 -0
- package/md_cg/dossier.py +151 -0
- package/md_cg/entities.py +133 -0
- package/md_cg/fsutil.py +135 -0
- package/md_cg/govern/__init__.py +23 -0
- package/md_cg/iface/__init__.py +46 -0
- package/md_cg/iface/_verify.py +609 -0
- package/md_cg/iface/contracts.py +337 -0
- package/md_cg/judgment_manifest.py +14 -0
- package/md_cg/kernel/__init__.py +27 -0
- package/md_cg/mcp_server.py +6044 -5761
- package/md_cg/mdcg.py +501 -16
- package/md_cg/mdcos.py +546 -12
- package/md_cg/metacognition.py +66 -13
- package/md_cg/mutate_extract_shipped.py +126 -0
- package/md_cg/mutate_ledger_e2e.py +100 -0
- package/md_cg/op_table.py +22 -0
- package/md_cg/probe_c_ledger_integrity.py +258 -0
- package/md_cg/probe_d_retirement_discipline.py +263 -0
- package/md_cg/profiles.py +298 -0
- package/md_cg/protocol.py +7 -1
- package/md_cg/reldate.py +142 -0
- package/md_cg/review_cli.py +6 -1
- package/md_cg/run_tests.py +30 -1
- package/md_cg/scopeselect.py +214 -0
- package/md_cg/scrub.py +66 -9
- package/md_cg/selfstmt.py +139 -0
- package/md_cg/state_extract.py +149 -0
- package/md_cg/state_slots.py +62 -10
- package/md_cg/state_view.py +181 -0
- package/md_cg/structboost.py +179 -0
- package/md_cg/sustain.py +28 -2
- package/md_cg/test_alternatives_historical.py +165 -0
- package/md_cg/test_b2_sensitivity_landing.py +226 -223
- package/md_cg/test_b3_merge_keeps_content.py +608 -583
- package/md_cg/test_bucket_cards.py +85 -0
- package/md_cg/test_bucket_day_order.py +74 -0
- package/md_cg/test_bucket_quota.py +79 -0
- package/md_cg/test_bucket_recency_guards.py +75 -0
- package/md_cg/test_bucket_three_causes.py +83 -0
- package/md_cg/test_bucket_time_rounds.py +87 -0
- package/md_cg/test_bucketindex.py +130 -0
- package/md_cg/test_bucketscope_early.py +71 -0
- package/md_cg/test_cohesion.py +107 -0
- package/md_cg/test_cond_term_fold_gate.py +1210 -0
- package/md_cg/test_cond_term_neg_scope.py +147 -0
- package/md_cg/test_constraint_audit.py +87 -0
- package/md_cg/test_constraint_watch.py +108 -0
- package/md_cg/test_context_audit.py +110 -0
- package/md_cg/test_context_plan.py +81 -0
- package/md_cg/test_detrerank.py +157 -0
- package/md_cg/test_gate_four_kpi.py +136 -0
- package/md_cg/test_generation_guard.py +14 -2
- package/md_cg/test_issue192_move_layer_indices_guard.py +48 -0
- package/md_cg/test_issue39_utf8_stdio.py +572 -572
- package/md_cg/test_issue413_preimage_failclosed.py +147 -9
- package/md_cg/test_issue768_overwrite_time_window_anchor.py +135 -0
- package/md_cg/test_issue789_log_plaintext_probe.py +100 -0
- package/md_cg/test_issue790_verb_negation.py +77 -0
- package/md_cg/test_issue810_root_guard.py +71 -0
- package/md_cg/test_issue812_info_index_stale.py +78 -0
- package/md_cg/test_issue827_weaken_dedup.py +361 -0
- package/md_cg/test_issue829_calibration_positive_gate.py +473 -0
- package/md_cg/test_kernel_iface_contracts.py +18 -0
- package/md_cg/test_md_cg_no_hive_dep.py +238 -17
- package/md_cg/test_probe_cd_shipped.py +385 -0
- package/md_cg/test_recent_tail.py +93 -0
- package/md_cg/test_run_tests_entry_gates.py +110 -0
- package/md_cg/test_scopeselect.py +70 -0
- package/md_cg/test_scopeselect_clock.py +70 -0
- package/md_cg/test_session_attribution_consistency.py +835 -813
- package/md_cg/test_state_extract_shipped.py +162 -0
- package/md_cg/test_state_ledger_chain.py +207 -0
- package/md_cg/test_state_ledger_e2e.py +145 -0
- package/md_cg/test_structboost.py +102 -0
- package/md_cg/test_timeboost.py +83 -0
- package/md_cg/test_timeboost_relativedate.py +71 -0
- package/md_cg/test_topicagg.py +129 -0
- package/md_cg/timeboost.py +223 -0
- package/md_cg/tokens.py +1 -1
- package/md_cg/topicagg.py +142 -0
- package/md_cg/version_anchor.py +84 -1
- package/md_cg/whitebox_kb/wisdom/code_compose.py +33 -5
- package/md_cg/whitebox_kb/wisdom/code_solidified.json +6195 -6195
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +5 -1
- package/md_cg/whitebox_kb/wisdom/role_compose.py +271 -268
- package/package.json +1 -1
- package/src/hooks.ts +43 -1
- package/ui/minimal/payload/build-ui.mjs +41 -2
- package/ui/minimal/payload/lib/client.js +29 -3
- package/ui/minimal/payload/web/app.js +3 -1
- package/ui/minimal/payload/web/client.js +30 -3
- package/web/app.js +3 -1
- package/web/client.js +30 -3
- package/zcode/AGENTS.md +1 -1
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""上下文规划器(context_plan)——把五步判据链做成可执行建议。
|
|
3
|
+
|
|
4
|
+
来源:上下文管理研究(三十轮)框架 v4。五步链是:
|
|
5
|
+
① 判级 按不可逆性 → L1 / L2 / L3 / 灰区
|
|
6
|
+
② 定形态 该级需要「并列」还是「连续」
|
|
7
|
+
③ 定声明 L2 起四维条件空间;L3 加逐条不可逆清单
|
|
8
|
+
④ 拼装 并列→检索拼装+指针;连续→原文优先
|
|
9
|
+
⑤ 核对 读数比对;不达标三选一,不得静默
|
|
10
|
+
|
|
11
|
+
**本模块是启发式,不是判级本身。**
|
|
12
|
+
判级需要判断「后果可否撤销」,那是语义判断;本模块只做两件机械的事:
|
|
13
|
+
* 扫强制词(不能/不得/必须…)→ 抬到至少 L2
|
|
14
|
+
* 扫不可逆词(覆盖/删除/永久…)→ 抬到 L3
|
|
15
|
+
判不出时**按 L2 兜底,不按 L1**——因为误判为低是不可逆损失(研究第六轮)。
|
|
16
|
+
|
|
17
|
+
诚实边界:
|
|
18
|
+
1. 词表是启发式,会漏(用户可用陈述句表达强制:「我们希望旧文件保留」);
|
|
19
|
+
2. 判级准确率未知——判级本身属盲区 13;
|
|
20
|
+
3. 输出是**建议**,不是裁决;人可覆盖,但覆盖须留理由(降级留痕,升级不用);
|
|
21
|
+
4. 不读上下文,只读传入的任务描述。
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
__all__ = ["plan", "LEVELS", "MANDATORY_RE", "IRREVERSIBLE_RE", "FACT_SEEKING_RE"]
|
|
27
|
+
|
|
28
|
+
LEVELS = ("L1", "L2", "L3", "灰区")
|
|
29
|
+
|
|
30
|
+
#: 强制语气词:出现即至少 L2
|
|
31
|
+
MANDATORY_RE = re.compile(r"(不能|不得|不许|禁止|必须|务必|一律|绝不|不可|一定要|严禁)")
|
|
32
|
+
|
|
33
|
+
#: 不可逆词:出现即 L3(后果不可撤销)
|
|
34
|
+
IRREVERSIBLE_RE = re.compile(
|
|
35
|
+
r"(覆盖|删除|销毁|永久|不可恢复|不可逆|彻底清除|重写历史|force|drop|truncate)")
|
|
36
|
+
|
|
37
|
+
#: 查事实型:倾向 L1(并列够用)
|
|
38
|
+
FACT_SEEKING_RE = re.compile(r"(是什么|是多少|哪一年|哪个版本|查一下|确认一下|是否等于)")
|
|
39
|
+
|
|
40
|
+
#: 各层级的策略建议
|
|
41
|
+
_PLAYBOOK = {
|
|
42
|
+
"L1": {
|
|
43
|
+
"form": "并列",
|
|
44
|
+
"declaration": "可不声明(或一句话记约束)",
|
|
45
|
+
"strategy": "滑窗/截断均可;检索拼装够用,不追求接缝",
|
|
46
|
+
"check": "不做(错了可重来)",
|
|
47
|
+
},
|
|
48
|
+
"L2": {
|
|
49
|
+
"form": "半连续",
|
|
50
|
+
"declaration": "四维条件空间(观测位置/工具/时间窗/存在约束)",
|
|
51
|
+
"strategy": "分层折叠+指针;原文下沉但保留可回取",
|
|
52
|
+
"check": "核对关键约束能否从当前上下文推出",
|
|
53
|
+
},
|
|
54
|
+
"L3": {
|
|
55
|
+
"form": "连续",
|
|
56
|
+
"declaration": "四维 + 逐条不可逆清单(清单必须外置,不得留在上下文里)",
|
|
57
|
+
"strategy": "原文优先;必须拼装时用指针,不硬拼",
|
|
58
|
+
"check": "全量核对;推不出必须三选一(回取/报警/显式降级),不得静默",
|
|
59
|
+
},
|
|
60
|
+
"灰区": {
|
|
61
|
+
"form": "按 L3(声明)/按实际(策略)",
|
|
62
|
+
"declaration": "按 L3 声明(清单外置),但策略可用便宜方式",
|
|
63
|
+
"strategy": "声明升 L3、策略按实际能力——灰区解耦(研究第七轮)",
|
|
64
|
+
"check": "核对清单;不可用则显式降级并留理由",
|
|
65
|
+
},
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def plan(task_text):
|
|
70
|
+
"""给定任务描述,返回规划建议。
|
|
71
|
+
|
|
72
|
+
返回 dict:
|
|
73
|
+
level L1 / L2 / L3 / 灰区
|
|
74
|
+
level_reason 判定依据(命中的词或「无命中,按 L2 兜底」)
|
|
75
|
+
mandatory 命中的强制词
|
|
76
|
+
irreversible 命中的不可逆词
|
|
77
|
+
fact_seeking 是否为查事实型
|
|
78
|
+
form / declaration / strategy / check 各步建议
|
|
79
|
+
caveat 必读的边界提示
|
|
80
|
+
"""
|
|
81
|
+
text = str(task_text or "")
|
|
82
|
+
mand = sorted(set(MANDATORY_RE.findall(text)))
|
|
83
|
+
irrev = sorted(set(IRREVERSIBLE_RE.findall(text)))
|
|
84
|
+
fact = bool(FACT_SEEKING_RE.search(text))
|
|
85
|
+
|
|
86
|
+
if irrev:
|
|
87
|
+
level = "L3"
|
|
88
|
+
reason = "命中不可逆词 %s → 后果不可撤销" % ("、".join(irrev))
|
|
89
|
+
elif mand:
|
|
90
|
+
level = "L2"
|
|
91
|
+
reason = "命中强制词 %s → 至少 L2(含不可逆则升 L3)" % ("、".join(mand))
|
|
92
|
+
elif fact:
|
|
93
|
+
level = "L1"
|
|
94
|
+
reason = "查事实型且无强制/不可逆词 → L1(并列够用)"
|
|
95
|
+
else:
|
|
96
|
+
level = "灰区"
|
|
97
|
+
reason = "无强制/不可逆命中,也未识别为查事实型 → 灰区,声明按 L3、策略按实际"
|
|
98
|
+
|
|
99
|
+
pb = _PLAYBOOK[level]
|
|
100
|
+
return {
|
|
101
|
+
"level": level,
|
|
102
|
+
"level_reason": reason,
|
|
103
|
+
"mandatory": mand,
|
|
104
|
+
"irreversible": irrev,
|
|
105
|
+
"fact_seeking": fact,
|
|
106
|
+
"form": pb["form"],
|
|
107
|
+
"declaration": pb["declaration"],
|
|
108
|
+
"strategy": pb["strategy"],
|
|
109
|
+
"check": pb["check"],
|
|
110
|
+
"caveat": "本输出是启发式建议非裁决;词表会漏(陈述句也可表达强制);"
|
|
111
|
+
"降级须留理由,升级不用。",
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def main(argv=None):
|
|
116
|
+
import json
|
|
117
|
+
import sys
|
|
118
|
+
try:
|
|
119
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
120
|
+
except Exception:
|
|
121
|
+
pass
|
|
122
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
123
|
+
if not argv:
|
|
124
|
+
print("用法:python -m md_cg.context_plan \"任务描述\" [--json]")
|
|
125
|
+
return 2
|
|
126
|
+
rep = plan(argv[0])
|
|
127
|
+
if "--json" in argv:
|
|
128
|
+
print(json.dumps(rep, ensure_ascii=False, indent=1))
|
|
129
|
+
else:
|
|
130
|
+
print("层级:%s" % rep["level"])
|
|
131
|
+
print("依据:%s" % rep["level_reason"])
|
|
132
|
+
print("形态:%s" % rep["form"])
|
|
133
|
+
print("声明:%s" % rep["declaration"])
|
|
134
|
+
print("策略:%s" % rep["strategy"])
|
|
135
|
+
print("核对:%s" % rep["check"])
|
|
136
|
+
print("")
|
|
137
|
+
print("注:%s" % rep["caveat"])
|
|
138
|
+
return 0
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
if __name__ == "__main__":
|
|
142
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""确定性重排(detrerank)——检索结论的**唯一裁决口**。
|
|
3
|
+
|
|
4
|
+
裁定(structural: mem_ruling_vector_ref_deterministic):
|
|
5
|
+
向量/语义相似度 **只提供参考**(候选扩展),结论必须由**词法·时间·空间·交互**
|
|
6
|
+
四类确定性信号重排,且**必须由确定性规则裁决**:固定常量 + 可数信号,
|
|
7
|
+
同输入同输出(可对拍、可回放)。
|
|
8
|
+
|
|
9
|
+
为什么这样切:相似度回答「像不像」,因果观察三元组(时间/空间/交互)回答
|
|
10
|
+
「是不是真的发生过」,词法回答「原文对不对得上」。三者性质不同,
|
|
11
|
+
用一个浮点分数混合会让「像」压过「真」——这正是 #753/#790 那类 fail-open 的同源病。
|
|
12
|
+
|
|
13
|
+
确定性约束(本模块的三条硬纪律):
|
|
14
|
+
1. **无模型分**:函数体不得出现 embedding / 相似度 / 模型调用;
|
|
15
|
+
2. **时钟显式**:`now` 必须由调用方传入(不内部取 time.time()),否则不可回放;
|
|
16
|
+
3. **全序**:分数相同按 id 字典序兜底,保证任意两次结果逐位相同。
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
#: 四信号权重(固定常量;改值等于改裁决口径,必须走评审 + 读数)。
|
|
23
|
+
WEIGHTS = {
|
|
24
|
+
"lexical": 4, # 词法最重:原文对得上才算证据
|
|
25
|
+
"interaction": 3, # 交互次之:真的发生过作用
|
|
26
|
+
"spatial": 2, # 空间可达:在同域/同链路里
|
|
27
|
+
"temporal": 1, # 时间只做轻微加权(新旧不代表对错)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
#: 时间衰减半衰期(天)——与 temporal 路同源口径,固定常量,不随环境变。
|
|
31
|
+
TEMPORAL_HALF_LIFE_D = 30.0
|
|
32
|
+
|
|
33
|
+
#: window 模式:只在融合前 WINDOW_SIZE 名内允许有界上浮(防弱条目顶上第一)。
|
|
34
|
+
WINDOW_SIZE = 8
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _terms(text) -> list:
|
|
38
|
+
return [t.lower() for t in re.split(r"[\s,,、;;。!?::()()\[\]【】]+", text or "")
|
|
39
|
+
if len(t) >= 2]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _snippet(node, limit: int = 1200) -> str:
|
|
43
|
+
parts = [str(node.get("content") or "")[:limit]]
|
|
44
|
+
fm = node.get("frontmatter") or {}
|
|
45
|
+
for key in ("tags", "title", "conditions"):
|
|
46
|
+
val = fm.get(key)
|
|
47
|
+
if val:
|
|
48
|
+
parts.append(str(val))
|
|
49
|
+
return " ".join(parts).lower()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def signals(node, query_terms, now, session=None) -> dict:
|
|
53
|
+
"""四信号取值(全部确定性可数)。"""
|
|
54
|
+
text = _snippet(node)
|
|
55
|
+
# 词法:query 词项在原文/标签里的命中数(整数)
|
|
56
|
+
lexical = sum(1 for t in query_terms if t in text)
|
|
57
|
+
fm = node.get("frontmatter") or {}
|
|
58
|
+
# 交互:出边数 + 入标记(edges 是「真的发生过作用」的痕迹)
|
|
59
|
+
edges = fm.get("edges")
|
|
60
|
+
out_edges = len(edges) if isinstance(edges, list) else 0
|
|
61
|
+
refs = fm.get("derived_from") or fm.get("alias_of") or []
|
|
62
|
+
refs_n = len(refs) if isinstance(refs, (list, tuple)) else (1 if refs else 0)
|
|
63
|
+
interaction = out_edges + refs_n
|
|
64
|
+
# 空间:同会话(同域)+ 同层(结构可达)——两个可数布尔
|
|
65
|
+
same_session = 1 if (session and str(fm.get("session") or "") == str(session)) else 0
|
|
66
|
+
layer = str(fm.get("layer") or "")
|
|
67
|
+
spatial = same_session + (1 if layer in ("knowledge", "structural") else 0)
|
|
68
|
+
# 时间:按半衰期衰减到 [0,1] 后取整到 3 位(确定性,无时钟读取)
|
|
69
|
+
try:
|
|
70
|
+
created = float(fm.get("created_at") or 0)
|
|
71
|
+
except (TypeError, ValueError):
|
|
72
|
+
created = 0.0
|
|
73
|
+
if created > 0 and now:
|
|
74
|
+
age_days = max(0.0, (float(now) - created) / 86400.0)
|
|
75
|
+
temporal = round(0.5 ** (age_days / TEMPORAL_HALF_LIFE_D), 3)
|
|
76
|
+
else:
|
|
77
|
+
temporal = 0.0
|
|
78
|
+
return {"lexical": lexical, "interaction": interaction,
|
|
79
|
+
"spatial": spatial, "temporal": temporal}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def score(sig: dict) -> float:
|
|
83
|
+
"""加权和(整数权重 × 确定性信号),保留 6 位。"""
|
|
84
|
+
total = 0.0
|
|
85
|
+
for key, weight in WEIGHTS.items():
|
|
86
|
+
total += float(weight) * float(sig.get(key) or 0)
|
|
87
|
+
return round(total, 6)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def rerank(results, query, now, session=None):
|
|
91
|
+
"""对 [(node, fused_score, qual, prov), ...] 做确定性重排。
|
|
92
|
+
|
|
93
|
+
返回新列表(不改入参):按 (det_score desc, id asc) 全序。
|
|
94
|
+
`now` 必须由调用方给(时钟显式,可回放)。
|
|
95
|
+
"""
|
|
96
|
+
terms = _terms(query)
|
|
97
|
+
scored = []
|
|
98
|
+
for item in results or ():
|
|
99
|
+
node = item[0]
|
|
100
|
+
sig = signals(node, terms, now, session=session)
|
|
101
|
+
scored.append((score(sig), str(node.get("id") or ""), sig, item))
|
|
102
|
+
scored.sort(key=lambda x: (-x[0], x[1]))
|
|
103
|
+
return [(item, det, sig) for det, _nid, sig, item in scored]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def rerank_v2(results, query, now, mode="tiebreak", session=None):
|
|
107
|
+
"""v2:**保留 RRF 为主序**,四信号只做有界修正。
|
|
108
|
+
|
|
109
|
+
bench 读数(hive-memory-bench · e2e-simple/蜂巢指南 38 题 · 块级词面覆盖):
|
|
110
|
+
v1(replace,按 det 总分重排):cov@1 **0.625 → 0.313**(g1 28 → 11)。
|
|
111
|
+
原因:词法计数奖励「长块/词多」,与「这段是否解答了问题」无关;
|
|
112
|
+
把 RRF 的候选质量丢掉,等于用一个更粗的信号覆盖更细的信号。
|
|
113
|
+
|
|
114
|
+
三种模式:
|
|
115
|
+
· `tiebreak`(**缺省**):fused 相同才看四信号——零风险,只在并列时用证据说话;
|
|
116
|
+
· `window`:融合前 WINDOW_SIZE 名内、词法 ≥ 2 的条目允许上浮(有界);
|
|
117
|
+
· `replace`:v1 行为,保留以便复现上述读数,**不建议**。
|
|
118
|
+
"""
|
|
119
|
+
terms = _terms(query)
|
|
120
|
+
scored = []
|
|
121
|
+
for idx, item in enumerate(results or ()):
|
|
122
|
+
node = item[0]
|
|
123
|
+
sig = signals(node, terms, now, session=session)
|
|
124
|
+
fused = float(item[1]) if len(item) > 1 and isinstance(item[1], (int, float)) else 0.0
|
|
125
|
+
scored.append({"idx": idx, "id": str(node.get("id") or ""), "sig": sig,
|
|
126
|
+
"det": score(sig), "fused": fused, "item": item})
|
|
127
|
+
if mode == "replace":
|
|
128
|
+
scored.sort(key=lambda x: (-x["det"], x["id"]))
|
|
129
|
+
elif mode == "window":
|
|
130
|
+
scored.sort(key=lambda x: (-x["fused"], x["idx"]))
|
|
131
|
+
head, tail = scored[:WINDOW_SIZE], scored[WINDOW_SIZE:]
|
|
132
|
+
head.sort(key=lambda x: (0 if x["sig"]["lexical"] >= 2 else 1,
|
|
133
|
+
-x["det"], -x["fused"], x["id"]))
|
|
134
|
+
scored = head + tail
|
|
135
|
+
else: # tiebreak(缺省,零风险)
|
|
136
|
+
scored.sort(key=lambda x: (-x["fused"], -x["det"], x["id"]))
|
|
137
|
+
return [(x["item"], x["det"], x["sig"]) for x in scored]
|
package/md_cg/distill.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""写入时信息整理(LLM 填表 + **原文复核**)——规则对自然对话天生失效后的换代方案。
|
|
3
|
+
|
|
4
|
+
判据(使用者 2026-11-12):
|
|
5
|
+
> 在记忆写入时用 LLM 做一个信息整理。因为规则对多变的自然对话天生失效。
|
|
6
|
+
> 我们用**表**做一个确定性内容让 LLM 填写。同时做**原文复核机制**,避免 LLM 的信息整理失误。
|
|
7
|
+
|
|
8
|
+
分工(本模块只做确定性那半):
|
|
9
|
+
· **表**(SCHEMA / table_text):字段名、类型、取值范围由我们定死 —— LLM 只填值,不发明字段;
|
|
10
|
+
· **复核**(verify):每个填进去的值都必须在**原文里找得到**(归一化子串匹配),
|
|
11
|
+
找不到 ⇒ 拒收(unverified),不进档案;
|
|
12
|
+
· **落地**(apply):只写通过复核的字段,并记 `ev_span`(原文里命中的片段)+ `ev_node` + `source`;
|
|
13
|
+
· **调用**:LLM 由宿主提供(本模块**不自带模型**),传入 filled 字典即可。
|
|
14
|
+
|
|
15
|
+
为什么复核必须写死在数据面而不是提示词里:
|
|
16
|
+
外测实证——纯规则会把「在包里」当城市;纯 LLM 则可能写出原文没有的值。
|
|
17
|
+
「值必须在原文出现」是**可机械判定**的,不依赖模型自觉。
|
|
18
|
+
|
|
19
|
+
不适用条件:
|
|
20
|
+
· 同义改写(原文「二十四」、LLM 填 24)会被判 unverified —— 这是**有意的保守**,
|
|
21
|
+
宁可漏收也不放行(要放行请用别名/规范化表,而不是放松复核);
|
|
22
|
+
· 数值型字段(年龄)按字符串匹配 + 数值等值两种口径都试,仍失败即拒收。
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import re
|
|
27
|
+
|
|
28
|
+
#: 表的字段定义(**确定性**:字段名/类型/是否必填/提示语都由这里定)。
|
|
29
|
+
SCHEMA = (
|
|
30
|
+
("姓名", "str", False, "自称或全名,2–4 个汉字"),
|
|
31
|
+
("年龄", "int", False, "数字,语料常见无「岁」字"),
|
|
32
|
+
("城市", "str", False, "必须是**地名**(区/市/县/路/街/镇/州 之一,或明确城市名)"),
|
|
33
|
+
("职业", "str", False, "职业短语,如 全职带娃 / 晚班调酒师"),
|
|
34
|
+
("单位", "str", False, "工作地点/机构名"),
|
|
35
|
+
("学历", "str", False, "本科/大专/硕士/研究生/博士/高中/中专/技校"),
|
|
36
|
+
("专业", "str", False, "专业名"),
|
|
37
|
+
("父", "str", False, "父亲的名字或职业(原文出现的形式)"),
|
|
38
|
+
("母", "str", False, "母亲的名字或状况(原文出现的形式)"),
|
|
39
|
+
# 【外测 2026-11-12】档案不能只是当前身份快照——这 19 题要的是经历/家庭**序列**。
|
|
40
|
+
("经历", "list", False, "职业经历序列,每项 {单位, 职位, 从, 到};原文只给部分就只填部分"),
|
|
41
|
+
("家庭", "list", False, "家庭成员序列,每项 {关系, 姓名, 职业}"),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
#: 允许的字段集合(LLM 填了表外字段 ⇒ 直接丢弃,不让它发明字段)。
|
|
45
|
+
FIELDS = tuple(f for f, _t, _r, _h in SCHEMA)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# 生效条件:无参;返回给 LLM 的确定性表模板(字段/类型/必填/提示)
|
|
49
|
+
def table_text():
|
|
50
|
+
"""生成「让 LLM 填」的表(纯文本,便于贴进提示词)。"""
|
|
51
|
+
lines = ["字段 | 类型 | 必填 | 说明", "--- | --- | --- | ---"]
|
|
52
|
+
for name, typ, req, hint in SCHEMA:
|
|
53
|
+
lines.append("%s | %s | %s | %s" % (name, typ, "是" if req else "否", hint))
|
|
54
|
+
lines.append("")
|
|
55
|
+
lines.append("规则:只填原文**明确出现**的值;原文没有的字段填空字符串;"
|
|
56
|
+
"不要推断、不要补全、不要换算。")
|
|
57
|
+
return "\n".join(lines)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# 生效条件:text 为原文;返回归一化文本(去空白,全角数字转半角)
|
|
61
|
+
def _norm(text):
|
|
62
|
+
s = str(text or "")
|
|
63
|
+
s = re.sub(r"\s+", "", s)
|
|
64
|
+
s = s.translate(str.maketrans("0123456789", "0123456789"))
|
|
65
|
+
return s
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# 生效条件:value 为待核值、text 为原文;在原文中可命中(或数值等值)返回命中的片段,否则 None
|
|
69
|
+
def span_of(value, text):
|
|
70
|
+
"""**原文复核单点**:返回命中片段(用于 ev_span),命中不了返回 None。"""
|
|
71
|
+
if value is None or value == "":
|
|
72
|
+
return None
|
|
73
|
+
nv = _norm(value)
|
|
74
|
+
nt = _norm(text)
|
|
75
|
+
if not nv or not nt:
|
|
76
|
+
return None
|
|
77
|
+
if nv in nt:
|
|
78
|
+
return str(value)
|
|
79
|
+
# 数值字段:24 vs 「24岁」「,24」——数值等值也算命中(但必须是原文的子串形式)
|
|
80
|
+
if isinstance(value, int) or (isinstance(value, str) and value.isdigit()):
|
|
81
|
+
if str(int(value)) in nt:
|
|
82
|
+
return str(value)
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
#: 关系词别名(**只对「家庭」项的「关系」分量**生效):LLM 惯写「母/父」,语料多写「妈/爸」。
|
|
87
|
+
#: 这是**确定性白名单**,不是放松复核——白名单外的写法仍然拒收。
|
|
88
|
+
RELATION_ALIASES = {
|
|
89
|
+
"母": ("妈妈", "妈", "母亲", "老娘"),
|
|
90
|
+
"父": ("爸爸", "爸", "父亲", "老爷子"),
|
|
91
|
+
"妻": ("妻子", "老婆", "媳妇"),
|
|
92
|
+
"夫": ("丈夫", "老公"),
|
|
93
|
+
"子": ("儿子",),
|
|
94
|
+
"女": ("女儿",),
|
|
95
|
+
"兄": ("哥哥", "哥"),
|
|
96
|
+
"姐": ("姐姐", "姐"),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# 生效条件:k 为分量名、v 为值、text 为原文;命中(含关系白名单)返回 True
|
|
101
|
+
def _part_in_source(k, v, text):
|
|
102
|
+
"""分量复核:普通分量走 span_of;「关系」分量额外接受别名白名单。"""
|
|
103
|
+
if span_of(v, text) is not None:
|
|
104
|
+
return True
|
|
105
|
+
if k == "关系":
|
|
106
|
+
for alias in RELATION_ALIASES.get(str(v), ()):
|
|
107
|
+
if span_of(alias, text) is not None:
|
|
108
|
+
return True
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
#: 槽位值的长度上限(超限即拒收,reason=too_long)。
|
|
113
|
+
#: 依据(外测 2026-11-12):LLM 会把「母」填成整句
|
|
114
|
+
#: "61了,从社区超市理货员退下来,现在当广场舞领队" —— 那不是槽位值。
|
|
115
|
+
#: 语义上这些槽位是**短值**;长内容应当拆成 家庭/经历 序列项。
|
|
116
|
+
MAX_LEN = {"姓名": 4, "年龄": 3, "城市": 12, "职业": 20, "单位": 20,
|
|
117
|
+
"学历": 6, "专业": 12, "父": 12, "母": 12}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# 生效条件:item 为序列项 dict;返回**扁平化**后的新 dict(值若是 {value,_raw} 取 value)
|
|
121
|
+
def _flat_item(item):
|
|
122
|
+
"""序列项扁平化:LLM 有时把值写成 {value, _raw},扁平后避免嵌套进卡。"""
|
|
123
|
+
out = {}
|
|
124
|
+
raw = None
|
|
125
|
+
for kk, vv in (item or {}).items():
|
|
126
|
+
if isinstance(vv, dict) and "value" in vv:
|
|
127
|
+
out[kk] = vv.get("value")
|
|
128
|
+
if vv.get("_raw") and not raw:
|
|
129
|
+
raw = vv.get("_raw")
|
|
130
|
+
else:
|
|
131
|
+
out[kk] = vv
|
|
132
|
+
if raw and "_raw" not in out:
|
|
133
|
+
out["_raw"] = raw
|
|
134
|
+
return out
|
|
135
|
+
|
|
136
|
+
#: 地名后缀(值必须带其一,或在已知地名词典里)。
|
|
137
|
+
PLACE_SUFFIX = ("市", "区", "县", "路", "街", "镇", "州", "村", "岛", "港")
|
|
138
|
+
#: 已知地名(可扩展;用于「南宁」「江北」这类无后缀写法)。
|
|
139
|
+
PLACE_KNOWN = ("南宁", "江北", "渝中", "半坡", "江北市", "渝中区")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
# 生效条件:v 为城市候选值;形如地名(后缀命中或词典命中)返回 True
|
|
143
|
+
def is_place(v):
|
|
144
|
+
"""城市值校验:挡「外出包里 / 包里 / 家里」这类方位短语。"""
|
|
145
|
+
s = str(v or "").strip()
|
|
146
|
+
if not s:
|
|
147
|
+
return False
|
|
148
|
+
if any(s.endswith(x) for x in PLACE_SUFFIX):
|
|
149
|
+
return True
|
|
150
|
+
return any(x in s for x in PLACE_KNOWN)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# 生效条件:item 为表项(dict,可能含 _raw);text 为原文;返回证据片段或 None
|
|
154
|
+
def _evidence_of(item, text):
|
|
155
|
+
"""取证据片段:优先用调用方给的 _raw(原文摘录),否则返回 None。
|
|
156
|
+
|
|
157
|
+
为什么必须支持 _raw(外测 502 次填表:947 次 no_item_in_source):
|
|
158
|
+
LLM 会把「做过一年学徒」归一成 职位=学徒、「江北那家咖啡店」写成 单位=江北咖啡店——
|
|
159
|
+
值对、字面不同。按值做子串匹配必然失败,按**证据片段**匹配应当通过。
|
|
160
|
+
复核对象 = 证据(必须真的出现在原文),而不是归一化后的值:既保住防幻觉
|
|
161
|
+
(_raw 不在原文 ⇒ 拒收),又不逼 LLM 逐字复制。
|
|
162
|
+
"""
|
|
163
|
+
raw = (item or {}).get("_raw") if isinstance(item, dict) else None
|
|
164
|
+
if raw:
|
|
165
|
+
return str(raw) if span_of(raw, text) is not None else None
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
# 生效条件:filled 为 LLM 填的表(dict)、text 为原文;返回 (accepted, rejected)
|
|
169
|
+
def verify(filled, text):
|
|
170
|
+
from . import dossier as _dos
|
|
171
|
+
"""逐字段复核:命中保留并给 ev_span;未命中拒收并给原因。"""
|
|
172
|
+
accepted, rejected = {}, []
|
|
173
|
+
for k, v in (filled or {}).items():
|
|
174
|
+
if k not in FIELDS:
|
|
175
|
+
rejected.append({"field": k, "value": v, "reason": "field_not_in_schema"})
|
|
176
|
+
continue
|
|
177
|
+
if v is None or v == "":
|
|
178
|
+
continue
|
|
179
|
+
# 【序列型】逐项复核:每项的字符串分量都要能在原文找到,否则丢该项
|
|
180
|
+
if k in _dos.LIST_FIELDS and isinstance(v, list):
|
|
181
|
+
_kept = []
|
|
182
|
+
for _it in v:
|
|
183
|
+
if not isinstance(_it, dict):
|
|
184
|
+
continue
|
|
185
|
+
# 【B 方案】项自带 _raw 原文片段且能在原文找到 ⇒ 接受该项(结构化改写允许)
|
|
186
|
+
_raw = _it.get("_raw")
|
|
187
|
+
if isinstance(_raw, str) and _raw.strip() and span_of(_raw, text) is not None:
|
|
188
|
+
_kept.append(_it)
|
|
189
|
+
continue
|
|
190
|
+
_it = _flat_item(_it)
|
|
191
|
+
_item_raw = _it.get("_raw")
|
|
192
|
+
if _item_raw:
|
|
193
|
+
_ok_item = span_of(_item_raw, text) is not None
|
|
194
|
+
else:
|
|
195
|
+
_ok_item = all(_part_in_source(_k2, x, text)
|
|
196
|
+
for _k2, x in _it.items()
|
|
197
|
+
if x not in (None, "") and _k2 != "_raw")
|
|
198
|
+
if _ok_item:
|
|
199
|
+
_kept.append(_it)
|
|
200
|
+
if not _kept:
|
|
201
|
+
rejected.append({"field": k, "value": v, "reason": "no_item_in_source"})
|
|
202
|
+
continue
|
|
203
|
+
accepted[k] = {"value": _kept, "ev_span": ""}
|
|
204
|
+
continue
|
|
205
|
+
_val = v.get("value") if isinstance(v, dict) else v
|
|
206
|
+
_raw_span = _evidence_of(v, text)
|
|
207
|
+
span = _raw_span or span_of(_val, text)
|
|
208
|
+
if span is None:
|
|
209
|
+
rejected.append({"field": k, "value": _val, "reason": "not_in_source"})
|
|
210
|
+
continue
|
|
211
|
+
# 字段级形态校验(目前只对城市):原文有 ≠ 是地名
|
|
212
|
+
_lim = MAX_LEN.get(k)
|
|
213
|
+
if _lim and isinstance(_val, str) and len(_val.strip()) > _lim:
|
|
214
|
+
rejected.append({"field": k, "value": _val, "reason": "too_long"})
|
|
215
|
+
continue
|
|
216
|
+
if k == "城市" and not is_place(_val):
|
|
217
|
+
rejected.append({"field": k, "value": _val, "reason": "not_a_place"})
|
|
218
|
+
continue
|
|
219
|
+
accepted[k] = {"value": _val, "ev_span": span}
|
|
220
|
+
return accepted, rejected
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# 生效条件:cg 提供 get;node_id 存在、filled 为 dict;返回 {applied, rejected, dossier}
|
|
224
|
+
def apply(cg, node_id, filled, text=None, ts=None, source="llm"):
|
|
225
|
+
"""把 LLM 填的表复核后并入档案。
|
|
226
|
+
|
|
227
|
+
**不写盘**(无副作用):返回合并后的 dossier,由调用方在正常写入路径里落盘
|
|
228
|
+
(这样只走一条写链,避免第二套写入面)。
|
|
229
|
+
"""
|
|
230
|
+
node = cg.get(node_id) or {}
|
|
231
|
+
if not node:
|
|
232
|
+
return {"ok": False, "error": "node_not_found", "node_id": node_id}
|
|
233
|
+
body = text if text is not None else (node.get("content") or "")
|
|
234
|
+
fm = node.get("frontmatter") or {}
|
|
235
|
+
ts = ts if ts is not None else fm.get("created_at")
|
|
236
|
+
accepted, rejected = verify(filled, body)
|
|
237
|
+
if not accepted:
|
|
238
|
+
return {"ok": True, "applied": {}, "rejected": rejected,
|
|
239
|
+
"dossier": dict(fm.get("dossier") or {}),
|
|
240
|
+
"note": "无可复核字段(全部拒收或空表)"}
|
|
241
|
+
from . import dossier as _dos
|
|
242
|
+
cur = dict(fm.get("dossier") or {})
|
|
243
|
+
incoming = {k: {"value": v["value"], "t": ts, "ev": node_id,
|
|
244
|
+
"ev_span": v["ev_span"], "source": source}
|
|
245
|
+
for k, v in accepted.items()}
|
|
246
|
+
merged = _dos.merge(cur, incoming)
|
|
247
|
+
return {"ok": True, "applied": accepted, "rejected": rejected, "dossier": merged}
|