@furongjun1999/dsh-memory 0.9.23 → 0.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +515 -404
- package/CHANGELOG.md +642 -449
- package/README.en.md +244 -240
- package/README.md +272 -262
- package/codebuddy/CODEBUDDY.md +1 -1
- package/docs/discipline/templates/full.md.tmpl +1 -1
- package/docs/discipline/templates/rules.mdc.tmpl +1 -1
- package/docs/en/mdcg/world-model-endpoint.md +1 -1
- package/docs/mdcg/cogmap_ops.md +19 -16
- package/lib/client.js +29 -3
- package/lib/hooks.d.ts +12 -0
- package/lib/hooks.js +43 -1
- package/lib/tools.d.ts +2 -2
- package/md_cg/bucketindex.py +231 -0
- package/md_cg/bucketscope.py +357 -0
- package/md_cg/cohesion.py +337 -0
- package/md_cg/config_registry_bulk.py +24 -10
- package/md_cg/config_validate.py +14 -2
- package/md_cg/constraint_audit.py +231 -0
- package/md_cg/constraint_watch.py +206 -0
- package/md_cg/context_audit.py +240 -0
- package/md_cg/context_plan.py +142 -0
- package/md_cg/detrerank.py +137 -0
- package/md_cg/distill.py +247 -0
- package/md_cg/dossier.py +151 -0
- package/md_cg/entities.py +133 -0
- package/md_cg/fsutil.py +135 -0
- package/md_cg/govern/__init__.py +23 -0
- package/md_cg/iface/__init__.py +46 -0
- package/md_cg/iface/_verify.py +609 -0
- package/md_cg/iface/contracts.py +337 -0
- package/md_cg/judgment_manifest.py +14 -0
- package/md_cg/kernel/__init__.py +27 -0
- package/md_cg/mcp_server.py +6044 -5761
- package/md_cg/mdcg.py +501 -16
- package/md_cg/mdcos.py +546 -12
- package/md_cg/metacognition.py +66 -13
- package/md_cg/mutate_extract_shipped.py +126 -0
- package/md_cg/mutate_ledger_e2e.py +100 -0
- package/md_cg/op_table.py +22 -0
- package/md_cg/probe_c_ledger_integrity.py +258 -0
- package/md_cg/probe_d_retirement_discipline.py +263 -0
- package/md_cg/profiles.py +298 -0
- package/md_cg/protocol.py +7 -1
- package/md_cg/reldate.py +142 -0
- package/md_cg/review_cli.py +6 -1
- package/md_cg/run_tests.py +30 -1
- package/md_cg/scopeselect.py +214 -0
- package/md_cg/scrub.py +66 -9
- package/md_cg/selfstmt.py +139 -0
- package/md_cg/state_extract.py +149 -0
- package/md_cg/state_slots.py +62 -10
- package/md_cg/state_view.py +181 -0
- package/md_cg/structboost.py +179 -0
- package/md_cg/sustain.py +28 -2
- package/md_cg/test_alternatives_historical.py +165 -0
- package/md_cg/test_b2_sensitivity_landing.py +226 -223
- package/md_cg/test_b3_merge_keeps_content.py +608 -583
- package/md_cg/test_bucket_cards.py +85 -0
- package/md_cg/test_bucket_day_order.py +74 -0
- package/md_cg/test_bucket_quota.py +79 -0
- package/md_cg/test_bucket_recency_guards.py +75 -0
- package/md_cg/test_bucket_three_causes.py +83 -0
- package/md_cg/test_bucket_time_rounds.py +87 -0
- package/md_cg/test_bucketindex.py +130 -0
- package/md_cg/test_bucketscope_early.py +71 -0
- package/md_cg/test_cohesion.py +107 -0
- package/md_cg/test_cond_term_fold_gate.py +1210 -0
- package/md_cg/test_cond_term_neg_scope.py +147 -0
- package/md_cg/test_constraint_audit.py +87 -0
- package/md_cg/test_constraint_watch.py +108 -0
- package/md_cg/test_context_audit.py +110 -0
- package/md_cg/test_context_plan.py +81 -0
- package/md_cg/test_detrerank.py +157 -0
- package/md_cg/test_gate_four_kpi.py +136 -0
- package/md_cg/test_generation_guard.py +14 -2
- package/md_cg/test_issue192_move_layer_indices_guard.py +48 -0
- package/md_cg/test_issue39_utf8_stdio.py +572 -572
- package/md_cg/test_issue413_preimage_failclosed.py +147 -9
- package/md_cg/test_issue768_overwrite_time_window_anchor.py +135 -0
- package/md_cg/test_issue789_log_plaintext_probe.py +100 -0
- package/md_cg/test_issue790_verb_negation.py +77 -0
- package/md_cg/test_issue810_root_guard.py +71 -0
- package/md_cg/test_issue812_info_index_stale.py +78 -0
- package/md_cg/test_issue827_weaken_dedup.py +361 -0
- package/md_cg/test_issue829_calibration_positive_gate.py +473 -0
- package/md_cg/test_kernel_iface_contracts.py +18 -0
- package/md_cg/test_md_cg_no_hive_dep.py +238 -17
- package/md_cg/test_probe_cd_shipped.py +385 -0
- package/md_cg/test_recent_tail.py +93 -0
- package/md_cg/test_run_tests_entry_gates.py +110 -0
- package/md_cg/test_scopeselect.py +70 -0
- package/md_cg/test_scopeselect_clock.py +70 -0
- package/md_cg/test_session_attribution_consistency.py +835 -813
- package/md_cg/test_state_extract_shipped.py +162 -0
- package/md_cg/test_state_ledger_chain.py +207 -0
- package/md_cg/test_state_ledger_e2e.py +145 -0
- package/md_cg/test_structboost.py +102 -0
- package/md_cg/test_timeboost.py +83 -0
- package/md_cg/test_timeboost_relativedate.py +71 -0
- package/md_cg/test_topicagg.py +129 -0
- package/md_cg/timeboost.py +223 -0
- package/md_cg/tokens.py +1 -1
- package/md_cg/topicagg.py +142 -0
- package/md_cg/version_anchor.py +84 -1
- package/md_cg/whitebox_kb/wisdom/code_compose.py +33 -5
- package/md_cg/whitebox_kb/wisdom/code_solidified.json +6195 -6195
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +5 -1
- package/md_cg/whitebox_kb/wisdom/role_compose.py +271 -268
- package/package.json +1 -1
- package/src/hooks.ts +43 -1
- package/ui/minimal/payload/build-ui.mjs +41 -2
- package/ui/minimal/payload/lib/client.js +29 -3
- package/ui/minimal/payload/web/app.js +3 -1
- package/ui/minimal/payload/web/client.js +30 -3
- package/web/app.js +3 -1
- package/web/client.js +30 -3
- package/zcode/AGENTS.md +1 -1
package/md_cg/dossier.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""人物档案(画像)抽取——按"公民档案"结构,从会话里**增量补字段**。
|
|
3
|
+
|
|
4
|
+
判据(使用者 2026-11-12):按中国公安/人社局的公民档案思路做人物档案(人物画像),会话中不断补充。
|
|
5
|
+
|
|
6
|
+
字段:基本信息(姓名/年龄/城市)· 身份职业(职业/单位/学历/专业)· 家庭(父/母)。
|
|
7
|
+
每条带 {value, t(陈述时刻), ev(来源节点)},可回溯;merge() 增量合并,旧值进 _history。
|
|
8
|
+
|
|
9
|
+
语料实证(外测提供的原句):
|
|
10
|
+
P01 #15 我叫沈知夏,知了的知,夏天的夏,24
|
|
11
|
+
P12 #8 我叫罗杏子,今年38,在南宁,全职带娃,你叫我罗姐就行。
|
|
12
|
+
P16 #1 …我叫祁野,25,在渝中区半坡酒馆做晚班调酒师。
|
|
13
|
+
P17 #13 还没正经跟你说我名字。我叫谈书宁,谈是谈话的谈,书是书本的书…
|
|
14
|
+
P17 #27 明天还得陪我爸谈国华去社区医院开降糖药
|
|
15
|
+
|
|
16
|
+
由此定下的五条形态(正则按此写,不按教科书):
|
|
17
|
+
① 姓名后可能跟**拆字解释**("谈是谈话的谈…")⇒ 截断在分隔符前;
|
|
18
|
+
② 年龄**常常没有"岁"字**("我叫X,24" / "今年38");
|
|
19
|
+
③ 城市**没有稳定后缀**("在南宁" / "在渝中区")⇒ 带行政区后缀 或 「在X,/。」;
|
|
20
|
+
④ 职业是「做<短语>」或「全职带娃」这类整体短语;
|
|
21
|
+
⑤ 父母两种形态:「我爸<人名>」与「我妈<状况>」——不硬把状况当人名。
|
|
22
|
+
|
|
23
|
+
**隐私边界(不做)**:身份证号、证件号、银行卡、手机号、详细住址——命中任一敏感字样则整段不建档。
|
|
24
|
+
档案只服务回忆与检索,不服务身份核验。
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import re
|
|
29
|
+
|
|
30
|
+
_RULES = (
|
|
31
|
+
("姓名", re.compile(r"(?:我叫|我的名字是|我名字叫|叫我)([\u4e00-\u9fa5]{2,4})(?=[,,。!!;;\s]|$)")),
|
|
32
|
+
("年龄", re.compile(r"今年(\d{1,2})(?!\d)|(\d{1,2})\s*岁")),
|
|
33
|
+
("城市", re.compile(r"我(?:住在|住|在)([\u4e00-\u9fa5]{2,6}?(?:市|区|县))"
|
|
34
|
+
r"|(?:我)?在([\u4e00-\u9fa5]{2,4})(?=[,,。])")),
|
|
35
|
+
("职业", re.compile(r"做([\u4e00-\u9fa5]{2,8}?(?:师|员|工|手|调酒|设计|护理))"
|
|
36
|
+
r"|(全职带娃|全职妈妈|全职爸爸)")),
|
|
37
|
+
("单位", re.compile(r"在([\u4e00-\u9fa5]{2,12}?(?:店|馆|公司|学校|医院|厂|所|局))"
|
|
38
|
+
r"(?:做|上班|工作)")),
|
|
39
|
+
("学历", re.compile(r"(本科|大专|硕士|研究生|博士|高中|中专|技校)")),
|
|
40
|
+
("专业", re.compile(r"([\u4e00-\u9fa5]{2,8})专业")),
|
|
41
|
+
("父", re.compile(r"我(?:爸|父亲)([\u4e00-\u9fa5]{2,4})(?=[去说来在陪嫌,,。])")),
|
|
42
|
+
("母", re.compile(r"我(?:妈|母亲)([^,,。;;]{2,12})")),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
#: 列表型字段(值为 list[dict],逐项复核;合并时按项去重追加,互不覆盖)
|
|
46
|
+
LIST_FIELDS = ("经历", "家庭")
|
|
47
|
+
|
|
48
|
+
#: 明确 **不采** 的敏感字段(出现即整段拒收)。
|
|
49
|
+
FORBIDDEN = ("身份证", "证件号", "银行卡", "手机号", "电话", "详细住址", "门牌")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# 生效条件:text 为正文、ts 为陈述时刻(float)、nid 为来源节点;返回 {字段: {value, t, ev}}
|
|
53
|
+
def extract(text, ts=None, nid=""):
|
|
54
|
+
"""从一段正文里抽档案字段(确定性;抽不到不写)。"""
|
|
55
|
+
body = str(text or "")
|
|
56
|
+
if any(w in body for w in FORBIDDEN):
|
|
57
|
+
return {}
|
|
58
|
+
out = {}
|
|
59
|
+
for field, rx in _RULES:
|
|
60
|
+
for m in rx.finditer(body):
|
|
61
|
+
grp = next((g for g in m.groups() if g), None)
|
|
62
|
+
if not grp:
|
|
63
|
+
continue
|
|
64
|
+
val = grp.strip()
|
|
65
|
+
if field == "年龄":
|
|
66
|
+
try:
|
|
67
|
+
n = int(val)
|
|
68
|
+
except ValueError:
|
|
69
|
+
continue
|
|
70
|
+
if not (1 <= n <= 120):
|
|
71
|
+
continue
|
|
72
|
+
val = n
|
|
73
|
+
out[field] = {"value": val, "t": ts, "ev": nid}
|
|
74
|
+
break
|
|
75
|
+
# 自我介绍里的**裸数字**也算年龄(「我叫沈知夏,…,24」无「岁」字)。
|
|
76
|
+
# 只在同段已认出姓名时启用,避免把任意数字当年龄。
|
|
77
|
+
if "姓名" in out and "年龄" not in out:
|
|
78
|
+
m2 = re.search(r"[,,]\s*(\d{1,2})\s*(?:[,,。]|$)", body)
|
|
79
|
+
if m2:
|
|
80
|
+
try:
|
|
81
|
+
n2 = int(m2.group(1))
|
|
82
|
+
except ValueError:
|
|
83
|
+
n2 = None
|
|
84
|
+
if n2 and 1 <= n2 <= 120:
|
|
85
|
+
out["年龄"] = {"value": n2, "t": ts, "ev": nid}
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
#: 字段的时间可变性:只有这些字段允许「同权重下较新值覆盖」;其余同权重**保留旧值**。
|
|
90
|
+
#: 理由(外测实证):多来源 dossier 合并时,弱证据曾把「姓名/城市」覆盖成错值。
|
|
91
|
+
TIME_VARYING = ("年龄",)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# 生效条件:text 为原文;返回该节点作为档案证据的**权重**(3=自我介绍/声明,2=行为句,1=泛提)
|
|
95
|
+
def weight_of(text):
|
|
96
|
+
"""证据权重:自我介绍最强;含「在/做/住」的行为句次之;其余最弱。"""
|
|
97
|
+
body = str(text or "")
|
|
98
|
+
if re.search(r"(我叫|我的名字是|我名字叫|叫我)[\u4e00-\u9fa5]{2,4}", body):
|
|
99
|
+
return 3
|
|
100
|
+
if re.search(r"(今年\d|\d{1,2}\s*岁|我(?:住在|住|在)|做[\u4e00-\u9fa5]{2,8})", body):
|
|
101
|
+
return 2
|
|
102
|
+
return 1
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# 生效条件:old/new 为两条档案 dict;返回合并结果(新值覆盖;旧值进 _history 不丢)
|
|
106
|
+
def merge(old, new):
|
|
107
|
+
"""档案增量合并:同字段取**较新**的值,旧值保留在 _history。"""
|
|
108
|
+
out = dict(old or {})
|
|
109
|
+
hist = dict(out.get("_history") or {})
|
|
110
|
+
for k, v in (new or {}).items():
|
|
111
|
+
if k.startswith("_"):
|
|
112
|
+
continue
|
|
113
|
+
prev = out.get(k)
|
|
114
|
+
if prev and prev.get("value") == v.get("value"):
|
|
115
|
+
# 同值:仅抬高权重、补 ev_span(不新增历史)
|
|
116
|
+
if (v.get("w") or 0) > (prev.get("w") or 0):
|
|
117
|
+
prev["w"] = v["w"]
|
|
118
|
+
continue
|
|
119
|
+
# 【序列型】经历/家庭:按项去重追加(不同项都保留,不互相覆盖)
|
|
120
|
+
if k in LIST_FIELDS and isinstance(prev, dict) and isinstance(v, dict):
|
|
121
|
+
_old = list(prev.get("value") or [])
|
|
122
|
+
_new = list(v.get("value") or [])
|
|
123
|
+
def _key(it):
|
|
124
|
+
return tuple(sorted((str(a), str(b)) for a, b in (it or {}).items()
|
|
125
|
+
if b not in (None, "")))
|
|
126
|
+
_seen = {_key(x) for x in _old}
|
|
127
|
+
for _it in _new:
|
|
128
|
+
if _key(_it) not in _seen:
|
|
129
|
+
_old.append(_it)
|
|
130
|
+
_seen.add(_key(_it))
|
|
131
|
+
prev = dict(prev)
|
|
132
|
+
prev["value"] = _old
|
|
133
|
+
if (v.get("w") or 0) > (prev.get("w") or 0):
|
|
134
|
+
prev["w"] = v["w"]
|
|
135
|
+
out[k] = prev
|
|
136
|
+
continue
|
|
137
|
+
if prev:
|
|
138
|
+
_pw, _nw = (prev.get("w") or 0), (v.get("w") or 0)
|
|
139
|
+
# 【证据权重】强证据胜;同权重时只有时间可变字段允许较新值覆盖,
|
|
140
|
+
# 其余**保留旧值**——避免「在包里」这类弱证据盖掉自我介绍的姓名/城市。
|
|
141
|
+
if _nw < _pw:
|
|
142
|
+
hist.setdefault(k, []).append(v)
|
|
143
|
+
continue
|
|
144
|
+
if _nw == _pw and k not in TIME_VARYING:
|
|
145
|
+
hist.setdefault(k, []).append(v)
|
|
146
|
+
continue
|
|
147
|
+
hist.setdefault(k, []).append(prev)
|
|
148
|
+
out[k] = v
|
|
149
|
+
if hist:
|
|
150
|
+
out["_history"] = hist
|
|
151
|
+
return out
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""实体抽取(确定性)——人物 / 地点 / 时间 核心;物品 / 情绪 辅助。
|
|
3
|
+
|
|
4
|
+
判据(使用者 2026-11-12):人物·地点·时间·事件是**核心状态**,物品与情绪是**辅助**
|
|
5
|
+
(只用于辅助检索,不建跟踪树)。
|
|
6
|
+
|
|
7
|
+
本模块**不用模型**,全部确定性规则:
|
|
8
|
+
· 人物:自我介绍式(我叫X/叫我X)、称谓式(X阿姨/师傅/姐/哥/老师/编辑)、
|
|
9
|
+
对话式(X说/X问/和X约),以及已有 tags 里的 person: 前缀;
|
|
10
|
+
· 地点:动词后接处所词干(在/去/到/回/从 + X店/馆/院/校/门口/小区/路/街…);
|
|
11
|
+
· 时间:先按绝对日期正则(YYYY-MM-DD、M月D日),再由调用方用 `reldate` 以陈述时刻换算相对表述;
|
|
12
|
+
· 辅助(物品/情绪):物品取引号内与「拿/借/寄/买/放」宾语;情绪取小词表。**只作为 facet**,不进状态树。
|
|
13
|
+
|
|
14
|
+
不适用条件(不猜):
|
|
15
|
+
· 两字以上才认人名(单字姓名歧义过大);
|
|
16
|
+
· 「我是X」不认(跟职业/身份混在一起,如「我是面包师」);
|
|
17
|
+
· 处所词必须命中词尾表,否则不认(宁缺不错);
|
|
18
|
+
· 本模块**不判断真假/归属关系**,只产出候选,去重与归属由 profiles 层做。
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
#: 人物称谓后缀(出现在名字之后)。
|
|
25
|
+
_HONORIFIC = ("阿姨", "叔叔", "师傅", "姐", "哥", "老师", "医生", "编辑", "编",
|
|
26
|
+
"总", "老板", "园长", "经理")
|
|
27
|
+
|
|
28
|
+
#: 处所词尾(地点候选必须以此结尾)。
|
|
29
|
+
_PLACE_TAIL = ("店", "馆", "院", "校", "门口", "小区", "路", "街", "市", "省",
|
|
30
|
+
"楼", "层", "公园", "书店", "医院", "学校", "咖啡店", "公司")
|
|
31
|
+
|
|
32
|
+
#: 情绪小词表(辅助面)。
|
|
33
|
+
_EMOTION = ("开心", "高兴", "难过", "累", "烦", "怕", "紧张", "委屈", "生气",
|
|
34
|
+
"踏实", "欣慰", "焦虑", "轻松", "感动")
|
|
35
|
+
|
|
36
|
+
_NAME = r"[\u4e00-\u9fa5]{2,4}"
|
|
37
|
+
_SELF_INTRO = re.compile(r"(?:我叫|我叫作|叫我|名字叫)(" + _NAME + ")")
|
|
38
|
+
_HONOR = re.compile(r"([\u4e00-\u9fa5]{1,3})(?:" + "|".join(_HONORIFIC) + ")")
|
|
39
|
+
_SPEAK = re.compile(r"(" + _NAME + r")(?:说|讲|问|回|答|笑|提醒|建议)")
|
|
40
|
+
_WITH = re.compile(r"(?:和|跟|与)(" + _NAME + r")(?:说好|约|一起|商量|碰面)")
|
|
41
|
+
_PLACE = re.compile(r"(?:在|去|到|回|从|来)([\u4e00-\u9fa5]{1,10}?(?:" + "|".join(_PLACE_TAIL) + r"))")
|
|
42
|
+
_DATE = re.compile(r"(\d{4}-\d{2}-\d{2}|\d{4}/(\d{1,2})/(\d{1,2})|(\d{1,2})月(\d{1,2})[日号])")
|
|
43
|
+
_QUOTED = re.compile(r"[「『“\"]([^」』”\"]{1,20})[」』”\"]")
|
|
44
|
+
_OBJ_VERB = re.compile(r"(?:拿|借|寄|买|放|带|送|收)([\u4e00-\u9fa5]{1,8})")
|
|
45
|
+
|
|
46
|
+
#: 明显不是人名的词(动词/代词/常见名词),避免「他说」「我说」被当人名。
|
|
47
|
+
_NOT_NAME = {"他说", "我说", "你说", "她说", "他说", "告诉", "觉得", "知道",
|
|
48
|
+
"今天", "昨天", "明天", "晚上", "上午", "下午", "那边", "这里",
|
|
49
|
+
"我们", "你们", "他们", "自己", "大家", "有人", "别人"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# 代词/泛称:绝不能当人名(外测实证:曾抽出 profile-person-号你)。
|
|
53
|
+
_PRONOUN = {
|
|
54
|
+
"你", "我", "他", "她", "它", "您", "咱", "谁", "这", "那", "其",
|
|
55
|
+
"你们", "我们", "他们", "她们", "它们", "咱们", "大家", "各位",
|
|
56
|
+
"有人", "别人", "自己", "人家", "某人",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
# 语气/助词:出现在姓名首或尾即判非人名(如「号你」「了吗」)。
|
|
60
|
+
_PARTICLE_CHARS = set(
|
|
61
|
+
"了吗呢吧啊呀嘛哦哈欸诶啦咯喔噢得了着过地"
|
|
62
|
+
"你您我他她它这那"
|
|
63
|
+
"说问讲让把给和跟与在是有要就都也还什怎"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# 生效条件:name 为候选串;通过(非代词、长度 2-4、纯汉字、非语气助词首尾)返回 True
|
|
68
|
+
def gate_name(name):
|
|
69
|
+
"""人名闸:代词 / 长度 / 字符类 / 语气助词首尾 四道。宁缺不错。"""
|
|
70
|
+
n = (name or "").strip()
|
|
71
|
+
if not n or n in _PRONOUN or n in _NOT_NAME:
|
|
72
|
+
return False
|
|
73
|
+
if not (2 <= len(n) <= 4):
|
|
74
|
+
return False
|
|
75
|
+
if not all("\u4e00" <= ch <= "\u9fff" for ch in n):
|
|
76
|
+
return False
|
|
77
|
+
if n[0] in _PARTICLE_CHARS or n[-1] in _PARTICLE_CHARS:
|
|
78
|
+
return False
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# 生效条件:items 为候选串集合;返回去重后的有序列表(排除 _NOT_NAME 与空)
|
|
83
|
+
def _dedupe(items):
|
|
84
|
+
seen, out = set(), []
|
|
85
|
+
for x in items or []:
|
|
86
|
+
x = (x or "").strip()
|
|
87
|
+
if not x or x in seen or x in _NOT_NAME or len(x) < 2:
|
|
88
|
+
continue
|
|
89
|
+
seen.add(x)
|
|
90
|
+
out.append(x)
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# 生效条件:text 为正文、tags 为节点标签;返回人物候选(自我介绍/称谓/对话/和X约 + person: 标签)
|
|
95
|
+
def persons(text, tags=()):
|
|
96
|
+
body = str(text or "")
|
|
97
|
+
got = []
|
|
98
|
+
got += [m.group(1) for m in _SELF_INTRO.finditer(body)]
|
|
99
|
+
got += [m.group(1) for m in _HONOR.finditer(body)]
|
|
100
|
+
got += [m.group(1) for m in _SPEAK.finditer(body)]
|
|
101
|
+
got += [m.group(1) for m in _WITH.finditer(body)]
|
|
102
|
+
for t in tags or []:
|
|
103
|
+
t = str(t)
|
|
104
|
+
for pre in ("person:", "人物:"):
|
|
105
|
+
if t.startswith(pre):
|
|
106
|
+
got.append(t[len(pre):])
|
|
107
|
+
return [n for n in _dedupe(got) if gate_name(n)]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# 生效条件:text 为正文、tags 为节点标签;返回地点候选(动词 + 处所词尾;place: 标签)
|
|
111
|
+
def places(text, tags=()):
|
|
112
|
+
body = str(text or "")
|
|
113
|
+
got = [m.group(1) for m in _PLACE.finditer(body)]
|
|
114
|
+
for t in tags or []:
|
|
115
|
+
t = str(t)
|
|
116
|
+
for pre in ("place:", "地点:"):
|
|
117
|
+
if t.startswith(pre):
|
|
118
|
+
got.append(t[len(pre):])
|
|
119
|
+
return _dedupe(got)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# 生效条件:text 为正文;返回正文里出现的**绝对日期**原文串(相对表述由 reldate 另换算)
|
|
123
|
+
def dates(text):
|
|
124
|
+
return _dedupe([m.group(0) for m in _DATE.finditer(str(text or ""))])
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# 生效条件:text 为正文;返回辅助面 facet {"objects": [...], "emotions": [...]}(**不进状态树**)
|
|
128
|
+
def auxiliary(text):
|
|
129
|
+
body = str(text or "")
|
|
130
|
+
objs = [m.group(1) for m in _OBJ_VERB.finditer(body)]
|
|
131
|
+
objs += [m.group(1) for m in _QUOTED.finditer(body)]
|
|
132
|
+
emos = [w for w in _EMOTION if w in body]
|
|
133
|
+
return {"objects": _dedupe(objs), "emotions": _dedupe(emos)}
|
package/md_cg/fsutil.py
CHANGED
|
@@ -999,6 +999,36 @@ def ensure_shard_dir(directory: str, where: str = "append") -> bool:
|
|
|
999
999
|
|
|
1000
1000
|
|
|
1001
1001
|
# 生效条件:directory 经 abspath 存入 self.dir 后经 ensure_shard_dir(self.dir, "init") 保证目录存在(首建不记账;失败抛 ShardDirError 而非裸 OSError),实例分片文件名由 os.getpid() 与 uuid.uuid4().hex[:8] 拼成 "{pid}-{hex8}.log",append 时各写者只写自己这一分片,从而不共享写入点。
|
|
1002
|
+
#: 增量游标回看最后一个换行的**有界**扫描窗口,以及总上限。
|
|
1003
|
+
#: 窗口内找不到换行(超长行)时退一个窗口继续;累计到上限仍未找到 ⇒ 记 0
|
|
1004
|
+
#: (安全方向:下次整片重读,重放幂等,不会漏记录)。
|
|
1005
|
+
_CURSOR_CHUNK = 64 * 1024
|
|
1006
|
+
_CURSOR_SCAN_CAP = 8 * 1024 * 1024
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
# 生效条件:path 存在且 size>0 时,从 size 起向前回看找最后一个 "\n" 并返回其字节后一位(即已提交行边界);回看到 _CURSOR_SCAN_CAP 仍未找到、或读失败时返回 0(全片重读,安全方向)。
|
|
1010
|
+
def _last_newline_boundary(path: str, size: int) -> int:
|
|
1011
|
+
"""分片「已提交行边界」字节水位(只认完整行,与 read_jsonl_tail 同口径)。"""
|
|
1012
|
+
scanned = 0
|
|
1013
|
+
end = size
|
|
1014
|
+
try:
|
|
1015
|
+
with open(path, "rb") as f:
|
|
1016
|
+
while end > 0 and scanned < _CURSOR_SCAN_CAP:
|
|
1017
|
+
start = max(0, end - _CURSOR_CHUNK)
|
|
1018
|
+
f.seek(start)
|
|
1019
|
+
buf = f.read(end - start)
|
|
1020
|
+
if not buf:
|
|
1021
|
+
break
|
|
1022
|
+
i = buf.rfind(b"\n")
|
|
1023
|
+
if i >= 0:
|
|
1024
|
+
return start + i + 1
|
|
1025
|
+
scanned += len(buf)
|
|
1026
|
+
end = start
|
|
1027
|
+
except OSError:
|
|
1028
|
+
return 0
|
|
1029
|
+
return 0
|
|
1030
|
+
|
|
1031
|
+
|
|
1002
1032
|
class ShardedLog:
|
|
1003
1033
|
"""每写者独占一个分片的 append-only 日志——不能丢记录时用它。
|
|
1004
1034
|
|
|
@@ -1142,6 +1172,111 @@ class ShardedLog:
|
|
|
1142
1172
|
recs.sort(key=lambda r: (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))))
|
|
1143
1173
|
return recs
|
|
1144
1174
|
|
|
1175
|
+
@staticmethod
|
|
1176
|
+
# 生效条件:directory 为目录时返回 dict(分片文件名 -> 该文件当前**已提交行边界**的字节水位);文件不存在/不可读的条目跳过;目录不可列时返回 {}。
|
|
1177
|
+
def committed_cursor(directory: str):
|
|
1178
|
+
"""各分片**已提交行边界**的字节水位(增量重载的基线游标)。
|
|
1179
|
+
|
|
1180
|
+
只需知道「整行到哪里为止」,不必解析内容。做法:从文件尾回看一个
|
|
1181
|
+
有界窗口找最后一个 `\\n`;窗口内找不到(超长行)则退一个窗口继续,
|
|
1182
|
+
最多回看 `_CURSOR_SCAN_CAP` 字节,仍未找到就记 0——**记 0 是安全方向**:
|
|
1183
|
+
下次增量会重读整片,重放幂等,只是慢一点,不会漏记录。
|
|
1184
|
+
|
|
1185
|
+
调用方的使用约束(**安全方向**):本函数返回的水位必须在
|
|
1186
|
+
`read_all()` **之前**取。取在前 ⇒ 水位 ⊆ 实际被消费的记录 ⇒ 下次
|
|
1187
|
+
至多重放已应用过的记录(幂等,last-wins)。取在后则可能把「尚未
|
|
1188
|
+
消费的记录」也算进水位,造成**漏重放**(静默丢写入)。
|
|
1189
|
+
"""
|
|
1190
|
+
out = {}
|
|
1191
|
+
if not os.path.isdir(directory):
|
|
1192
|
+
return out
|
|
1193
|
+
try:
|
|
1194
|
+
names = sorted(os.listdir(directory))
|
|
1195
|
+
except OSError:
|
|
1196
|
+
return out
|
|
1197
|
+
for fn in names:
|
|
1198
|
+
if not fn.endswith(".log"):
|
|
1199
|
+
continue
|
|
1200
|
+
p = os.path.join(directory, fn)
|
|
1201
|
+
try:
|
|
1202
|
+
size = os.stat(p).st_size
|
|
1203
|
+
except OSError:
|
|
1204
|
+
continue
|
|
1205
|
+
if size == 0:
|
|
1206
|
+
out[fn] = 0
|
|
1207
|
+
continue
|
|
1208
|
+
out[fn] = _last_newline_boundary(p, size)
|
|
1209
|
+
return out
|
|
1210
|
+
|
|
1211
|
+
@staticmethod
|
|
1212
|
+
# 生效条件:directory 为目录且 cursor 为 dict(分片文件名 -> 已消费字节水位)时,逐分片读**新增尾部**并按 (_t,_s) 排序返回 (记录列表, 新游标);游标里的分片已消失(被 compact 清走)、或某分片当前大小小于其水位(被截断/重建)时返回 (None, None) 表示**不可增量**,调用方须回退全量重放;directory 不是目录时返回 ([], {})。
|
|
1213
|
+
def read_tails(directory: str, cursor: dict):
|
|
1214
|
+
"""读分片日志的**新增尾部**(issue #766 增量化:W3 读侧重载成本)。
|
|
1215
|
+
|
|
1216
|
+
为什么必须有它:`flush()` 只 append 分片、**从不清**;只有
|
|
1217
|
+
`close()`→`compact_index` 才 `ShardedLog.clear`。故长期存活写方
|
|
1218
|
+
(MCP serve 本体即 `autoflush=1 只 flush 不 close`)的分片日志
|
|
1219
|
+
**只增不减**,而 `read_all` 每次都把全部记录重读重解析一遍 ⇒
|
|
1220
|
+
单次重载成本随「该库写入总量」无界增长(实测 300 节点、日志
|
|
1221
|
+
2000→12000 条,单次 `_load_index` 65→258 ms,斜率约 18ms/千条)。
|
|
1222
|
+
增量后该成本降为 O(新增)。
|
|
1223
|
+
|
|
1224
|
+
**正确性边界(诚实声明)**:本函数只负责「读到并排序新增记录」,
|
|
1225
|
+
**不判**这些记录能否安全叠加上已有物化态——排序水位判定、快照
|
|
1226
|
+
是否变过、盘面是否被非合作写者改过,全在调用方(`MdCG._load_index`)。
|
|
1227
|
+
返回 `(None, None)` 的三类情形即「无法证明增量正确」,一律回退。
|
|
1228
|
+
|
|
1229
|
+
`read_jsonl_tail` 只认**完整行**(末行无换行属未提交,留在水位内
|
|
1230
|
+
下次再读),故并发写者正写到一半时不会读到半截 JSON,也不会把
|
|
1231
|
+
水位推过它——与 `append_jsonl` 的 ends_mid_line 同一防线。
|
|
1232
|
+
"""
|
|
1233
|
+
if not os.path.isdir(directory):
|
|
1234
|
+
return [], {}
|
|
1235
|
+
|
|
1236
|
+
def _norm(v):
|
|
1237
|
+
return v if (isinstance(v, (int, float)) and v == v) else 0
|
|
1238
|
+
|
|
1239
|
+
try:
|
|
1240
|
+
names = sorted(os.listdir(directory))
|
|
1241
|
+
except OSError:
|
|
1242
|
+
return None, None
|
|
1243
|
+
|
|
1244
|
+
present = set()
|
|
1245
|
+
recs, new_cursor = [], {}
|
|
1246
|
+
for fn in names:
|
|
1247
|
+
if not fn.endswith(".log"):
|
|
1248
|
+
continue
|
|
1249
|
+
present.add(fn)
|
|
1250
|
+
p = os.path.join(directory, fn)
|
|
1251
|
+
try:
|
|
1252
|
+
size = os.stat(p).st_size
|
|
1253
|
+
except OSError:
|
|
1254
|
+
continue # delete-pending:视同消失,交由下面的校验判
|
|
1255
|
+
off = cursor.get(fn, 0)
|
|
1256
|
+
if size < off:
|
|
1257
|
+
# 分片被截断/重建:水位之后的内容已不可信,全量重放
|
|
1258
|
+
return None, None
|
|
1259
|
+
if size == off:
|
|
1260
|
+
new_cursor[fn] = off
|
|
1261
|
+
continue
|
|
1262
|
+
batch, nxt = read_jsonl_tail(p, off)
|
|
1263
|
+
if nxt is None:
|
|
1264
|
+
return None, None
|
|
1265
|
+
new_cursor[fn] = nxt
|
|
1266
|
+
recs.extend(batch)
|
|
1267
|
+
|
|
1268
|
+
# 游标里记着、盘上已没有的分片:只可能是 compact 清走了(记录已并入
|
|
1269
|
+
# 快照)⇒ 快照必已重写、物化态作废,不可增量。
|
|
1270
|
+
if set(cursor) - present:
|
|
1271
|
+
return None, None
|
|
1272
|
+
|
|
1273
|
+
bad = [r for r in recs if not isinstance(r, dict)]
|
|
1274
|
+
if bad:
|
|
1275
|
+
note_nonobject_rows([(directory, 0, r) for r in bad])
|
|
1276
|
+
recs = [r for r in recs if isinstance(r, dict)]
|
|
1277
|
+
recs.sort(key=lambda r: (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))))
|
|
1278
|
+
return recs, new_cursor
|
|
1279
|
+
|
|
1145
1280
|
@staticmethod
|
|
1146
1281
|
# 生效条件:directory 是目录时,遍历 os.listdir(directory),对以 ".log" 结尾且不满足「keep 为真值且 os.path.abspath(p) == keep」的条目调用 os.remove(keep 为 None/空串等假值时该排除条件恒不成立,所有 ".log" 条目都会被删),删除时的 OSError 被忽略;directory 不是目录时直接返回。
|
|
1147
1282
|
def clear(directory: str, keep: str = None):
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""灵枢 1.0 治理层(govern)——W1 骨架,无实现。
|
|
3
|
+
|
|
4
|
+
真源:docs/architecture/灵枢1.0_建设路线_v0.1.md §2 / §5 M4。
|
|
5
|
+
|
|
6
|
+
治理层职责(路线稿 §2):
|
|
7
|
+
生命周期四档 · 审计台账 · 回滚/2PC · 资格判定(DEFER/BLINDSPOT)
|
|
8
|
+
|
|
9
|
+
**本包为 W1 骨架(M1)**:只放类型与协议。M4 才把 revise/retire/rollback
|
|
10
|
+
真正切换过来;届时 `rollback` 必须**覆盖正文**——该能力已由
|
|
11
|
+
`md_cg/rollback.py` 实现(`method=preimage`,逐字节回前像),M4 的验收是
|
|
12
|
+
**回归保持全绿**,不是重新实现(详见路线稿 §4 注)。
|
|
13
|
+
|
|
14
|
+
与内核层的关系:治理层可依赖内核层;内核层**不得**反向依赖本包。
|
|
15
|
+
打分层不得输出 ``state``(#111 既定契约)——`state` 只由治理层给出。
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
__all__ = ["GOVERN_LAYER", "LIFECYCLE_STATES"]
|
|
19
|
+
|
|
20
|
+
GOVERN_LAYER = "govern"
|
|
21
|
+
|
|
22
|
+
#: 生命周期四档(路线稿 §2 的「生命周期四档」;具体语义在 M4 落定)。
|
|
23
|
+
LIFECYCLE_STATES = ("active", "dormant", "retired", "tombstoned")
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""灵枢 1.0 接口层(iface)——四个契约的唯一出口。
|
|
3
|
+
|
|
4
|
+
真源:docs/architecture/灵枢1.0_建设路线_v0.1.md §3。
|
|
5
|
+
|
|
6
|
+
适配层(dsh 插件 / MCP server / hive worker)**只允许**从本包导入契约类型;
|
|
7
|
+
不得直接 import 内核层或治理层的实现模块(路线稿 §2 禁止项)。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .contracts import (
|
|
11
|
+
CONTRACT_NAMES,
|
|
12
|
+
IFACE_LAYER,
|
|
13
|
+
ContractViolation,
|
|
14
|
+
GovernContract,
|
|
15
|
+
ReadContract,
|
|
16
|
+
ReadItem,
|
|
17
|
+
ReadRequest,
|
|
18
|
+
RecordState,
|
|
19
|
+
SelfCheckContract,
|
|
20
|
+
SelfCheckReport,
|
|
21
|
+
WriteContract,
|
|
22
|
+
WriteReceipt,
|
|
23
|
+
WriteRequest,
|
|
24
|
+
reject_state_in_scoring,
|
|
25
|
+
require_provenance,
|
|
26
|
+
require_why,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"CONTRACT_NAMES",
|
|
31
|
+
"IFACE_LAYER",
|
|
32
|
+
"ContractViolation",
|
|
33
|
+
"WriteContract",
|
|
34
|
+
"ReadContract",
|
|
35
|
+
"GovernContract",
|
|
36
|
+
"SelfCheckContract",
|
|
37
|
+
"WriteRequest",
|
|
38
|
+
"WriteReceipt",
|
|
39
|
+
"ReadRequest",
|
|
40
|
+
"ReadItem",
|
|
41
|
+
"SelfCheckReport",
|
|
42
|
+
"RecordState",
|
|
43
|
+
"require_provenance",
|
|
44
|
+
"require_why",
|
|
45
|
+
"reject_state_in_scoring",
|
|
46
|
+
]
|