@furongjun1999/dsh-memory 0.4.11 → 0.5.0

package/md_cg/semantic/export_en_zh_map.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""导出 en→zh 扁平映射（en_zh_map.json）——rust/mcdg-eval 统一归一层的词表源。
+
+批次 15（2026-09-23 使用者口径：统一翻译为中文→归一化到标准中文集→检索）。
+
+为何导出而非 Rust 移植：en_normalizer 的词表面（MORPHEME/COMPOUND_ZH/
+CEDICT 链/IRREGULAR）若在 Rust 复刻，即出现第二份词典真源——词典版本
+敏感（同词面不同版本映射不同 → 两侧分数漂移）正是当初弃用外部分词器的
+原因。故词表真源单侧化：本脚本用 **normalize_en_query 自身** 算每个词面
+的终态（生成即经全链：strip_tense→停用词→CEDICT 链→复合词），产物是
+派生缓存——词表更新后重跑本脚本再生成，两侧永不漂移。
+
+键覆盖：MORPHEME ∪ COMPOUND_ZH ∪ IRREGULAR 全部键（IRREGULAR 键经
+strip_tense 展开到原形后再走链，如 ate→eat→吃）；值为该词面经完整归一
+链的中文原子串（空格 join）。表外词两侧一致保留原词（unknown_keep）。
+
+Rust 侧消费：MDCG_EN_ZH_MAP 指向本文件（rust/src/atoms.rs）。
+用法：python -m md_cg.semantic.export_en_zh_map
+"""
+import json
+import os
+
+from .en_normalizer import (IRREGULAR, COMPOUND_ZH, EN_ZH, cedict_map,
+                            normalize_en_query)
+from .zh_en_atoms import ZH_EN
+
+OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "en_zh_map.json")
+
+
+# 生效条件：无 required 形参；keys 取 MORPHEME_ZH、COMPOUND_ZH、IRREGULAR 三表键的并集（IRREGULAR 值原形并入 keys）；对每个 key 经 normalize_en_query 归一取首个非空中文词序列并以单空格 join，含字母的产物（unknown_keep）跳过不落表；返回排序后的 dict 并写入 OUT（encoding=utf-8，ensure_ascii=False，indent=1）。
+def build():
+    """生成扁平 word→zh 映射并写盘。返回映射 dict（供测试断言）。"""
+    keys = set(EN_ZH) | set(COMPOUND_ZH)
+    keys |= set(cedict_map())            # CEDICT 链命中词（compiler/test…）同权导出
+    keys |= set(IRREGULAR.values())       # 原形（write）与屈折形（wrote）都要能查
+    keys |= set(IRREGULAR.keys())
+    out = {}
+    for w in sorted(keys):
+        try:
+            terms, _d = normalize_en_query(w)
+        except Exception:
+            continue
+        zh = " ".join(t for t in terms if t and not any(
+            "a" <= c.lower() <= "z" for c in t)).strip()
+        if zh and zh != w:
+            out[w.lower()] = zh
+    with open(OUT, "w", encoding="utf-8") as f:
+        json.dump({"meta": {"version": "b15", "entries": len(out),
+                            "derived_from": "en_normalizer 全链",
+                            "zh_atoms": len(ZH_EN)},
+                   "map": out,
+                   # segment 贪心匹配键源（zh_en_atoms.ZH_EN 键 = 标准原子集，
+                   # 与 canonical.atoms_zh() 的 OOV 审计基准同源）
+                   "zh_keys": sorted(ZH_EN.keys())}, f,
+                  ensure_ascii=False, indent=1)
+    return out
+
+
+if __name__ == "__main__":
+    m = build()
+    print(f"en_zh_map.json: {len(m)} entries -> {OUT}")
+    for k in ("wrote", "write", "tests", "beef", "soul hub"):
+        print(f"  {k!r} -> {m.get(k)!r}")

package/md_cg/semantic/unify.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""灵枢 · 检索 query 统一归一层（批次 15，2026-09-23 使用者口径定案）
+
+口径：**统一翻译为中文 → 归一化到标准中文集（atoms.json）→ 检索**。
+任何语言的 query 先归一为标准原子序列（空格 join）再进检索各路：
+  - 英文：en_normalizer（时态还原→停用词剔除→en→zh 语素映射）→
+    segment 展开为标准原子（「牛肉」→「牛 肉」，与 doc 侧 fm.semantic
+    原子形态同构）；
+  - 中文：segment 贪心最长匹配归一到 atoms.json 标准概念；
+  - 纯中文 query **原样返回**——segment+join+bigrams(去空白) 后与原文
+    等价，归一只对跨语 query 有信息增益（beef 与「牛肉」零共享字符，
+    形态归一救不了跨语），纯中文零变更纪律；
+  - 专名/未登录词原样保留（词表边界 unknown_keep，非错误）。
+
+开关：MDCG_UNIFY_QUERY 默认 "1"（2026-09-23 使用者拍板口径转正）；
+"=0" 显式关回退。两侧同步：rust/mcdg-eval 读同一 env + 同一份
+atoms.json（scripts/rank_parity.py --dataset 对拍在两侧同开关态进行，
+A/B 差异归档为转正证据）。
+
+归一失败（semantic 模块缺失等）静默原样返回——与 en_zh_terms 同降级
+风格，不阻断检索主链路。
+"""
+import os
+import re
+
+# 生效条件：无 required 形参，锚定环境变量名 MDCG_UNIFY_QUERY；当 os.environ.get("MDCG_UNIFY_QUERY", "1") == "1" 时返回 True，否则返回 False。
+def unify_on() -> bool:
+    """统一归一层开关（默认开=口径定案转正；=0 显式关回退）。"""
+    return os.environ.get("MDCG_UNIFY_QUERY", "1") == "1"
+
+
+# 生效条件：text 为 None/空串或开关关闭或不含 [A-Za-z] 时原样返回；否则经 canonical.query_atoms 归一为标准原子序列并以单空格 join 返回（归一产物为空或抛异常时原样返回）。
+def unify_query(text):
+    """检索入口统一归一：任意语言 query → 标准原子序列。"""
+    t = (text or "").strip()
+    if not t or not unify_on() or not re.search(r"[A-Za-z]", t):
+        return text
+    try:
+        from .canonical import query_atoms
+        atoms = query_atoms(t)
+    except Exception:
+        return text
+    u = " ".join(a for a in atoms if a).strip()
+    return u if u else text

package/md_cg/semantic/zh_en_atoms.py

@@ -1,139 +1,139 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""灵枢 · 原子级中英序列化器
-
-语义原子（中文）→ 标准英文原子序列
-不是翻译，是原子的一对一映射。每个中文原子有唯一英文原子对应。
-
-用法：
-    from zh_en_atoms import serialize
-    result = serialize("我昨天吃牛肉")
-    # → "i yester day eat cow meat"
-
-    result = serialize("灵枢记忆库")
-    # → "soul hub memory store"
-"""
-import re
-
-ZH_EN = {}
-
-# 生效条件：传入任意 zh 与 en（含空串等假值）时，段内无条件执行 ZH_EN[zh] = en，即以 zh 为键写入 en 并返回。
-def M(zh, en):
-    ZH_EN[zh] = en
-
-# ---- 代词 ----
-for zh, en in [("我","i"),("你","you"),("他","he"),("她","she"),("我们","we"),("他们","they")]:
-    M(zh, en)
-
-# ---- 动作（基形，无时态）----
-for zh, en in [("吃","eat"),("喝","drink"),("看","see"),("听","hear"),("读","read"),
-               ("写","write"),("走","walk"),("跑","run"),("飞","fly"),("来","come"),
-               ("去","go"),("买","buy"),("卖","sell"),("说","speak"),("想","think"),
-               ("开","open"),("关","close"),("睡","sleep"),("爱","love"),("做","do"),
-               ("学","study"),("教","teach"),("用","use"),("找","find"),("给","give")]:
-    M(zh, en)
-
-# ---- 时间（复合→原子序列）----
-M("昨天", "yester day"); M("今天", "to day"); M("明天", "next day"); M("现在", "now")
-for zh, en in [("年","year"),("月","month"),("日","day"),("天","day"),("周","week"),
-               ("时","hour"),("分","minute"),("秒","second"),
-               ("春","spring"),("夏","summer"),("秋","autumn"),("冬","winter"),
-               ("早","morning"),("晚","evening"),("夜","night")]:
-    M(zh, en)
-
-# ---- 数 ----
-for zh, en in [("一","one"),("二","two"),("三","three"),("四","four"),("五","five"),
-               ("六","six"),("七","seven"),("八","eight"),("九","nine"),("十","ten"),
-               ("百","hundred"),("千","thousand"),("零","zero")]:
-    M(zh, en)
-
-# ---- 自然 ----
-for zh, en in [("牛","cow"),("猪","pig"),("羊","sheep"),("鸡","chicken"),("鸭","duck"),
-               ("马","horse"),("狗","dog"),("猫","cat"),("鱼","fish"),("鸟","bird"),
-               ("肉","meat"),("奶","milk"),("蛋","egg"),("水","water"),("火","fire"),
-               ("山","mountain"),("海","sea"),("河","river"),("树","tree"),("花","flower"),
-               ("光","light"),("电","electric"),("风","wind"),("雨","rain"),("雪","snow"),
-               ("星","star"),("石","stone"),("云","cloud"),("土","soil"),("金","metal")]:
-    M(zh, en)
-
-# ---- 身体 ----
-for zh, en in [("头","head"),("手","hand"),("脚","foot"),("眼","eye"),("口","mouth"),
-               ("耳","ear"),("鼻","nose"),("心","heart"),("血","blood"),("骨","bone"),("皮","skin")]:
-    M(zh, en)
-
-# ---- 亲属 ----
-for zh, en in [("父","father"),("母","mother"),("兄","elder_brother"),("弟","younger_brother"),
-               ("姐","elder_sister"),("妹","younger_sister"),("儿","son"),("女","daughter")]:
-    M(zh, en)
-
-# ---- 器物 ----
-for zh, en in [("车","vehicle"),("船","boat"),("机","machine"),("刀","knife"),("笔","pen"),
-               ("书","book"),("门","door"),("窗","window"),("桥","bridge"),("路","road"),
-               ("池","cell")]:
-    M(zh, en)
-
-# ---- 形容词 ----
-for zh, en in [("大","big"),("小","small"),("高","high"),("低","low"),("新","new"),("旧","old"),
-               ("好","good"),("坏","bad"),("长","long"),("短","short"),("快","fast"),("慢","slow"),
-               ("热","hot"),("冷","cold"),("多","many"),("少","few"),("重","heavy"),("轻","light"),
-               ("深","deep"),("真","true")]:
-    M(zh, en)
-
-# ---- 抽象 ----
-for zh, en in [("知识","knowledge"),("工作","work"),("纪律","discipline"),("记忆","memory"),
-               ("认知","cognition"),("力","force"),("气","gas"),("道","way"),("理","principle"),
-               ("文","text"),("化","transform"),("话","speech"),("影","shadow"),("音","sound"),
-               ("名","name"),("国","country"),("家","home"),("城","city"),("民","people"),
-               ("字","word"),("灵","soul"),("枢","hub"),("数","number"),("物","thing"),
-               ("地","earth"),("人","person"),("脑","brain"),("视","view"),("脏","organ"),
-               ("汽","gas"),("魂","soul")]:
-    M(zh, en)
-
-# ---- 复合词（→ 原子序列）----
-for zh, en in [("电脑","electric brain"),("电话","electric speech"),("电视","electric view"),
-               ("电影","electric shadow"),("电池","electric cell"),("火车","fire vehicle"),
-               ("汽车","gas vehicle"),("飞机","fly machine"),("手机","hand machine"),
-               ("牛肉","cow meat"),("猪肉","pig meat"),("羊肉","sheep meat"),
-               ("鸡肉","chicken meat"),("牛奶","cow milk"),("鸡蛋","chicken egg"),
-               ("火山","fire mountain"),("海水","sea water"),("月光","moon light"),
-               ("雪山","snow mountain"),("知识图谱","knowledge graph map"),
-               ("认知图","cognition graph"),("工作纪律","work discipline"),
-               ("记忆库","memory store"),("灵枢","soul hub"),
-               ("数学","number study"),("化学","transform study"),
-               ("物理学","thing principle study"),("生物学","living thing study"),
-               ("地理学","earth principle study"),("热力学","heat force study")]:
-    M(zh, en)
-
-# ---- 贪心最长匹配 ----
-_sorted = sorted(ZH_EN.keys(), key=len, reverse=True)
-
-# 生效条件：text 为空串时 while pos < len(text) 不进入、直接返回空列表；否则按 _sorted 顺序用 text.startswith(key, pos) 命中即 append 该 key、pos 增 len(key) 并 break，未命中则 append text[pos] 且 pos 增 1，直至 pos 达 len(text) 后返回该原子列表。
-def segment(text):
-    """贪心最长匹配：文本 → 原子列表"""
-    result = []
-    pos = 0
-    while pos < len(text):
-        matched = False
-        for key in _sorted:
-            if text.startswith(key, pos):
-                result.append(key)
-                pos += len(key)
-                matched = True
-                break
-        if not matched:
-            result.append(text[pos])
-            pos += 1
-    return result
-
-
-# 生效条件：对每个由 text 经 segment(text) 得到的原子 a，按 ZH_EN.get(a, "[" + a + "]") 取值（a 为 ZH_EN 缺失键时用 "[" + a + "]"），再以单空格 " ".join 连接为字符串返回。
-def serialize(text):
-    """中文文本 → 标准英文原子序列"""
-    return " ".join(ZH_EN.get(a, "[" + a + "]") for a in segment(text))
-
-
-if __name__ == "__main__":
-    import sys
-    q = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "灵枢记忆库"
-    print("中文:", q)
-    print("标准英文:", serialize(q))
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""灵枢 · 原子级中英序列化器
+
+语义原子（中文）→ 标准英文原子序列
+不是翻译，是原子的一对一映射。每个中文原子有唯一英文原子对应。
+
+用法：
+    from zh_en_atoms import serialize
+    result = serialize("我昨天吃牛肉")
+    # → "i yester day eat cow meat"
+
+    result = serialize("灵枢记忆库")
+    # → "soul hub memory store"
+"""
+import re
+
+ZH_EN = {}
+
+# 生效条件：传入任意 zh 与 en（含空串等假值）时，段内无条件执行 ZH_EN[zh] = en，即以 zh 为键写入 en 并返回。
+def M(zh, en):
+    ZH_EN[zh] = en
+
+# ---- 代词 ----
+for zh, en in [("我","i"),("你","you"),("他","he"),("她","she"),("我们","we"),("他们","they")]:
+    M(zh, en)
+
+# ---- 动作（基形，无时态）----
+for zh, en in [("吃","eat"),("喝","drink"),("看","see"),("听","hear"),("读","read"),
+               ("写","write"),("走","walk"),("跑","run"),("飞","fly"),("来","come"),
+               ("去","go"),("买","buy"),("卖","sell"),("说","speak"),("想","think"),
+               ("开","open"),("关","close"),("睡","sleep"),("爱","love"),("做","do"),
+               ("学","study"),("教","teach"),("用","use"),("找","find"),("给","give")]:
+    M(zh, en)
+
+# ---- 时间（复合→原子序列）----
+M("昨天", "yester day"); M("今天", "to day"); M("明天", "next day"); M("现在", "now")
+for zh, en in [("年","year"),("月","month"),("日","day"),("天","day"),("周","week"),
+               ("时","hour"),("分","minute"),("秒","second"),
+               ("春","spring"),("夏","summer"),("秋","autumn"),("冬","winter"),
+               ("早","morning"),("晚","evening"),("夜","night")]:
+    M(zh, en)
+
+# ---- 数 ----
+for zh, en in [("一","one"),("二","two"),("三","three"),("四","four"),("五","five"),
+               ("六","six"),("七","seven"),("八","eight"),("九","nine"),("十","ten"),
+               ("百","hundred"),("千","thousand"),("零","zero")]:
+    M(zh, en)
+
+# ---- 自然 ----
+for zh, en in [("牛","cow"),("猪","pig"),("羊","sheep"),("鸡","chicken"),("鸭","duck"),
+               ("马","horse"),("狗","dog"),("猫","cat"),("鱼","fish"),("鸟","bird"),
+               ("肉","meat"),("奶","milk"),("蛋","egg"),("水","water"),("火","fire"),
+               ("山","mountain"),("海","sea"),("河","river"),("树","tree"),("花","flower"),
+               ("光","light"),("电","electric"),("风","wind"),("雨","rain"),("雪","snow"),
+               ("星","star"),("石","stone"),("云","cloud"),("土","soil"),("金","metal")]:
+    M(zh, en)
+
+# ---- 身体 ----
+for zh, en in [("头","head"),("手","hand"),("脚","foot"),("眼","eye"),("口","mouth"),
+               ("耳","ear"),("鼻","nose"),("心","heart"),("血","blood"),("骨","bone"),("皮","skin")]:
+    M(zh, en)
+
+# ---- 亲属 ----
+for zh, en in [("父","father"),("母","mother"),("兄","elder_brother"),("弟","younger_brother"),
+               ("姐","elder_sister"),("妹","younger_sister"),("儿","son"),("女","daughter")]:
+    M(zh, en)
+
+# ---- 器物 ----
+for zh, en in [("车","vehicle"),("船","boat"),("机","machine"),("刀","knife"),("笔","pen"),
+               ("书","book"),("门","door"),("窗","window"),("桥","bridge"),("路","road"),
+               ("池","cell")]:
+    M(zh, en)
+
+# ---- 形容词 ----
+for zh, en in [("大","big"),("小","small"),("高","high"),("低","low"),("新","new"),("旧","old"),
+               ("好","good"),("坏","bad"),("长","long"),("短","short"),("快","fast"),("慢","slow"),
+               ("热","hot"),("冷","cold"),("多","many"),("少","few"),("重","heavy"),("轻","light"),
+               ("深","deep"),("真","true")]:
+    M(zh, en)
+
+# ---- 抽象 ----
+for zh, en in [("知识","knowledge"),("工作","work"),("纪律","discipline"),("记忆","memory"),
+               ("认知","cognition"),("力","force"),("气","gas"),("道","way"),("理","principle"),
+               ("文","text"),("化","transform"),("话","speech"),("影","shadow"),("音","sound"),
+               ("名","name"),("国","country"),("家","home"),("城","city"),("民","people"),
+               ("字","word"),("灵","soul"),("枢","hub"),("数","number"),("物","thing"),
+               ("地","earth"),("人","person"),("脑","brain"),("视","view"),("脏","organ"),
+               ("汽","gas"),("魂","soul")]:
+    M(zh, en)
+
+# ---- 复合词（→ 原子序列）----
+for zh, en in [("电脑","electric brain"),("电话","electric speech"),("电视","electric view"),
+               ("电影","electric shadow"),("电池","electric cell"),("火车","fire vehicle"),
+               ("汽车","gas vehicle"),("飞机","fly machine"),("手机","hand machine"),
+               ("牛肉","cow meat"),("猪肉","pig meat"),("羊肉","sheep meat"),
+               ("鸡肉","chicken meat"),("牛奶","cow milk"),("鸡蛋","chicken egg"),
+               ("火山","fire mountain"),("海水","sea water"),("月光","moon light"),
+               ("雪山","snow mountain"),("知识图谱","knowledge graph map"),
+               ("认知图","cognition graph"),("工作纪律","work discipline"),
+               ("记忆库","memory store"),("灵枢","soul hub"),
+               ("数学","number study"),("化学","transform study"),
+               ("物理学","thing principle study"),("生物学","living thing study"),
+               ("地理学","earth principle study"),("热力学","heat force study")]:
+    M(zh, en)
+
+# ---- 贪心最长匹配 ----
+_sorted = sorted(ZH_EN.keys(), key=len, reverse=True)
+
+# 生效条件：text 为空串时 while pos < len(text) 不进入、直接返回空列表；否则按 _sorted 顺序用 text.startswith(key, pos) 命中即 append 该 key、pos 增 len(key) 并 break，未命中则 append text[pos] 且 pos 增 1，直至 pos 达 len(text) 后返回该原子列表。
+def segment(text):
+    """贪心最长匹配：文本 → 原子列表"""
+    result = []
+    pos = 0
+    while pos < len(text):
+        matched = False
+        for key in _sorted:
+            if text.startswith(key, pos):
+                result.append(key)
+                pos += len(key)
+                matched = True
+                break
+        if not matched:
+            result.append(text[pos])
+            pos += 1
+    return result
+
+
+# 生效条件：对每个由 text 经 segment(text) 得到的原子 a，按 ZH_EN.get(a, "[" + a + "]") 取值（a 为 ZH_EN 缺失键时用 "[" + a + "]"），再以单空格 " ".join 连接为字符串返回。
+def serialize(text):
+    """中文文本 → 标准英文原子序列"""
+    return " ".join(ZH_EN.get(a, "[" + a + "]") for a in segment(text))
+
+
+if __name__ == "__main__":
+    import sys
+    q = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "灵枢记忆库"
+    print("中文:", q)
+    print("标准英文:", serialize(q))