@furongjun1999/dsh-memory 0.4.11 → 0.5.1

package/md_cg/test_mdstore_search_parity.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""MdStore.search_content 倒排预计算守卫（批次 22，issue #31 外评三步建议·第一步）。
+
+外部评价（2026-09-23）指出智慧之书面检索仍是全池遍历 + 每查询重复
+fold/json.dumps（实测 2000 节点 222.9ms，77% 热点在 translate）。
+本批落地「索引先行、算法不动」：装载期预计算每节点检索面串
+（fold content / fold tags_json / 去空白 content），检索面消重复构造。
+
+守卫三原则（外评判据的落地）：
+  P1 **输出逐位不变**——参考实现（改动前逐字节逻辑）与现实现对照，
+     覆盖命中/落空回退/layers 过滤/多 limit；
+  P2 reload 重建——写后 reload 快照刷新，预计算随 _nodes 同生命周期；
+  P3 冒烟级性能上界（宽松，避免 #30① 式负载敏感假红——2000 节点
+     实测 1.5ms，上界放到 50ms 仍比改动前低一个数量级）。
+"""
+import json
+import os
+import shutil
+import statistics
+import sys
+import tempfile
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), "md_cg", "whitebox_kb"))
+
+from md_cg.whitebox_kb.wisdom import md_store as _ms_mod
+from md_cg.whitebox_kb.wisdom.md_store import MdStore
+from aeis_core import LayeredStore, MemoryLayer
+
+PASS = 0
+FAIL = 0
+FAILS = []
+
+
+def check(name, cond, detail=""):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print(f"  OK   {name}")
+    else:
+        FAIL += 1
+        FAILS.append(name)
+        print(f"  FAIL {name}  {detail}")
+
+
+def reference_search(ms, query, layers=None, limit=20):
+    """改动前逐字节逻辑（对照真源）。"""
+    from md_cg.whitebox_kb.wisdom.md_store import _fold
+    q = (query or "").strip()
+    if not q:
+        return []
+    ms._ensure()
+    pool = list(ms._nodes.values())
+    if layers:
+        vals = {l.value if isinstance(l, MemoryLayer) else str(l)
+                for l in layers}
+        pool = [n for n in pool if n.layer.value in vals]
+    terms = LayeredStore.expand_query_terms(q)
+
+    def _hit(n):
+        c = _fold(n.content or "")
+        tj = _fold(json.dumps(n.tags or [], ensure_ascii=False))
+        return any(_fold(t) in c or _fold(t) in tj for t in terms)
+
+    rows = sorted((n for n in pool if _hit(n)), key=lambda n: n.id)[:300]
+    if not rows:
+        rows = sorted(pool, key=lambda n: n.id)[:500]
+    qb = ms._bigrams(q)
+    scored = []
+    for n in rows:
+        nb = ms._bigrams(n.content)
+        sim = (len(qb & nb) / len(qb)) if qb else 0.0
+        bonus = 0.05 if any(t in q or q in t for t in (n.tags or [])
+                            ) else 0.0
+        scored.append((n, min(1.0, sim + bonus)))
+    scored.sort(key=lambda x: (-x[1], -x[0].importance, x[0].id))
+    return scored[:max(0, int(limit))]
+
+
+def build(root, n=600):
+    body = ("蜂群调度与检索收敛的正文样本，含令牌验签、心跳判据、依赖门禁、"
+            "冲突闸放行、白箱裁决与知识图谱联动等内容段落。" * 12)
+    os.makedirs(os.path.join(root, "knowledge"), exist_ok=True)
+    for i in range(n):
+        fm = {"id": "n%05d" % i, "layer": "knowledge", "importance": 0.5,
+              "tags": json.dumps(["样本", "检索"], ensure_ascii=False)}
+        with open(os.path.join(root, "knowledge", "n%05d.md" % i), "w",
+                  encoding="utf-8") as f:
+            f.write("---\n" + json.dumps(fm, ensure_ascii=False)
+                    + "\n---\n# 正文\n" + body + " 编号 %d\n" % i)
+
+
+def main():
+    root = tempfile.mkdtemp(prefix="mdcg_mdstore_parity_")
+    build(root)
+    ms = MdStore(root)
+    try:
+        queries = ["蜂群调度 令牌", "心跳判据 白箱", "依赖门禁 收敛",
+                   "检索 样本", "不存在的词xyz", "样本"]
+
+        print("== P1 输出逐位不变（参考实现对照）==")
+        ok = True
+        for q in queries:
+            for lim in (5, 20):
+                ka = [(n.id, s) for n, s in reference_search(ms, q, limit=lim)]
+                kb = [(n.id, s) for n, s in ms.search_content(q, limit=lim)]
+                if ka != kb:
+                    ok = False
+                    print(f"    MISMATCH {q!r} limit={lim}: {ka[:3]} vs {kb[:3]}")
+        check("P1 逐位一致（6 query × 2 limit，含落空回退）", ok)
+
+        print("== P2 reload 后预计算重建 ==")
+        with open(os.path.join(root, "knowledge", "n_new.md"), "w",
+                  encoding="utf-8") as f:
+            f.write("---\n" + json.dumps(
+                {"id": "n_new", "layer": "knowledge", "importance": 0.9,
+                 "tags": "[]"}, ensure_ascii=False)
+                + "\n---\n# 正文\n独有标记量子纠缠态收尾词\n")
+        ms.reload()
+        r = ms.search_content("量子纠缠态收尾词")
+        check("P2a reload 后新节点可检索",
+              any(n.id == "n_new" for n, _ in r), f"ids={[n.id for n, _ in r]}")
+        ka = [(n.id, s) for n, s in reference_search(ms, "量子纠缠态收尾词")]
+        kb = [(n.id, s) for n, s in r]
+        check("P2b reload 后仍逐位一致", ka == kb, f"{ka[:3]} vs {kb[:3]}")
+
+        print("== P3 冒烟级性能上界（宽松，防负载敏感假红）==")
+        ms.search_content("蜂群调度")            # 预热
+        ts = []
+        for q in ("蜂群调度 令牌", "心跳判据 白箱", "依赖门禁 收敛"):
+            t0 = time.perf_counter()
+            ms.search_content(q)
+            ts.append((time.perf_counter() - t0) * 1000)
+        med = statistics.median(ts)
+        check("P3 2000 节点中位数 < 50ms（冒烟上界；实测 ~1.5ms，"
+              "改前 222.9ms）", med < 50.0, f"median={med:.1f}ms")
+
+        # layers 过滤路径也对照（资格面与预筛面组合）
+        ka = [(n.id, s) for n, s in reference_search(
+            ms, "蜂群调度", layers=[MemoryLayer.KNOWLEDGE], limit=10)]
+        kb = [(n.id, s) for n, s in ms.search_content(
+            "蜂群调度", layers=[MemoryLayer.KNOWLEDGE], limit=10)]
+        check("P4 layers 过滤路径逐位一致", ka == kb,
+              f"{ka[:3]} vs {kb[:3]}")
+    finally:
+        shutil.rmtree(root, ignore_errors=True)
+
+    print("\n" + "=" * 60)
+    print(f"结果：PASS {PASS} / FAIL {FAIL}")
+    if FAILS:
+        for f in FAILS:
+            print(f"  - {f}")
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/md_cg/test_merge_upsert.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""merge 裁决定向增量收尾守卫（issue：merge 用 rebuild_index O(N) 收尾）。
+
+缺陷形态（缺陷报告实测）：review_decide(merge) 只改一个目标节点
+（content 追加 + 不适用条件并集）却用 rebuild_index 收尾——_scan_nodes
+全库逐文件 open+parse（1500 池实测 ~97-247ms/次），而同场景 accept/edit
+走 add+flush 增量收尾（O(1)，~3.4ms）。
+
+本守卫断言（调用计数/对账形态，非时间阈值）：
+  M1 merge 零全库扫描：merge 裁决期间 _scan_nodes 调用 == 0
+     （旧代码 rebuild→1 次全库扫描 → 红）；
+  M2 upsert 与 rebuild 同源零漂移：merge 完成后目标节点索引条目与
+     之后再 rebuild_index 的条目逐位相等（entry 构造唯一真源）；
+  M3 merge 语义保持：content 追加命中、不适用条件并集、裁决记录
+     accepted、级联关闭同内容兄弟提案；
+  M4 accept 对照不受影响（零全库扫描、结果 ok）；
+  M5 定向收尾落增量日志：merge 后未 flush 尾部（_dirty）在统一收尾
+     flush 后，新进程重开可见目标新条目（增量账本，不靠 rebuild 救回）。
+"""
+import os
+import shutil
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import md_cg.mdcg as mdcg_mod               # noqa: E402
+from md_cg.mdcos import MdCGOS              # noqa: E402
+
+PASS = 0
+FAIL = 0
+FAILS = []
+
+
+def check(name, cond, detail=""):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print(f"  OK   {name}")
+    else:
+        FAIL += 1
+        FAILS.append(name)
+        print(f"  FAIL {name}  {detail}")
+
+
+def count_scans(fn):
+    n = 0
+    real = mdcg_mod.MdCG._scan_nodes
+
+    def _spy(self):
+        nonlocal n
+        n += 1
+        return real(self)
+
+    mdcg_mod.MdCG._scan_nodes = _spy
+    try:
+        return fn(), n
+    finally:
+        mdcg_mod.MdCG._scan_nodes = real
+
+
+def build(root, n=6):
+    cg = MdCGOS(root)
+    for i in range(n):
+        cg.add("mem_%02d" % i, "# 功能名：样本 %d\n# 正文：蜂群调度 内容 %d" % (i, i),
+               layer="knowledge")
+    cg.add("tgt", "# 功能名：合并目标\n# 正文：目标原始内容 星云\n目标正文。",
+           layer="knowledge", tags=["合并"],
+           non_applicable_conditions=["旧条件甲"])
+    cg.close()
+    return root
+
+
+def main():
+    root = tempfile.mkdtemp(prefix="mdcg_merge_guard_")
+    try:
+        build(root)
+        cg = MdCGOS(root)
+        pid = cg.propose("cand", "# 功能名：候选\n# 正文：候选正文 水位信箱",
+                         layer="knowledge")
+        # 同内容兄弟提案（级联关闭面）：propose 对同 payload_hash 幂等返回
+        # 既有 pid——兄弟须以他进程形态手工入队（同 phash 不同 pid）
+        from md_cg.fsutil import append_jsonl
+        from md_cg.mdcos import _sig
+        sib_content = "# 功能名：候选\n# 正文：候选正文 水位信箱"
+        append_jsonl(cg.inbox_log,
+                     {"t": 1.0, "pid": "prop_sib_manual", "id": "cand_sib",
+                      "content": sib_content, "layer": "knowledge",
+                      "tags": [], "condition_space": {}, "verify": {},
+                      "payload_hash": _sig(sib_content), "actor": "ext"})
+
+        out, n_scan = count_scans(
+            lambda: cg.review_decide(pid, "merge", merge_into="tgt",
+                                     reason="guard"))
+        print("  (merge 期间 _scan_nodes 调用 %d 次)" % n_scan)
+        check("M1a merge 裁决零全库扫描（旧代码 rebuild→1 次 O(N) 扫描）",
+              n_scan == 0, f"scans={n_scan}")
+        check("M3a merge 返回 ok 且 node_id 指向目标",
+              out.get("ok") and out.get("node_id") == "tgt", str(out)[:120])
+
+        # M2 upsert 与 rebuild 同源：先存 merge 后条目，rebuild 后逐位比对
+        e_after_merge = dict(cg.index["nodes"].get("tgt") or {})
+        cg.rebuild_index()
+        e_after_rebuild = dict(cg.index["nodes"].get("tgt") or {})
+        check("M2a merge 定向 upsert 条目与 rebuild 后逐位一致（同源零漂移）",
+              e_after_merge == e_after_rebuild,
+              f"diff={ {k: (e_after_merge.get(k), e_after_rebuild.get(k)) for k in set(e_after_merge) | set(e_after_rebuild) if e_after_merge.get(k) != e_after_rebuild.get(k)} }")
+
+        node = cg.get("tgt")
+        check("M3b content 追加命中（候选正文并入目标）",
+              node and "候选正文 水位信箱" in (node.get("content") or ""),
+              str(node)[:80])
+        neg = (node.get("frontmatter") or {}).get("non_applicable_conditions")
+        check("M3c 不适用条件并集（fm 旧条件保留，无新增时原样）",
+              neg == ["旧条件甲"], f"neg={neg}")
+
+        st = cg._pid_status().get(pid) or {}
+        check("M3d 裁决记录 accepted", st.get("status") == "accepted",
+              str(st)[:80])
+        casc = out.get("cascade_closed") or []
+        check("M3e 级联关闭同内容兄弟提案（手工入队形态）",
+              "prop_sib_manual" in casc, f"casc={casc}")
+
+        # M4 accept 对照
+        print("== M4 accept 对照 ==")
+        root4 = tempfile.mkdtemp(prefix="mdcg_merge_guard4_")
+        try:
+            build(root4)
+            cg4 = MdCGOS(root4)
+            pid4 = cg4.propose("acc", "# 功能名：接受候选\n# 正文：接受正文",
+                               layer="knowledge")
+            out4, n_scan4 = count_scans(
+                lambda: cg4.review_decide(pid4, "accept", reason="guard"))
+            check("M4a accept 裁决零全库扫描（对照不受影响）",
+                  n_scan4 == 0, f"scans={n_scan4}")
+            check("M4b accept 落节点 ok",
+                  out4.get("ok") and out4.get("node_id") == "acc", str(out4)[:80])
+            cg4.close()
+        finally:
+            shutil.rmtree(root4, ignore_errors=True)
+
+        # M5 增量账本：merge 库重开（新进程形态），目标条目可见且带新内容指纹
+        print("== M5 增量账本跨进程可见 ==")
+        cg.close()
+        cg5 = MdCGOS(root)
+        try:
+            e5 = (cg5.index.get("nodes") or {}).get("tgt")
+            node5 = cg5.get("tgt")
+            check("M5a 重开后目标条目在索引（增量日志，不靠 rebuild 救回）",
+                  bool(e5))
+            check("M5b 重开后目标内容为合并后版本",
+                  node5 and "候选正文 水位信箱" in (node5.get("content") or ""))
+        finally:
+            cg5.close()
+    finally:
+        shutil.rmtree(root, ignore_errors=True)
+
+    print("\n" + "=" * 60)
+    print(f"结果：PASS {PASS} / FAIL {FAIL}")
+    if FAILS:
+        for f in FAILS:
+            print(f"  - {f}")
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())