@furongjun1999/dsh-memory 0.4.11 → 0.5.1

package/md_cg/test_rrf_graph_seed_cache.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""search_rrf 图路种子读缓存守卫（v9:92 / v13 留档复测·修复，第 14 轮）。
+
+缺陷形态（缺陷报告实测，40 带 tags/edges 节点临时库，逐位复现）：
+  _path_graph 种子腿 mdcos.py `node = self.get(n["id"])` 在边循环前执行——
+  get 裸 open+nodefile.loads（mdcg.py get 不在 readcache 包装面，install 只包
+  cg._read），search_rrf 默认 paths 含 graph → 每查询固定 ≤5 次盘读
+  （Q1 loads=45=词法 40+种子 5；Q2 增量恰 5、get 恒 5/查；graph 零产出也
+  在边循环前发生）。边目标（:1017）早已走 self._read 同形。
+
+修复口径：种子读改走 self._read(by_id[...])（读缓存覆盖，Q2 增量归零）；
+种子 fm.id≠文件名等 by_id 不含形态回落 get 保旧口径（冷路径，正确性优先）。
+种子来自 lexical（entries/_candidates 派生，MdCGSecure._candidates 已过
+_readable）——绕过 get 的读隔离门无损失；图路扩展产物本就由
+MdCGSecure.search_rrf 终态二次过滤兜底。
+
+本守卫（调用计数断言形态，非时间阈值）：
+  S1 热缓存后同查询 loads 增量 == 0（旧实现恒 5 → 红）；
+  S2 热缓存后同查询 get 调用 == 0（旧实现恒 5 → 红）；
+  S3 图路功能在位：词法零命中的「幽灵」节点经种子边一跳进入结果
+     （graph 路不是被静默关掉）；
+  S4 两轮查询结果 ids 逐位一致（确定性）；meta.paths 含四路计数；
+  S5 开/关读缓存（MDCG_READ_CACHE=1 vs 0）同库同查询 ids 逐位一致
+     （缓存只改速度不改结果）。
+
+运行：python -m md_cg.test_rrf_graph_seed_cache
+"""
+import os
+import shutil
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from md_cg import nodefile                     # noqa: E402
+from md_cg.mdcos import MdCGOS                 # noqa: E402
+
+PASS = 0
+FAIL = 0
+FAILS = []
+
+QUERY = "蜂群调度"
+
+
+def check(name, cond, detail=""):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print(f"  OK   {name}")
+    else:
+        FAIL += 1
+        FAILS.append(name)
+        print(f"  FAIL {name}  {detail}")
+
+
+def build_lib(root, n=40):
+    cg = MdCGOS(root)
+    for i in range(n):
+        edges = []
+        if i % 4 == 0:                 # 部分节点带边（指向池内目标）
+            edges = [{"target": "mem_%03d" % ((i + 1) % n),
+                      "relation_type": "related_to"}]
+        if i == 0:
+            # 幽灵目标：词法零命中（正文无查询词），只经种子边一跳可达
+            edges.append({"target": "ghost_x", "relation_type": "related_to"})
+        cg.add("mem_%03d" % i,
+               "# 功能名：图路样本 %d\n# 正文：%s 节点 %d" % (i, QUERY, i),
+               layer="knowledge",
+               tags=["tag_%d" % (i % 5)] if i % 3 == 0 else None,
+               edges=edges)
+    cg.add("ghost_x", "# 功能名：幽灵目标\n# 正文：完全不相关的孤岛内容",
+           layer="knowledge")
+    cg.flush()
+    return cg
+
+
+def counters(cg, fn):
+    """同时计数 nodefile.loads 与实例 get。"""
+    st = {"loads": 0, "gets": 0}
+    real_loads = nodefile.loads
+    real_get = MdCGOS.get
+
+    def _spy_loads(*a, **kw):
+        st["loads"] += 1
+        return real_loads(*a, **kw)
+
+    def _spy_get(self, node_id):
+        st["gets"] += 1
+        return real_get(self, node_id)
+
+    nodefile.loads = _spy_loads
+    cg.get = _spy_get.__get__(cg, MdCGOS)
+    try:
+        return fn(), st
+    finally:
+        nodefile.loads = real_loads
+        del cg.get
+
+
+def main():
+    root = tempfile.mkdtemp(prefix="mdcg_rrf_seed_")
+    cg = None
+    try:
+        os.environ.pop("MDCG_READ_CACHE", None)
+        cg = build_lib(root)
+        (r1, m1), c1 = counters(cg, lambda: cg.search_rrf(QUERY))
+        ids1 = [x[0].get("id") for x in r1]
+        check("S0a 首查正常（40+1 节点池，词法+图路融合）",
+              len(r1) >= 1 and c1["loads"] > 0,
+              f"n={len(r1)} loads={c1['loads']}")
+
+        (r2, m2), c2 = counters(cg, lambda: cg.search_rrf(QUERY))
+        ids2 = [x[0].get("id") for x in r2]
+        check("S1a 热缓存后同查询 loads 增量 == 0（旧实现恒 5）",
+              c2["loads"] == 0, f"loads={c2['loads']}")
+        check("S2a 热缓存后同查询 get 调用 == 0（旧实现恒 5，种子改走 _read）",
+              c2["gets"] == 0, f"gets={c2['gets']}")
+        check("S3a 图路功能在位：词法零命中的 ghost 经种子边一跳进入结果",
+              "ghost_x" in ids1, f"ghost in={('ghost_x' in ids1)}")
+        paths_meta = (m2.get("paths") or {})
+        check("S3b meta.paths 四路计数在位（graph 未被静默关掉）",
+              {"lexical", "graph"} <= set(paths_meta),
+              f"paths={paths_meta}")
+        check("S4a 两轮查询结果 ids 逐位一致", ids1 == ids2,
+              f"n={len(ids1)}/{len(ids2)}")
+
+        # S5 开/关读缓存结果一致（新实例同库同查询）
+        r_on = ids2
+        os.environ["MDCG_READ_CACHE"] = "0"
+        try:
+            cg0 = MdCGOS(root)
+            r_off, _ = cg0.search_rrf(QUERY)
+            cg0.close()
+            check("S5a 读缓存开/关同库同查询 ids 逐位一致",
+                  [x[0].get("id") for x in r_off] == r_on,
+                  f"off={[x[0].get('id') for x in r_off][:5]}")
+        finally:
+            os.environ.pop("MDCG_READ_CACHE", None)
+        cg.close()
+    finally:
+        os.environ.pop("MDCG_READ_CACHE", None)
+        shutil.rmtree(root, ignore_errors=True)
+
+    print("\n" + "=" * 60)
+    print(f"结果：PASS {PASS} / FAIL {FAIL}")
+    if FAILS:
+        for f in FAILS:
+            print(f"  - {f}")
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/md_cg/test_security_audit.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""安全审计修复守卫（批次 24，外部审查报告 P0-1/P1-1/P2-1/P1-8）。
+
+P0-1：node_id 是模型可控输入、直接拼落盘路径——`..` 穿越出 root、
+Windows 绝对路径（os.path.join 丢弃前缀）= 任意 .md 覆盖。修复 = 白名单
++ realpath 纵深闸。守卫断言（能红修复前实现）：
+  - 穿越/绝对路径/超长/空 id 必 ValueError；
+  - 合法 id（存量形态）照常写入。
+P1-1：mdcg_* 工具此前完全绕过 ops_allow 白名单——只读令牌可经
+mdcg_remember 写入。守卫断言（能红修复前实现）：
+  - ops_allow=["read"] 的 principal 调 mdcg_remember 必 AccessDenied；
+  - ops_allow=None（不限）令牌照常放行（兼容性）；
+  - 读工具在 read 令牌下放行。
+P1-8：tarfile filter="data"（静态断言 + import 回归）。
+"""
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+PASS = 0
+FAIL = 0
+FAILS = []
+
+
+def check(name, cond, detail=""):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print(f"  OK   {name}")
+    else:
+        FAIL += 1
+        FAILS.append(name)
+        print(f"  FAIL {name}  {detail}")
+
+
+def main():
+    from md_cg.mdcg import _NODE_ID_RE
+    from md_cg.mdcos import MdCGSecure
+    from md_cg.mcp_server import _dispatch, _MDCG_OP_REQUIRE
+
+    root = tempfile.mkdtemp(prefix="mdcg_secaudit_")
+    # 生产形态（MdCGSecure——mdcg_search 分支走 OS 层签名，基类 MdCG 无
+    # roles 参数；P1-1c 因此必须用安全实例而非基类）
+    cg = MdCGSecure(root)
+
+    print("== P0-1 node_id 白名单 + realpath 纵深闸 ==")
+    bad_ids = [
+        (".. 穿越", "../evil"),
+        ("多层穿越", "sub/../../../evil"),
+        ("Windows 绝对路径", "C:/Users/test/evil"),
+        ("类 Unix 绝对路径", "/etc/evil"),
+        ("隐式 ..", "a/../b..c"),
+        ("空 id", ""),
+        ("超长 id", "a" * 129),
+    ]
+    for label, nid in bad_ids:
+        try:
+            cg.add(nid, "# 功能名：x\n# 正文：y", layer="knowledge")
+            check(f"P0-1 拒绝 {label}", False, "未抛异常且写入成功")
+        except ValueError:
+            check(f"P0-1 拒绝 {label}", True)
+        except Exception as e:  # noqa: BLE001
+            check(f"P0-1 拒绝 {label}", False, f"异常类型错误: {type(e).__name__}: {e}")
+
+    ok_ids = ["mem_1790156428627", "n00001", "mde_abcdef123456",
+              "uuid-4f2b-9c.a-d", "a" * 128]
+    ok = True
+    for i, nid in enumerate(ok_ids):
+        try:
+            cg.add(nid, f"# 功能名：合法 {i}\n# 正文：白名单内存活样本 {i}",
+                   layer="knowledge")
+        except Exception as e:  # noqa: BLE001
+            ok = False
+            check(f"P0-1 合法 id {nid[:16]}… 照常写入", False,
+                  f"{type(e).__name__}: {e}")
+    check("P0-1 合法 id（存量 5 形态）照常写入", ok and _NODE_ID_RE.match("mem_1"))
+
+    print("== P1-1 mdcg_* op 闸（能红无闸实现）==")
+    from md_cg.security import Principal
+
+    def principal_with(ops):
+        return Principal(actor="auditor", clearance="secret",
+                         can_write=True, role="recorder", auth_mode="test",
+                         ops_allow=ops)
+
+    # 只读令牌：mdcg_remember 必拒（能红：修复前无闸直接写入）
+    cg.principal = principal_with(["read"])
+    try:
+        _dispatch(cg, "mdcg_remember", {"content": "# 功能名：越权\n# 正文：x",
+                                        "layer": "contextual"})
+        check("P1-1a 只读令牌 mdcg_remember 必拒", False, "未抛 AccessDenied")
+    except Exception as e:  # noqa: BLE001
+        check("P1-1a 只读令牌 mdcg_remember 必拒（AccessDenied）",
+              type(e).__name__ == "AccessDenied",
+              f"{type(e).__name__}: {e}")
+
+    # ops_allow=None（不限）令牌：放行（兼容性——存量直构形态零影响；
+    # _in_scope(None)=True 是「未声明=不限制」的既有语义）
+    saved = cg.principal
+    cg.principal = principal_with(None)
+    try:
+        res = _dispatch(cg, "mdcg_remember",
+                        {"content": "# 功能名：兼容\n# 正文：不限令牌照常",
+                         "layer": "contextual"})
+        check("P1-1b ops_allow=None（不限）令牌照常放行", True)
+    except Exception as e:  # noqa: BLE001
+        check("P1-1b ops_allow=None（不限）令牌照常放行", False,
+              f"{type(e).__name__}: {e}")
+    finally:
+        cg.principal = saved
+
+    # 读令牌：读工具放行
+    cg.principal = principal_with(["read"])
+    try:
+        res = _dispatch(cg, "mdcg_search", {"query": "星云"})
+        check("P1-1c 读令牌读工具放行", not isinstance(res, dict) or not res.get("error"),
+              str(res)[:120])
+    except Exception as e:  # noqa: BLE001
+        check("P1-1c 读令牌读工具放行", False, f"{type(e).__name__}: {e}")
+    cg.principal = None
+
+    # 映射完备性：mdcg_* 全部注册工具都被映射覆盖（防未来新增工具漏闸）
+    from md_cg import mcp_server as ms
+    import re as _re
+    src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                            "mcp_server.py"), encoding="utf-8").read()
+    registered = set(_re.findall(r'"name": "(mdcg_[a-z_]+)"', src))
+    uncovered = sorted(registered - set(_MDCG_OP_REQUIRE) - {"mdcg_service_info"})
+    check("P1-1d mdcg_* 注册工具全覆盖（除自描述 service_info）",
+          not uncovered, f"uncovered={uncovered}")
+
+    print("== P1-8 tarfile filter ==")
+    src_cf = open(os.path.join(os.path.dirname(os.path.dirname(
+        os.path.abspath(__file__))), "scripts", "criteria_fingerprint.py"),
+        encoding="utf-8").read()
+    check("P1-8 extractall 显式 filter=\"data\"", 'extractall(tmp, filter="data")'
+          in src_cf)
+
+    # 清理
+    import shutil
+    shutil.rmtree(root, ignore_errors=True)
+
+    print("\n" + "=" * 60)
+    print(f"结果：PASS {PASS} / FAIL {FAIL}")
+    if FAILS:
+        for f in FAILS:
+            print(f"  - {f}")
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/md_cg/test_security_audit_b26.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""安全审计修复守卫 B（批次 26，外部审查报告 P1-2/3/4/5/6/7）。
+
+- P1-4：check_path_root 根白名单（env 未设=放开 / 设置=越界即拒）
+- P1-5：LEGACY env 身份不再自授 admin（需显式二次开关）
+- P1-7：bootstrap_loop AST 沙箱（合法排序函数过 / 恶意样本全拒）
+- P1-6 增量：hive read_file 敏感凭据路径拒读
+- P1-2/P1-3：TS 侧（roleplay_web）静态断言 + tsc 编译由 gate 覆盖
+"""
+import importlib.util
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+PASS = 0
+FAIL = 0
+FAILS = []
+
+
+def check(name, cond, detail=""):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+        print(f"  OK   {name}")
+    else:
+        FAIL += 1
+        FAILS.append(name)
+        print(f"  FAIL {name}  {detail}")
+
+
+def main():
+    from md_cg.security import check_path_root
+
+    print("== P1-4 check_path_root 根白名单 ==")
+    tmp = tempfile.mkdtemp(prefix="b26_root_")
+    os.environ["MDCG_INGEST_ROOT"] = tmp
+    try:
+        check_path_root(os.path.join(tmp, "a.jsonl"), "MDCG_INGEST_ROOT", "t")
+        check("P1-4a 根内路径放行", True)
+        try:
+            check_path_root(os.path.join(tmp, "..", "evil.jsonl"),
+                            "MDCG_INGEST_ROOT", "t")
+            check("P1-4b .. 穿越出根拒绝", False, "未抛")
+        except PermissionError:
+            check("P1-4b .. 穿越出根拒绝", True)
+        try:
+            check_path_root("C:/Windows/win.ini", "MDCG_INGEST_ROOT", "t")
+            check("P1-4c 根外绝对路径拒绝", False, "未抛")
+        except PermissionError:
+            check("P1-4c 根外绝对路径拒绝", True)
+        check_path_root(None, "MDCG_INGEST_ROOT", "t")
+        check("P1-4d 无路径参数（stat 类）放行", True)
+    finally:
+        os.environ.pop("MDCG_INGEST_ROOT", None)
+    check_path_root("C:/Windows/win.ini", "MDCG_INGEST_ROOT", "t")
+    check("P1-4e env 未设置=放开（默认部署零变更）", True)
+
+    print("== P1-5 LEGACY env 身份 admin 二次开关 ==")
+    saved = {k: os.environ.get(k) for k in
+             ("MDCG_TOKEN", "MDCG_LEGACY_ENV_AUTH", "MDCG_CAN_ADMIN",
+              "MDCG_LEGACY_ENV_ADMIN", "MDCG_CLEARANCE")}
+    try:
+        os.environ["MDCG_TOKEN"] = ""
+        os.environ["MDCG_LEGACY_ENV_AUTH"] = "1"
+        os.environ["MDCG_CAN_ADMIN"] = "1"
+        os.environ.pop("MDCG_LEGACY_ENV_ADMIN", None)
+        from md_cg.mcp_server import _build_principal
+        p, err = _build_principal()
+        check("P1-5a 只设 CAN_ADMIN：can_admin 强制 False",
+              p is not None and not p.can_admin, f"p={p}")
+        check("P1-5b 降级 recorder", p is not None and p.role == "recorder",
+              f"role={getattr(p, 'role', None)}")
+        os.environ["MDCG_LEGACY_ENV_ADMIN"] = "1"
+        p2, _ = _build_principal()
+        check("P1-5c 二次开关后 admin 恢复",
+              p2 is not None and p2.can_admin and p2.role == "designer",
+              f"p2={p2}")
+    finally:
+        for k, v in saved.items():
+            if v is None:
+                os.environ.pop(k, None)
+            else:
+                os.environ[k] = v
+
+    print("== P1-7 bootstrap_loop AST 沙箱 ==")
+    spec = importlib.util.spec_from_file_location(
+        "bl_b26", os.path.join(REPO, "scripts", "bootstrap_loop.py"))
+    m = importlib.util.module_from_spec(spec)
+    try:
+        spec.loader.exec_module(m)
+    except SystemExit:
+        pass
+    f = m._safe_exec_gen
+    ns = f("def solve(arr):\n"
+           "    n = len(arr)\n"
+           "    for i in range(n):\n"
+           "        for j in range(0, n - i - 1):\n"
+           "            if arr[j] > arr[j + 1]:\n"
+           "                arr[j], arr[j + 1] = arr[j + 1], arr[j]\n"
+           "    return arr\n")
+    check("P1-7a 合法排序函数通过且正确", ns["solve"]([3, 1, 2]) == [1, 2, 3])
+    bans = [
+        "import os", "from subprocess import run", 'open("x")',
+        'eval("1")', 'exec("x=1")', '__import__("os")',
+        "x.__class__", "getattr(int, 'real')",
+    ]
+    n_blocked = 0
+    for c in bans:
+        try:
+            f("def f():\n    return " + c if not c.startswith(("import", "from"))
+              else c)
+        except ValueError:
+            n_blocked += 1
+        except Exception:               # noqa: BLE001——其它异常也算被拦
+            n_blocked += 1
+    check(f"P1-7b 恶意样本全拒（{n_blocked}/{len(bans)}）",
+          n_blocked == len(bans))
+
+    print("== P1-6 增量：hive read_file 敏感路径拒读 ==")
+    spec2 = importlib.util.spec_from_file_location(
+        "hx_b26", os.path.join(REPO, "hive", "exec.py"))
+    hx = importlib.util.module_from_spec(spec2)
+    try:
+        spec2.loader.exec_module(hx)
+    except SystemExit:
+        pass
+    home = os.path.expanduser("~").replace("\\", "/")
+    check("P1-6a ~/.ssh/id_rsa 拒",
+          hx._sensitive_read(os.path.join(home, ".ssh", "id_rsa")) is not None)
+    check("P1-6b .env 拒",
+          hx._sensitive_read(os.path.join(home, "proj", ".env")) is not None)
+    check("P1-6c server.pem 拒",
+          hx._sensitive_read(os.path.join(home, "certs", "server.pem"))
+          is not None)
+    check("P1-6d 普通 md 放行",
+          hx._sensitive_read(os.path.join(home, "notes", "readme.md")) is None)
+    check("P1-6e 源码 .py 放行",
+          hx._sensitive_read(os.path.join(home, "proj", "main.py")) is None)
+
+    print("== P1-2/P1-3 roleplay_web 静态断言（行为由 tsc+集成覆盖）==")
+    ts = open(os.path.join(REPO, "src", "lib", "roleplay_web.ts"),
+              encoding="utf-8").read()
+    check("P1-2 fail-closed（未配置不再 return true）",
+          "if (!EDIT_KEY)\n                return false;" in ts
+          and "editKeyHint" in ts)
+    check("P1-3 角色白名单+危险键拒绝",
+          "validRoleId" in ts and "'__proto__'" in ts and "toNullProto" in ts)
+
+    print("\n" + "=" * 60)
+    print(f"结果：PASS {PASS} / FAIL {FAIL}")
+    for x in FAILS:
+        print(f"  - {x}")
+    return 0 if FAIL == 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())