@furongjun1999/dsh-memory 0.9.9 → 0.9.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/md_cg/config_registry_bulk.py +484 -483
- package/md_cg/config_validate.py +457 -456
- package/md_cg/mcp_server.py +5324 -5324
- package/md_cg/mdcg.py +48 -1
- package/md_cg/mdcos.py +7163 -7159
- package/md_cg/test_c3_transient_read_negative.py +797 -797
- package/md_cg/test_exact_text_cache.py +194 -194
- package/md_cg/test_hotcache_tool.py +121 -121
- package/md_cg/test_index_snapshot_fastpath.py +72 -0
- package/md_cg/test_issue274_read_node_id_none.py +364 -364
- package/md_cg/test_p3_legacy_closure.py +441 -441
- package/md_cg/test_search_bm25_default.py +60 -60
- package/md_cg/test_untrusted_audit.py +55 -55
- package/md_cg/test_untrusted_classifier.py +42 -42
- package/md_cg/test_untrusted_enforcement.py +66 -66
- package/md_cg/untrusted.py +45 -45
- package/md_cg/untrusted_audit.py +87 -87
- package/package.json +1 -1
- package/src/hooks.ts +986 -986
|
@@ -1,194 +1,194 @@
|
|
|
1
|
-
"""Runtime guards for exact-match projection reuse and its invalidation."""
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import os
|
|
5
|
-
import re
|
|
6
|
-
import tempfile
|
|
7
|
-
import unittest
|
|
8
|
-
from unittest.mock import call, patch
|
|
9
|
-
|
|
10
|
-
from . import mdcg, nodefile, readcache
|
|
11
|
-
from .mdcos import MdCGOS
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def frozen_exact(q, content, **kw):
|
|
15
|
-
"""Pre-optimization predicate, independent of all new projection helpers."""
|
|
16
|
-
query = re.sub(r"\s+", "", q or "")
|
|
17
|
-
return int(len(query) >= 2 and query in re.sub(r"\s+", "", content or ""))
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
class ExactTextCache(unittest.TestCase):
|
|
21
|
-
def setUp(self):
|
|
22
|
-
clean = {k: v for k, v in os.environ.items() if not k.startswith("MDCG_")}
|
|
23
|
-
clean.update(MDCG_HOTCACHE="0", MDCG_READ_CACHE="1", MDCG_FRESHNESS="0",
|
|
24
|
-
MDCG_UNIFY_QUERY="0", MDCG_EN_ATOMS="0")
|
|
25
|
-
self.env = patch.dict(os.environ, clean, clear=True)
|
|
26
|
-
self.env.start()
|
|
27
|
-
self.addCleanup(self.env.stop)
|
|
28
|
-
|
|
29
|
-
def make_store(self, bodies=None):
|
|
30
|
-
temp = tempfile.TemporaryDirectory(prefix="exact_projection_")
|
|
31
|
-
self.addCleanup(temp.cleanup)
|
|
32
|
-
cg = MdCGOS(temp.name)
|
|
33
|
-
self.addCleanup(cg.close)
|
|
34
|
-
for nid, body in (bodies or {"a": "AB 与 BC 分开", "z": "A B\u2003C", "case": "abc"}).items():
|
|
35
|
-
cg.add(nid, body, layer="knowledge", importance=0.5)
|
|
36
|
-
cg.flush()
|
|
37
|
-
return cg
|
|
38
|
-
|
|
39
|
-
def docs(self, cg):
|
|
40
|
-
return [(entry, *cg._read(entry)) for entry in list(cg.index["nodes"].values())]
|
|
41
|
-
|
|
42
|
-
def score(self, cg, query):
|
|
43
|
-
return cg._score(self.docs(cg), query, mdcg.bigrams(query))
|
|
44
|
-
|
|
45
|
-
def test_raw_and_prepared_predicates_preserve_whitespace_and_case(self):
|
|
46
|
-
cases = [("ABC", "A B\u2003C", 1), ("A\tB C", "ABC", 1),
|
|
47
|
-
("ABC", "AB 与 BC 分开", 0), ("ABC", "abc", 0),
|
|
48
|
-
("甲\n乙", "甲\u00a0乙", 1), ("甲乙", "甲\u3000乙", 1),
|
|
49
|
-
("", "ABC", 0), (None, None, 0), ("A", "A", 0)]
|
|
50
|
-
for query, body, expected in cases:
|
|
51
|
-
with self.subTest(query=query, body=body):
|
|
52
|
-
prepared = (mdcg.compact_exact_text(query), mdcg.compact_exact_text(body))
|
|
53
|
-
self.assertEqual(mdcg.exact_hit_of(query, body), expected)
|
|
54
|
-
self.assertEqual(mdcg.exact_hit_of(query, body, _prepared=prepared), expected)
|
|
55
|
-
|
|
56
|
-
def test_production_results_and_scores_equal_frozen_predicate(self):
|
|
57
|
-
cg = self.make_store()
|
|
58
|
-
for mode in ("legacy", "jaccard"):
|
|
59
|
-
with patch.object(mdcg, "SCORE_MODE", mode):
|
|
60
|
-
for query in ("ABC", "A B C", "甲乙", "unmatched", "", "abc"):
|
|
61
|
-
with self.subTest(mode=mode, query=query):
|
|
62
|
-
cached_scores = self.score(cg, query)
|
|
63
|
-
new_search, _ = cg.search(query, k=3, judge=False, record=False)
|
|
64
|
-
new_rrf, _ = cg.search_rrf(query, k=3, paths=("lexical",), judge=False, record=False)
|
|
65
|
-
with patch.object(mdcg, "exact_hit_of", frozen_exact):
|
|
66
|
-
self.assertEqual(cached_scores, self.score(cg, query))
|
|
67
|
-
self.assertEqual(new_search, cg.search(query, k=3, judge=False, record=False)[0])
|
|
68
|
-
self.assertEqual(new_rrf, cg.search_rrf(query, k=3, paths=("lexical",), judge=False, record=False)[0])
|
|
69
|
-
|
|
70
|
-
def test_warm_scoring_normalizes_only_query_once(self):
|
|
71
|
-
cg = self.make_store()
|
|
72
|
-
self.score(cg, "ABC")
|
|
73
|
-
with patch.object(mdcg, "compact_exact_text", wraps=mdcg.compact_exact_text) as normalize:
|
|
74
|
-
self.score(cg, "A B C")
|
|
75
|
-
self.assertEqual(normalize.call_args_list, [call("A B C")])
|
|
76
|
-
|
|
77
|
-
def test_one_write_invalidates_only_changed_document(self):
|
|
78
|
-
cg = self.make_store()
|
|
79
|
-
self.score(cg, "ABC")
|
|
80
|
-
before = {path: record[1] for path, record in cg._exact_text_cache.items()}
|
|
81
|
-
cg.add("a", "A\tB\nC 新内容", layer="knowledge", importance=0.5)
|
|
82
|
-
cg.flush()
|
|
83
|
-
with patch.object(mdcg, "compact_exact_text", wraps=mdcg.compact_exact_text) as normalize:
|
|
84
|
-
rows = self.score(cg, "ABC")
|
|
85
|
-
self.assertEqual(normalize.call_count, 2) # one query, one changed document
|
|
86
|
-
a = next(node for node, _ in rows if node["id"] == "a")
|
|
87
|
-
self.assertEqual(a[mdcg.EXACT_HIT_KEY], 1)
|
|
88
|
-
for path, record in cg._exact_text_cache.items():
|
|
89
|
-
if path != cg.index["nodes"]["a"]["path"]:
|
|
90
|
-
self.assertIs(record[1], before[path])
|
|
91
|
-
|
|
92
|
-
def test_clear_removes_projection_without_changing_legacy_count(self):
|
|
93
|
-
cg = self.make_store()
|
|
94
|
-
cg.search_rrf("ABC", paths=("lexical",), judge=False, record=False)
|
|
95
|
-
expected = sum(len(getattr(cg, name, {})) for name in
|
|
96
|
-
("_read_cache", "_norm_bigrams_cache", "_positive_body_cache"))
|
|
97
|
-
self.assertTrue(cg._exact_text_cache)
|
|
98
|
-
self.assertTrue(cg._like_lower_cache)
|
|
99
|
-
self.assertEqual(readcache.clear(cg), expected)
|
|
100
|
-
self.assertFalse(cg._exact_text_cache)
|
|
101
|
-
self.assertFalse(cg._like_lower_cache)
|
|
102
|
-
self.assertEqual(readcache.clear(cg), 0)
|
|
103
|
-
|
|
104
|
-
def test_short_query_does_not_normalize_documents(self):
|
|
105
|
-
cg = self.make_store()
|
|
106
|
-
with patch.object(cg, "_doc_exact_text", side_effect=AssertionError("unnecessary projection")):
|
|
107
|
-
rows = self.score(cg, "A")
|
|
108
|
-
self.assertTrue(all(node[mdcg.EXACT_HIT_KEY] == 0 for node, _ in rows))
|
|
109
|
-
|
|
110
|
-
def test_opt_out_uses_uncached_projection(self):
|
|
111
|
-
with patch.dict(os.environ, {"MDCG_READ_CACHE": "0"}):
|
|
112
|
-
cg = self.make_store()
|
|
113
|
-
self.assertFalse(hasattr(cg, "_exact_text_cache"))
|
|
114
|
-
self.assertFalse(hasattr(cg, "_like_lower_cache"))
|
|
115
|
-
rows = self.score(cg, "ABC")
|
|
116
|
-
self.assertEqual({node["id"]: node[mdcg.EXACT_HIT_KEY] for node, _ in rows},
|
|
117
|
-
{"a": 0, "z": 1, "case": 0})
|
|
118
|
-
|
|
119
|
-
def test_instances_with_same_relative_path_are_isolated(self):
|
|
120
|
-
first = self.make_store({"same": "A B C"})
|
|
121
|
-
second = self.make_store({"same": "D E F"})
|
|
122
|
-
self.assertEqual(self.score(first, "ABC")[0][0][mdcg.EXACT_HIT_KEY], 1)
|
|
123
|
-
self.assertEqual(self.score(second, "ABC")[0][0][mdcg.EXACT_HIT_KEY], 0)
|
|
124
|
-
self.assertIsNot(first._exact_text_cache, second._exact_text_cache)
|
|
125
|
-
|
|
126
|
-
def test_reinstall_does_not_keep_hidden_projection_cache(self):
|
|
127
|
-
cg = self.make_store()
|
|
128
|
-
entry = next(iter(cg.index["nodes"].values()))
|
|
129
|
-
cg._doc_exact_text(entry, "A B C")
|
|
130
|
-
cg._like_body_lower(entry, "A B C")
|
|
131
|
-
cg._doc_norm_bigrams(entry, "A B C")
|
|
132
|
-
readcache.install(cg)
|
|
133
|
-
readcache.clear(cg)
|
|
134
|
-
self.assertEqual(cg._doc_exact_text(entry, "D E F"), "DEF")
|
|
135
|
-
self.assertEqual(cg._like_body_lower(entry, "DIFFERENT"), "different")
|
|
136
|
-
readcache.clear(cg)
|
|
137
|
-
self.assertEqual(cg._positive_body(entry, "NEW BODY"), "NEW BODY")
|
|
138
|
-
self.assertEqual(cg._doc_norm_bigrams(entry, "NEW BODY"),
|
|
139
|
-
mdcg.bigrams(mdcg.normalize_en("NEW BODY")))
|
|
140
|
-
|
|
141
|
-
def test_lowercase_projection_preserves_old_like_predicate(self):
|
|
142
|
-
cg = self.make_store({"same": "# 正文:İΣ Alpha 中文\n# 不适用条件:Forbidden"})
|
|
143
|
-
entry, fm, body = self.docs(cg)[0]
|
|
144
|
-
fm["tags"] = ["MixedTag"]
|
|
145
|
-
lower = nodefile.positive_body(body).lower()
|
|
146
|
-
for terms in ([], ["alpha"], ["Alpha"], ["i\u0307"], ["中文"],
|
|
147
|
-
["forbidden"], ["mixedtag"], ["absent"]):
|
|
148
|
-
with self.subTest(terms=terms):
|
|
149
|
-
expected = any(t in lower or t.lower() in "mixedtag" for t in terms)
|
|
150
|
-
self.assertEqual(cg._like(body, fm, terms), expected)
|
|
151
|
-
self.assertEqual(cg._like(body, fm, terms, entry=entry,
|
|
152
|
-
body_lower=cg._like_body_lower), expected)
|
|
153
|
-
self.assertTrue(cg._like(body, fm, [], index_keys={"hit": True},
|
|
154
|
-
body_lower=lambda *_: self.fail("index hit should return early")))
|
|
155
|
-
|
|
156
|
-
def test_lowercase_warm_reuse_and_single_write_invalidation(self):
|
|
157
|
-
cg = self.make_store({"a": "Alpha OLD", "b": "Alpha OTHER"})
|
|
158
|
-
cg.search_rrf("alpha", paths=("lexical",), judge=False, record=False)
|
|
159
|
-
# BM25 并入默认检索后,首次 search 会为 bm25 索引投影一次正文(之后走缓存)。
|
|
160
|
-
# 故先**预热一次** search,再断言「热态下不再重复投影」——意图不变(热复用零投影)。
|
|
161
|
-
cg.search("alpha", judge=False, record=False)
|
|
162
|
-
before = dict(cg._like_lower_cache)
|
|
163
|
-
with patch.object(cg, "_positive_body", wraps=cg._positive_body) as project:
|
|
164
|
-
cg.search("alpha", judge=False, record=False)
|
|
165
|
-
self.assertEqual(project.call_count, 0)
|
|
166
|
-
cg.add("a", "Alpha NEW", layer="knowledge", importance=0.5)
|
|
167
|
-
cg.flush()
|
|
168
|
-
hits, _ = cg.search_rrf("alpha new", paths=("lexical",), judge=False, record=False)
|
|
169
|
-
self.assertEqual(project.call_count, 1)
|
|
170
|
-
self.assertIn("a", [hit[0]["id"] for hit in hits])
|
|
171
|
-
other = cg.index["nodes"]["b"]["path"]
|
|
172
|
-
self.assertIs(cg._like_lower_cache[other][1], before[other][1])
|
|
173
|
-
changed = cg.index["nodes"]["a"]["path"]
|
|
174
|
-
self.assertIn("new", cg._like_lower_cache[changed][1])
|
|
175
|
-
self.assertNotIn("old", cg._like_lower_cache[changed][1])
|
|
176
|
-
|
|
177
|
-
def test_rebuild_after_external_write_invalidates_both_projections(self):
|
|
178
|
-
cg = self.make_store({"same": "A B C Alpha"})
|
|
179
|
-
cg.search_rrf("alpha", paths=("lexical",), judge=False, record=False)
|
|
180
|
-
entry = cg.index["nodes"]["same"]
|
|
181
|
-
path = os.path.join(cg.root, entry["path"])
|
|
182
|
-
with open(path, encoding="utf-8") as stream:
|
|
183
|
-
original = stream.read()
|
|
184
|
-
with open(path, "w", encoding="utf-8") as stream:
|
|
185
|
-
stream.write(original.replace("A B C Alpha", "D E F Beta"))
|
|
186
|
-
cg.rebuild_index()
|
|
187
|
-
hits, _ = cg.search_rrf("beta", paths=("lexical",), judge=False, record=False)
|
|
188
|
-
self.assertIn("same", [hit[0]["id"] for hit in hits])
|
|
189
|
-
self.assertIn("DEF", cg._doc_exact_text(entry, cg._read(entry)[1]))
|
|
190
|
-
self.assertNotIn("ABC", cg._doc_exact_text(entry, cg._read(entry)[1]))
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
if __name__ == "__main__":
|
|
194
|
-
unittest.main()
|
|
1
|
+
"""Runtime guards for exact-match projection reuse and its invalidation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import tempfile
|
|
7
|
+
import unittest
|
|
8
|
+
from unittest.mock import call, patch
|
|
9
|
+
|
|
10
|
+
from . import mdcg, nodefile, readcache
|
|
11
|
+
from .mdcos import MdCGOS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def frozen_exact(q, content, **kw):
|
|
15
|
+
"""Pre-optimization predicate, independent of all new projection helpers."""
|
|
16
|
+
query = re.sub(r"\s+", "", q or "")
|
|
17
|
+
return int(len(query) >= 2 and query in re.sub(r"\s+", "", content or ""))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ExactTextCache(unittest.TestCase):
|
|
21
|
+
def setUp(self):
|
|
22
|
+
clean = {k: v for k, v in os.environ.items() if not k.startswith("MDCG_")}
|
|
23
|
+
clean.update(MDCG_HOTCACHE="0", MDCG_READ_CACHE="1", MDCG_FRESHNESS="0",
|
|
24
|
+
MDCG_UNIFY_QUERY="0", MDCG_EN_ATOMS="0")
|
|
25
|
+
self.env = patch.dict(os.environ, clean, clear=True)
|
|
26
|
+
self.env.start()
|
|
27
|
+
self.addCleanup(self.env.stop)
|
|
28
|
+
|
|
29
|
+
def make_store(self, bodies=None):
|
|
30
|
+
temp = tempfile.TemporaryDirectory(prefix="exact_projection_")
|
|
31
|
+
self.addCleanup(temp.cleanup)
|
|
32
|
+
cg = MdCGOS(temp.name)
|
|
33
|
+
self.addCleanup(cg.close)
|
|
34
|
+
for nid, body in (bodies or {"a": "AB 与 BC 分开", "z": "A B\u2003C", "case": "abc"}).items():
|
|
35
|
+
cg.add(nid, body, layer="knowledge", importance=0.5)
|
|
36
|
+
cg.flush()
|
|
37
|
+
return cg
|
|
38
|
+
|
|
39
|
+
def docs(self, cg):
|
|
40
|
+
return [(entry, *cg._read(entry)) for entry in list(cg.index["nodes"].values())]
|
|
41
|
+
|
|
42
|
+
def score(self, cg, query):
|
|
43
|
+
return cg._score(self.docs(cg), query, mdcg.bigrams(query))
|
|
44
|
+
|
|
45
|
+
def test_raw_and_prepared_predicates_preserve_whitespace_and_case(self):
|
|
46
|
+
cases = [("ABC", "A B\u2003C", 1), ("A\tB C", "ABC", 1),
|
|
47
|
+
("ABC", "AB 与 BC 分开", 0), ("ABC", "abc", 0),
|
|
48
|
+
("甲\n乙", "甲\u00a0乙", 1), ("甲乙", "甲\u3000乙", 1),
|
|
49
|
+
("", "ABC", 0), (None, None, 0), ("A", "A", 0)]
|
|
50
|
+
for query, body, expected in cases:
|
|
51
|
+
with self.subTest(query=query, body=body):
|
|
52
|
+
prepared = (mdcg.compact_exact_text(query), mdcg.compact_exact_text(body))
|
|
53
|
+
self.assertEqual(mdcg.exact_hit_of(query, body), expected)
|
|
54
|
+
self.assertEqual(mdcg.exact_hit_of(query, body, _prepared=prepared), expected)
|
|
55
|
+
|
|
56
|
+
def test_production_results_and_scores_equal_frozen_predicate(self):
|
|
57
|
+
cg = self.make_store()
|
|
58
|
+
for mode in ("legacy", "jaccard"):
|
|
59
|
+
with patch.object(mdcg, "SCORE_MODE", mode):
|
|
60
|
+
for query in ("ABC", "A B C", "甲乙", "unmatched", "", "abc"):
|
|
61
|
+
with self.subTest(mode=mode, query=query):
|
|
62
|
+
cached_scores = self.score(cg, query)
|
|
63
|
+
new_search, _ = cg.search(query, k=3, judge=False, record=False)
|
|
64
|
+
new_rrf, _ = cg.search_rrf(query, k=3, paths=("lexical",), judge=False, record=False)
|
|
65
|
+
with patch.object(mdcg, "exact_hit_of", frozen_exact):
|
|
66
|
+
self.assertEqual(cached_scores, self.score(cg, query))
|
|
67
|
+
self.assertEqual(new_search, cg.search(query, k=3, judge=False, record=False)[0])
|
|
68
|
+
self.assertEqual(new_rrf, cg.search_rrf(query, k=3, paths=("lexical",), judge=False, record=False)[0])
|
|
69
|
+
|
|
70
|
+
def test_warm_scoring_normalizes_only_query_once(self):
|
|
71
|
+
cg = self.make_store()
|
|
72
|
+
self.score(cg, "ABC")
|
|
73
|
+
with patch.object(mdcg, "compact_exact_text", wraps=mdcg.compact_exact_text) as normalize:
|
|
74
|
+
self.score(cg, "A B C")
|
|
75
|
+
self.assertEqual(normalize.call_args_list, [call("A B C")])
|
|
76
|
+
|
|
77
|
+
def test_one_write_invalidates_only_changed_document(self):
|
|
78
|
+
cg = self.make_store()
|
|
79
|
+
self.score(cg, "ABC")
|
|
80
|
+
before = {path: record[1] for path, record in cg._exact_text_cache.items()}
|
|
81
|
+
cg.add("a", "A\tB\nC 新内容", layer="knowledge", importance=0.5)
|
|
82
|
+
cg.flush()
|
|
83
|
+
with patch.object(mdcg, "compact_exact_text", wraps=mdcg.compact_exact_text) as normalize:
|
|
84
|
+
rows = self.score(cg, "ABC")
|
|
85
|
+
self.assertEqual(normalize.call_count, 2) # one query, one changed document
|
|
86
|
+
a = next(node for node, _ in rows if node["id"] == "a")
|
|
87
|
+
self.assertEqual(a[mdcg.EXACT_HIT_KEY], 1)
|
|
88
|
+
for path, record in cg._exact_text_cache.items():
|
|
89
|
+
if path != cg.index["nodes"]["a"]["path"]:
|
|
90
|
+
self.assertIs(record[1], before[path])
|
|
91
|
+
|
|
92
|
+
def test_clear_removes_projection_without_changing_legacy_count(self):
|
|
93
|
+
cg = self.make_store()
|
|
94
|
+
cg.search_rrf("ABC", paths=("lexical",), judge=False, record=False)
|
|
95
|
+
expected = sum(len(getattr(cg, name, {})) for name in
|
|
96
|
+
("_read_cache", "_norm_bigrams_cache", "_positive_body_cache"))
|
|
97
|
+
self.assertTrue(cg._exact_text_cache)
|
|
98
|
+
self.assertTrue(cg._like_lower_cache)
|
|
99
|
+
self.assertEqual(readcache.clear(cg), expected)
|
|
100
|
+
self.assertFalse(cg._exact_text_cache)
|
|
101
|
+
self.assertFalse(cg._like_lower_cache)
|
|
102
|
+
self.assertEqual(readcache.clear(cg), 0)
|
|
103
|
+
|
|
104
|
+
def test_short_query_does_not_normalize_documents(self):
|
|
105
|
+
cg = self.make_store()
|
|
106
|
+
with patch.object(cg, "_doc_exact_text", side_effect=AssertionError("unnecessary projection")):
|
|
107
|
+
rows = self.score(cg, "A")
|
|
108
|
+
self.assertTrue(all(node[mdcg.EXACT_HIT_KEY] == 0 for node, _ in rows))
|
|
109
|
+
|
|
110
|
+
def test_opt_out_uses_uncached_projection(self):
|
|
111
|
+
with patch.dict(os.environ, {"MDCG_READ_CACHE": "0"}):
|
|
112
|
+
cg = self.make_store()
|
|
113
|
+
self.assertFalse(hasattr(cg, "_exact_text_cache"))
|
|
114
|
+
self.assertFalse(hasattr(cg, "_like_lower_cache"))
|
|
115
|
+
rows = self.score(cg, "ABC")
|
|
116
|
+
self.assertEqual({node["id"]: node[mdcg.EXACT_HIT_KEY] for node, _ in rows},
|
|
117
|
+
{"a": 0, "z": 1, "case": 0})
|
|
118
|
+
|
|
119
|
+
def test_instances_with_same_relative_path_are_isolated(self):
|
|
120
|
+
first = self.make_store({"same": "A B C"})
|
|
121
|
+
second = self.make_store({"same": "D E F"})
|
|
122
|
+
self.assertEqual(self.score(first, "ABC")[0][0][mdcg.EXACT_HIT_KEY], 1)
|
|
123
|
+
self.assertEqual(self.score(second, "ABC")[0][0][mdcg.EXACT_HIT_KEY], 0)
|
|
124
|
+
self.assertIsNot(first._exact_text_cache, second._exact_text_cache)
|
|
125
|
+
|
|
126
|
+
def test_reinstall_does_not_keep_hidden_projection_cache(self):
|
|
127
|
+
cg = self.make_store()
|
|
128
|
+
entry = next(iter(cg.index["nodes"].values()))
|
|
129
|
+
cg._doc_exact_text(entry, "A B C")
|
|
130
|
+
cg._like_body_lower(entry, "A B C")
|
|
131
|
+
cg._doc_norm_bigrams(entry, "A B C")
|
|
132
|
+
readcache.install(cg)
|
|
133
|
+
readcache.clear(cg)
|
|
134
|
+
self.assertEqual(cg._doc_exact_text(entry, "D E F"), "DEF")
|
|
135
|
+
self.assertEqual(cg._like_body_lower(entry, "DIFFERENT"), "different")
|
|
136
|
+
readcache.clear(cg)
|
|
137
|
+
self.assertEqual(cg._positive_body(entry, "NEW BODY"), "NEW BODY")
|
|
138
|
+
self.assertEqual(cg._doc_norm_bigrams(entry, "NEW BODY"),
|
|
139
|
+
mdcg.bigrams(mdcg.normalize_en("NEW BODY")))
|
|
140
|
+
|
|
141
|
+
def test_lowercase_projection_preserves_old_like_predicate(self):
|
|
142
|
+
cg = self.make_store({"same": "# 正文:İΣ Alpha 中文\n# 不适用条件:Forbidden"})
|
|
143
|
+
entry, fm, body = self.docs(cg)[0]
|
|
144
|
+
fm["tags"] = ["MixedTag"]
|
|
145
|
+
lower = nodefile.positive_body(body).lower()
|
|
146
|
+
for terms in ([], ["alpha"], ["Alpha"], ["i\u0307"], ["中文"],
|
|
147
|
+
["forbidden"], ["mixedtag"], ["absent"]):
|
|
148
|
+
with self.subTest(terms=terms):
|
|
149
|
+
expected = any(t in lower or t.lower() in "mixedtag" for t in terms)
|
|
150
|
+
self.assertEqual(cg._like(body, fm, terms), expected)
|
|
151
|
+
self.assertEqual(cg._like(body, fm, terms, entry=entry,
|
|
152
|
+
body_lower=cg._like_body_lower), expected)
|
|
153
|
+
self.assertTrue(cg._like(body, fm, [], index_keys={"hit": True},
|
|
154
|
+
body_lower=lambda *_: self.fail("index hit should return early")))
|
|
155
|
+
|
|
156
|
+
def test_lowercase_warm_reuse_and_single_write_invalidation(self):
|
|
157
|
+
cg = self.make_store({"a": "Alpha OLD", "b": "Alpha OTHER"})
|
|
158
|
+
cg.search_rrf("alpha", paths=("lexical",), judge=False, record=False)
|
|
159
|
+
# BM25 并入默认检索后,首次 search 会为 bm25 索引投影一次正文(之后走缓存)。
|
|
160
|
+
# 故先**预热一次** search,再断言「热态下不再重复投影」——意图不变(热复用零投影)。
|
|
161
|
+
cg.search("alpha", judge=False, record=False)
|
|
162
|
+
before = dict(cg._like_lower_cache)
|
|
163
|
+
with patch.object(cg, "_positive_body", wraps=cg._positive_body) as project:
|
|
164
|
+
cg.search("alpha", judge=False, record=False)
|
|
165
|
+
self.assertEqual(project.call_count, 0)
|
|
166
|
+
cg.add("a", "Alpha NEW", layer="knowledge", importance=0.5)
|
|
167
|
+
cg.flush()
|
|
168
|
+
hits, _ = cg.search_rrf("alpha new", paths=("lexical",), judge=False, record=False)
|
|
169
|
+
self.assertEqual(project.call_count, 1)
|
|
170
|
+
self.assertIn("a", [hit[0]["id"] for hit in hits])
|
|
171
|
+
other = cg.index["nodes"]["b"]["path"]
|
|
172
|
+
self.assertIs(cg._like_lower_cache[other][1], before[other][1])
|
|
173
|
+
changed = cg.index["nodes"]["a"]["path"]
|
|
174
|
+
self.assertIn("new", cg._like_lower_cache[changed][1])
|
|
175
|
+
self.assertNotIn("old", cg._like_lower_cache[changed][1])
|
|
176
|
+
|
|
177
|
+
def test_rebuild_after_external_write_invalidates_both_projections(self):
|
|
178
|
+
cg = self.make_store({"same": "A B C Alpha"})
|
|
179
|
+
cg.search_rrf("alpha", paths=("lexical",), judge=False, record=False)
|
|
180
|
+
entry = cg.index["nodes"]["same"]
|
|
181
|
+
path = os.path.join(cg.root, entry["path"])
|
|
182
|
+
with open(path, encoding="utf-8") as stream:
|
|
183
|
+
original = stream.read()
|
|
184
|
+
with open(path, "w", encoding="utf-8") as stream:
|
|
185
|
+
stream.write(original.replace("A B C Alpha", "D E F Beta"))
|
|
186
|
+
cg.rebuild_index()
|
|
187
|
+
hits, _ = cg.search_rrf("beta", paths=("lexical",), judge=False, record=False)
|
|
188
|
+
self.assertIn("same", [hit[0]["id"] for hit in hits])
|
|
189
|
+
self.assertIn("DEF", cg._doc_exact_text(entry, cg._read(entry)[1]))
|
|
190
|
+
self.assertNotIn("ABC", cg._doc_exact_text(entry, cg._read(entry)[1]))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
if __name__ == "__main__":
|
|
194
|
+
unittest.main()
|
|
@@ -1,121 +1,121 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""工具面热缓存守卫(issue #614)。
|
|
3
|
-
|
|
4
|
-
C1 第二次同参命中且结果逐条相同
|
|
5
|
-
C2 写后读不陈旧
|
|
6
|
-
C3 换 k 不串味
|
|
7
|
-
C4 环境开关翻动不命中
|
|
8
|
-
C5 显式 node_id 绕过缓存
|
|
9
|
-
C6 值形状标签:命中体必须是本面写入的(surface/v 校验),异形一律当未命中
|
|
10
|
-
C7 命中返回体是深拷贝:调用方原地修改不得污染缓存
|
|
11
|
-
C8 validity=True 一律不缓存(无时钟窗语义)
|
|
12
|
-
"""
|
|
13
|
-
from __future__ import annotations
|
|
14
|
-
|
|
15
|
-
import os
|
|
16
|
-
import tempfile
|
|
17
|
-
|
|
18
|
-
from .mdcos import MdCGSecure
|
|
19
|
-
from .mcp_server import call_tool
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def main():
|
|
23
|
-
os.environ["MDCG_HOTCACHE"] = "1"
|
|
24
|
-
ok = 0
|
|
25
|
-
bad = 0
|
|
26
|
-
|
|
27
|
-
def chk(cond, label):
|
|
28
|
-
nonlocal ok, bad
|
|
29
|
-
if cond:
|
|
30
|
-
ok += 1
|
|
31
|
-
print(" PASS", label)
|
|
32
|
-
else:
|
|
33
|
-
bad += 1
|
|
34
|
-
print(" FAIL", label)
|
|
35
|
-
|
|
36
|
-
root = tempfile.mkdtemp(prefix="hc_tool2_")
|
|
37
|
-
cg = MdCGSecure(root)
|
|
38
|
-
for i in range(6):
|
|
39
|
-
cg.add("n%d" % i, "灵枢 记忆库 条目 %d alpha" % i, layer="knowledge")
|
|
40
|
-
|
|
41
|
-
r1 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
42
|
-
hc = getattr(cg, "_hotcache_tool", None)
|
|
43
|
-
chk(hc is not None, "C0 工具层用独立缓存实例")
|
|
44
|
-
s1 = hc.stats()
|
|
45
|
-
r2 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
46
|
-
s2 = hc.stats()
|
|
47
|
-
chk(s1["query_misses"] >= 1 and s2["query_hits"] >= 1, "C1 命中(misses=%d hits=%d)" % (s1["query_misses"], s2["query_hits"]))
|
|
48
|
-
chk(r1.get("results") == r2.get("results"), "C1b 命中结果逐条相同")
|
|
49
|
-
|
|
50
|
-
cg.add("n_new", "阿洛芬 独有词", layer="knowledge")
|
|
51
|
-
r3 = call_tool(cg, "mdcg_search", {"query": "阿洛芬", "k": 5})
|
|
52
|
-
chk("n_new" in [x["node"]["id"] for x in (r3.get("results") or [])], "C2 写后读不陈旧")
|
|
53
|
-
|
|
54
|
-
before = hc.stats()["query_hits"]
|
|
55
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 3})
|
|
56
|
-
chk(hc.stats()["query_hits"] == before, "C3 换 k 不串味")
|
|
57
|
-
|
|
58
|
-
os.environ["MDCG_SEMANTIC"] = "0"
|
|
59
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
60
|
-
b4 = hc.stats()["query_hits"]
|
|
61
|
-
os.environ["MDCG_SEMANTIC"] = "1"
|
|
62
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
63
|
-
chk(hc.stats()["query_hits"] == b4, "C4 开关翻动后不命中")
|
|
64
|
-
os.environ.pop("MDCG_SEMANTIC", None)
|
|
65
|
-
|
|
66
|
-
b5 = hc.stats()["query_hits"]
|
|
67
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "node_id": None})
|
|
68
|
-
chk(hc.stats()["query_hits"] == b5, "C5 显式 node_id 绕过")
|
|
69
|
-
|
|
70
|
-
# C6:往同一实例塞一个异形值(模拟内部层写进来的形状),必须当未命中而不是返回垃圾
|
|
71
|
-
call_tool(cg, "mdcg_search", {"query": "形状探针", "k": 5})
|
|
72
|
-
_q = "形状探针"
|
|
73
|
-
try:
|
|
74
|
-
_keys = list(hc._queries.keys())
|
|
75
|
-
_k0 = [k for k in _keys if k[0] == _q]
|
|
76
|
-
if _k0:
|
|
77
|
-
hc._queries[_k0[0]] = (hc._queries[_k0[0]][0], [("row", 0.1, {}, "p")], {})
|
|
78
|
-
except Exception:
|
|
79
|
-
pass
|
|
80
|
-
r6 = call_tool(cg, "mdcg_search", {"query": _q, "k": 5})
|
|
81
|
-
chk(isinstance(r6, dict) and "results" in r6, "C6 异形缓存值被当未命中(不返回垃圾)")
|
|
82
|
-
|
|
83
|
-
# C7:命中体是深拷贝——改它不能污染缓存
|
|
84
|
-
_r7a = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
85
|
-
_r7a.setdefault("results", []).append({"node": {"id": "污染"}, "score": 99})
|
|
86
|
-
_r7b = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
87
|
-
chk("污染" not in [x["node"]["id"] for x in (_r7b.get("results") or [])], "C7 命中体深拷贝(原地修改不污染)")
|
|
88
|
-
|
|
89
|
-
# C8:validity=True 不缓存
|
|
90
|
-
b8 = hc.stats()["query_hits"]
|
|
91
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
|
|
92
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
|
|
93
|
-
chk(hc.stats()["query_hits"] == b8, "C8 validity=True 不缓存")
|
|
94
|
-
|
|
95
|
-
# C9 安全:切换调用者身份(cg.session)后不得命中前者缓存——
|
|
96
|
-
# 检索结果按 session/密级过滤,缓存键必须含有效身份,否则跨会话/跨密级泄漏。
|
|
97
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
98
|
-
b9 = hc.stats()["query_hits"]
|
|
99
|
-
_old_sess = getattr(cg, "session", None)
|
|
100
|
-
cg.session = "another-session"
|
|
101
|
-
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
102
|
-
chk(hc.stats()["query_hits"] == b9, "C9 换会话身份后不命中(防跨会话/跨密级泄漏)")
|
|
103
|
-
cg.session = _old_sess
|
|
104
|
-
|
|
105
|
-
# C10 安全:索引突变(写入/遗忘都经 _on_index_mutation)必须清空工具层缓存,
|
|
106
|
-
# 否则遗忘后 TTL 内还能被同参 query 读回正文。
|
|
107
|
-
# 注:本用例直接调突变单点——forget 需管理权限,测试 principal 没有(AccessDenied),
|
|
108
|
-
# 故验证「突变即失效」这条接线本身,而不是绕开权限去删节点。
|
|
109
|
-
cg.add("n_del", "波塞冬 独有词 待删除", layer="knowledge")
|
|
110
|
-
_r = call_tool(cg, "mdcg_search", {"query": "波塞冬", "k": 5})
|
|
111
|
-
chk(len(getattr(cg, "_hotcache_tool")._queries) >= 1, "C10a 查询已进工具层缓存")
|
|
112
|
-
cg._on_index_mutation("n_del")
|
|
113
|
-
chk(len(getattr(cg, "_hotcache_tool")._queries) == 0, "C10b 索引突变即清空工具层缓存(删除承诺不被缓存吃掉)")
|
|
114
|
-
|
|
115
|
-
print("=" * 60)
|
|
116
|
-
print("工具面热缓存守卫:%d 通过 / %d 失败" % (ok, bad))
|
|
117
|
-
return 0 if bad == 0 else 1
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if __name__ == "__main__":
|
|
121
|
-
raise SystemExit(main())
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""工具面热缓存守卫(issue #614)。
|
|
3
|
+
|
|
4
|
+
C1 第二次同参命中且结果逐条相同
|
|
5
|
+
C2 写后读不陈旧
|
|
6
|
+
C3 换 k 不串味
|
|
7
|
+
C4 环境开关翻动不命中
|
|
8
|
+
C5 显式 node_id 绕过缓存
|
|
9
|
+
C6 值形状标签:命中体必须是本面写入的(surface/v 校验),异形一律当未命中
|
|
10
|
+
C7 命中返回体是深拷贝:调用方原地修改不得污染缓存
|
|
11
|
+
C8 validity=True 一律不缓存(无时钟窗语义)
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import tempfile
|
|
17
|
+
|
|
18
|
+
from .mdcos import MdCGSecure
|
|
19
|
+
from .mcp_server import call_tool
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main():
|
|
23
|
+
os.environ["MDCG_HOTCACHE"] = "1"
|
|
24
|
+
ok = 0
|
|
25
|
+
bad = 0
|
|
26
|
+
|
|
27
|
+
def chk(cond, label):
|
|
28
|
+
nonlocal ok, bad
|
|
29
|
+
if cond:
|
|
30
|
+
ok += 1
|
|
31
|
+
print(" PASS", label)
|
|
32
|
+
else:
|
|
33
|
+
bad += 1
|
|
34
|
+
print(" FAIL", label)
|
|
35
|
+
|
|
36
|
+
root = tempfile.mkdtemp(prefix="hc_tool2_")
|
|
37
|
+
cg = MdCGSecure(root)
|
|
38
|
+
for i in range(6):
|
|
39
|
+
cg.add("n%d" % i, "灵枢 记忆库 条目 %d alpha" % i, layer="knowledge")
|
|
40
|
+
|
|
41
|
+
r1 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
42
|
+
hc = getattr(cg, "_hotcache_tool", None)
|
|
43
|
+
chk(hc is not None, "C0 工具层用独立缓存实例")
|
|
44
|
+
s1 = hc.stats()
|
|
45
|
+
r2 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
46
|
+
s2 = hc.stats()
|
|
47
|
+
chk(s1["query_misses"] >= 1 and s2["query_hits"] >= 1, "C1 命中(misses=%d hits=%d)" % (s1["query_misses"], s2["query_hits"]))
|
|
48
|
+
chk(r1.get("results") == r2.get("results"), "C1b 命中结果逐条相同")
|
|
49
|
+
|
|
50
|
+
cg.add("n_new", "阿洛芬 独有词", layer="knowledge")
|
|
51
|
+
r3 = call_tool(cg, "mdcg_search", {"query": "阿洛芬", "k": 5})
|
|
52
|
+
chk("n_new" in [x["node"]["id"] for x in (r3.get("results") or [])], "C2 写后读不陈旧")
|
|
53
|
+
|
|
54
|
+
before = hc.stats()["query_hits"]
|
|
55
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 3})
|
|
56
|
+
chk(hc.stats()["query_hits"] == before, "C3 换 k 不串味")
|
|
57
|
+
|
|
58
|
+
os.environ["MDCG_SEMANTIC"] = "0"
|
|
59
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
60
|
+
b4 = hc.stats()["query_hits"]
|
|
61
|
+
os.environ["MDCG_SEMANTIC"] = "1"
|
|
62
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
63
|
+
chk(hc.stats()["query_hits"] == b4, "C4 开关翻动后不命中")
|
|
64
|
+
os.environ.pop("MDCG_SEMANTIC", None)
|
|
65
|
+
|
|
66
|
+
b5 = hc.stats()["query_hits"]
|
|
67
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "node_id": None})
|
|
68
|
+
chk(hc.stats()["query_hits"] == b5, "C5 显式 node_id 绕过")
|
|
69
|
+
|
|
70
|
+
# C6:往同一实例塞一个异形值(模拟内部层写进来的形状),必须当未命中而不是返回垃圾
|
|
71
|
+
call_tool(cg, "mdcg_search", {"query": "形状探针", "k": 5})
|
|
72
|
+
_q = "形状探针"
|
|
73
|
+
try:
|
|
74
|
+
_keys = list(hc._queries.keys())
|
|
75
|
+
_k0 = [k for k in _keys if k[0] == _q]
|
|
76
|
+
if _k0:
|
|
77
|
+
hc._queries[_k0[0]] = (hc._queries[_k0[0]][0], [("row", 0.1, {}, "p")], {})
|
|
78
|
+
except Exception:
|
|
79
|
+
pass
|
|
80
|
+
r6 = call_tool(cg, "mdcg_search", {"query": _q, "k": 5})
|
|
81
|
+
chk(isinstance(r6, dict) and "results" in r6, "C6 异形缓存值被当未命中(不返回垃圾)")
|
|
82
|
+
|
|
83
|
+
# C7:命中体是深拷贝——改它不能污染缓存
|
|
84
|
+
_r7a = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
85
|
+
_r7a.setdefault("results", []).append({"node": {"id": "污染"}, "score": 99})
|
|
86
|
+
_r7b = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
87
|
+
chk("污染" not in [x["node"]["id"] for x in (_r7b.get("results") or [])], "C7 命中体深拷贝(原地修改不污染)")
|
|
88
|
+
|
|
89
|
+
# C8:validity=True 不缓存
|
|
90
|
+
b8 = hc.stats()["query_hits"]
|
|
91
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
|
|
92
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
|
|
93
|
+
chk(hc.stats()["query_hits"] == b8, "C8 validity=True 不缓存")
|
|
94
|
+
|
|
95
|
+
# C9 安全:切换调用者身份(cg.session)后不得命中前者缓存——
|
|
96
|
+
# 检索结果按 session/密级过滤,缓存键必须含有效身份,否则跨会话/跨密级泄漏。
|
|
97
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
98
|
+
b9 = hc.stats()["query_hits"]
|
|
99
|
+
_old_sess = getattr(cg, "session", None)
|
|
100
|
+
cg.session = "another-session"
|
|
101
|
+
call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
|
|
102
|
+
chk(hc.stats()["query_hits"] == b9, "C9 换会话身份后不命中(防跨会话/跨密级泄漏)")
|
|
103
|
+
cg.session = _old_sess
|
|
104
|
+
|
|
105
|
+
# C10 安全:索引突变(写入/遗忘都经 _on_index_mutation)必须清空工具层缓存,
|
|
106
|
+
# 否则遗忘后 TTL 内还能被同参 query 读回正文。
|
|
107
|
+
# 注:本用例直接调突变单点——forget 需管理权限,测试 principal 没有(AccessDenied),
|
|
108
|
+
# 故验证「突变即失效」这条接线本身,而不是绕开权限去删节点。
|
|
109
|
+
cg.add("n_del", "波塞冬 独有词 待删除", layer="knowledge")
|
|
110
|
+
_r = call_tool(cg, "mdcg_search", {"query": "波塞冬", "k": 5})
|
|
111
|
+
chk(len(getattr(cg, "_hotcache_tool")._queries) >= 1, "C10a 查询已进工具层缓存")
|
|
112
|
+
cg._on_index_mutation("n_del")
|
|
113
|
+
chk(len(getattr(cg, "_hotcache_tool")._queries) == 0, "C10b 索引突变即清空工具层缓存(删除承诺不被缓存吃掉)")
|
|
114
|
+
|
|
115
|
+
print("=" * 60)
|
|
116
|
+
print("工具面热缓存守卫:%d 通过 / %d 失败" % (ok, bad))
|
|
117
|
+
return 0 if bad == 0 else 1
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
if __name__ == "__main__":
|
|
121
|
+
raise SystemExit(main())
|