dsh-biocore 0.3.3 → 0.5.0

package/python/biocore_model/tests/test_memory_inject.py

@@ -1,211 +1,211 @@
-# -*- coding: utf-8 -*-
-"""task-221 (batch 2): query recall injection + embedding stats persistence."""
-import json
-import os
-import tempfile
-import unittest
-from pathlib import Path
-
-from biocore_memory.recall import embedding
-from biocore_runtime import pipeline
-
-
-def entry(i, text, ts, importance=0.5):
-    return {"id": i, "text": text, "role": "user", "ts": ts,
-            "importance": importance, "hits": 1, "scope": "default",
-            "memory_confidence": 0.7, "memory_source": "direct"}
-
-
-class InjectCase(unittest.TestCase):
-    def setUp(self):
-        self.tmp = tempfile.TemporaryDirectory()
-        self.root = Path(self.tmp.name)
-        self.store = self.root / "data" / "memory"
-        self.entries = [
-            entry("new", "今天买了咖啡", 1700000300.0),
-            entry("old", "上次说的苹果很甜", 1700000100.0),
-            entry("mid", "明天要去公园", 1700000200.0),
-        ]
-        (self.store / "sessions" / "s1").mkdir(parents=True)
-        (self.store / "sessions" / "s1" / "memory.json").write_text(
-            json.dumps({"entries": self.entries}), encoding="utf-8")
-
-    def tearDown(self):
-        self.tmp.cleanup()
-
-    def inject(self, **kwargs):
-        return pipeline.inject_block("s1", str(self.store), **kwargs)
-
-
-class QueryModeTest(InjectCase):
-    def test_query_recall_orders_by_query(self):
-        out = self.inject(query="苹果", mode="query", top_k=3)
-        self.assertEqual(out["ids"][0], "old")
-
-    def test_recent_mode_keeps_newest_first(self):
-        out = self.inject(query="苹果", mode="recent", top_k=3)
-        self.assertEqual(out["ids"][0], "new")
-
-    def test_default_mode_is_query(self):
-        out = self.inject(query="苹果", top_k=2)
-        self.assertEqual(out["ids"][0], "old")
-        self.assertEqual(len(out["ids"]), 2)
-
-    def test_empty_query_falls_back_recent(self):
-        out = self.inject(query="", mode="query", top_k=3)
-        self.assertEqual(out["ids"][0], "new")
-
-    def test_engine_failure_falls_back_recent(self):
-        saved = pipeline._inject_engine
-        pipeline._inject_engine = lambda store, entries: (_ for _ in ()).throw(RuntimeError("boom"))
-        try:
-            out = self.inject(query="苹果", mode="query", top_k=2)
-        finally:
-            pipeline._inject_engine = saved
-        self.assertEqual(out["ids"][0], "new")
-        self.assertEqual(len(out["ids"]), 2)
-
-    def test_output_format_and_budget(self):
-        out = self.inject(query="苹果", mode="query", top_k=3, max_chars=60)
-        self.assertTrue(out["text"].startswith("你记得这些事："))
-        self.assertTrue(all(line.startswith("- ") for line in out["text"].split("\n")[1:]))
-        self.assertLessEqual(out["chars"], 60)
-        self.assertEqual(out["chars"], len(out["text"]))
-        self.assertTrue(set(out["ids"]).issubset({"new", "old", "mid"}))
-
-
-class RerankTest(InjectCase):
-    def test_rerank_reorders_when_enabled(self):
-        class FakeRerank:
-            def __init__(self):
-                self.calls = 0
-
-            def rerank_documents(self, query, documents, config):
-                self.calls += 1
-                return [{"index": 2}, {"index": 0}, {"index": 1}]
-
-        fake = FakeRerank()
-        saved = pipeline._rerank_mod
-        pipeline._rerank_mod = fake
-        try:
-            out = self.inject(query="苹果", mode="query", top_k=8,
-                              rerank_config={"rerank_enabled": True})
-        finally:
-            pipeline._rerank_mod = saved
-        self.assertEqual(fake.calls, 1)
-        # engine put 苹果 (old) first; rerank's order must win instead
-        self.assertEqual(out["ids"], ["mid", "old", "new"])
-
-    def test_rerank_off_does_not_call(self):
-        class Exploding:
-            def rerank_documents(self, *args, **kwargs):
-                raise AssertionError("rerank must not run")
-
-        saved = pipeline._rerank_mod
-        pipeline._rerank_mod = Exploding()
-        try:
-            out = self.inject(query="苹果", mode="query", top_k=8,
-                              rerank_config={"rerank_enabled": False})
-        finally:
-            pipeline._rerank_mod = saved
-        self.assertEqual(out["ids"][0], "old")
-
-
-class EmbeddingFallbackTest(InjectCase):
-    def test_embedding_provider_failure_degrades_to_lexical(self):
-        class BadProvider:
-            def embed_text(self, text):
-                raise RuntimeError("no network")
-
-        from biocore_memory.engine import MemoryEngine
-        from biocore_memory.store.json_store import MemoryStore
-
-        def engine_with_bad_embedder(store, entries):
-            memory = MemoryStore(doc={"entries": [dict(item) for item in entries]})
-            return MemoryEngine(store=memory, embedder=BadProvider(),
-                                config={"flags": {}})
-
-        saved = pipeline._inject_engine
-        pipeline._inject_engine = engine_with_bad_embedder
-        try:
-            out = self.inject(query="苹果", mode="query", top_k=3)
-        finally:
-            pipeline._inject_engine = saved
-        self.assertEqual(out["ids"][0], "old")
-
-    def test_learning_flags_default_on(self):
-        captured = {}
-
-        class CaptureEngine:
-            def __init__(self, **kwargs):
-                captured.update(kwargs)
-
-            def recall(self, query, budget=5):
-                return []
-
-        saved = pipeline._inject_engine
-        saved_engine = None
-        try:
-            from biocore_memory.engine import MemoryEngine
-            saved_engine = MemoryEngine
-            import biocore_memory.engine as engine_module
-            engine_module.MemoryEngine = CaptureEngine
-            try:
-                pipeline.inject_block("s1", str(self.store), query="苹果")
-            finally:
-                engine_module.MemoryEngine = saved_engine
-        finally:
-            pipeline._inject_engine = saved
-        flags = (captured.get("config") or {}).get("flags") or {}
-        for key in ("mood_recall", "affect_encoding",
-                    "memory_feedback", "preference_drift"):
-            self.assertIs(flags.get(key), True)
-
-
-class EmbedStatsTest(unittest.TestCase):
-    def setUp(self):
-        self.tmp = tempfile.TemporaryDirectory()
-        self.dir = Path(self.tmp.name)
-        self.stats_file = self.dir / "embed_stats.json"
-        self.cache_dir = self.dir / "cache"
-        self._env = {k: os.environ.get(k) for k in
-                     ("BIOCORE_EMBED_STATS_FILE", "BIOCORE_EMBEDDING_CACHE_FILE")}
-        os.environ["BIOCORE_EMBED_STATS_FILE"] = str(self.stats_file)
-        os.environ["BIOCORE_EMBEDDING_CACHE_FILE"] = str(self.cache_dir)
-        embedding.embed_stats(reset=True)
-
-    def tearDown(self):
-        for key, value in self._env.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
-        self.tmp.cleanup()
-
-    def test_miss_remote_then_disk_hit(self):
-        class Provider:
-            def embed_text(self, text):
-                return [0.1] * 8
-
-        self.assertIsNotNone(embedding.embed("hello world", Provider()))
-        first = json.loads(self.stats_file.read_text(encoding="utf-8"))
-        self.assertEqual(first["miss"], 1)
-        self.assertEqual(first["remote"], 1)
-        self.assertEqual(first["disk_hit"], 0)
-        embedding.embed("hello world", Provider())
-        second = json.loads(self.stats_file.read_text(encoding="utf-8"))
-        self.assertEqual(second["disk_hit"], 1)
-
-    def test_stats_action_reads_persisted_file(self):
-        self.stats_file.write_text(json.dumps(
-            {"hit": 2, "disk_hit": 3, "miss": 4, "remote": 5,
-             "local": 6, "fail": 7}), encoding="utf-8")
-        store = self.dir / "memory"
-        out = pipeline.dispatch("stats", {"store": str(store)})
-        self.assertEqual(out["embed"]["disk_hit"], 3)
-        self.assertEqual(out["embed"]["miss"], 4)
-        self.assertEqual(out["embed"]["remote"], 5)
-
-
-if __name__ == "__main__":
-    unittest.main()
+# -*- coding: utf-8 -*-
+"""task-221 (batch 2): query recall injection + embedding stats persistence."""
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from biocore_memory.recall import embedding
+from biocore_runtime import pipeline
+
+
+def entry(i, text, ts, importance=0.5):
+    return {"id": i, "text": text, "role": "user", "ts": ts,
+            "importance": importance, "hits": 1, "scope": "default",
+            "memory_confidence": 0.7, "memory_source": "direct"}
+
+
+class InjectCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self.tmp.name)
+        self.store = self.root / "data" / "memory"
+        self.entries = [
+            entry("new", "今天买了咖啡", 1700000300.0),
+            entry("old", "上次说的苹果很甜", 1700000100.0),
+            entry("mid", "明天要去公园", 1700000200.0),
+        ]
+        (self.store / "sessions" / "s1").mkdir(parents=True)
+        (self.store / "sessions" / "s1" / "memory.json").write_text(
+            json.dumps({"entries": self.entries}), encoding="utf-8")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def inject(self, **kwargs):
+        return pipeline.inject_block("s1", str(self.store), **kwargs)
+
+
+class QueryModeTest(InjectCase):
+    def test_query_recall_orders_by_query(self):
+        out = self.inject(query="苹果", mode="query", top_k=3)
+        self.assertEqual(out["ids"][0], "old")
+
+    def test_recent_mode_keeps_newest_first(self):
+        out = self.inject(query="苹果", mode="recent", top_k=3)
+        self.assertEqual(out["ids"][0], "new")
+
+    def test_default_mode_is_query(self):
+        out = self.inject(query="苹果", top_k=2)
+        self.assertEqual(out["ids"][0], "old")
+        self.assertEqual(len(out["ids"]), 2)
+
+    def test_empty_query_falls_back_recent(self):
+        out = self.inject(query="", mode="query", top_k=3)
+        self.assertEqual(out["ids"][0], "new")
+
+    def test_engine_failure_falls_back_recent(self):
+        saved = pipeline._inject_engine
+        pipeline._inject_engine = lambda store, entries: (_ for _ in ()).throw(RuntimeError("boom"))
+        try:
+            out = self.inject(query="苹果", mode="query", top_k=2)
+        finally:
+            pipeline._inject_engine = saved
+        self.assertEqual(out["ids"][0], "new")
+        self.assertEqual(len(out["ids"]), 2)
+
+    def test_output_format_and_budget(self):
+        out = self.inject(query="苹果", mode="query", top_k=3, max_chars=60)
+        self.assertTrue(out["text"].startswith("你记得这些事："))
+        self.assertTrue(all(line.startswith("- ") for line in out["text"].split("\n")[1:]))
+        self.assertLessEqual(out["chars"], 60)
+        self.assertEqual(out["chars"], len(out["text"]))
+        self.assertTrue(set(out["ids"]).issubset({"new", "old", "mid"}))
+
+
+class RerankTest(InjectCase):
+    def test_rerank_reorders_when_enabled(self):
+        class FakeRerank:
+            def __init__(self):
+                self.calls = 0
+
+            def rerank_documents(self, query, documents, config):
+                self.calls += 1
+                return [{"index": 2}, {"index": 0}, {"index": 1}]
+
+        fake = FakeRerank()
+        saved = pipeline._rerank_mod
+        pipeline._rerank_mod = fake
+        try:
+            out = self.inject(query="苹果", mode="query", top_k=8,
+                              rerank_config={"rerank_enabled": True})
+        finally:
+            pipeline._rerank_mod = saved
+        self.assertEqual(fake.calls, 1)
+        # engine put 苹果 (old) first; rerank's order must win instead
+        self.assertEqual(out["ids"], ["mid", "old", "new"])
+
+    def test_rerank_off_does_not_call(self):
+        class Exploding:
+            def rerank_documents(self, *args, **kwargs):
+                raise AssertionError("rerank must not run")
+
+        saved = pipeline._rerank_mod
+        pipeline._rerank_mod = Exploding()
+        try:
+            out = self.inject(query="苹果", mode="query", top_k=8,
+                              rerank_config={"rerank_enabled": False})
+        finally:
+            pipeline._rerank_mod = saved
+        self.assertEqual(out["ids"][0], "old")
+
+
+class EmbeddingFallbackTest(InjectCase):
+    def test_embedding_provider_failure_degrades_to_lexical(self):
+        class BadProvider:
+            def embed_text(self, text):
+                raise RuntimeError("no network")
+
+        from biocore_memory.engine import MemoryEngine
+        from biocore_memory.store.json_store import MemoryStore
+
+        def engine_with_bad_embedder(store, entries):
+            memory = MemoryStore(doc={"entries": [dict(item) for item in entries]})
+            return MemoryEngine(store=memory, embedder=BadProvider(),
+                                config={"flags": {}})
+
+        saved = pipeline._inject_engine
+        pipeline._inject_engine = engine_with_bad_embedder
+        try:
+            out = self.inject(query="苹果", mode="query", top_k=3)
+        finally:
+            pipeline._inject_engine = saved
+        self.assertEqual(out["ids"][0], "old")
+
+    def test_learning_flags_default_on(self):
+        captured = {}
+
+        class CaptureEngine:
+            def __init__(self, **kwargs):
+                captured.update(kwargs)
+
+            def recall(self, query, budget=5):
+                return []
+
+        saved = pipeline._inject_engine
+        saved_engine = None
+        try:
+            from biocore_memory.engine import MemoryEngine
+            saved_engine = MemoryEngine
+            import biocore_memory.engine as engine_module
+            engine_module.MemoryEngine = CaptureEngine
+            try:
+                pipeline.inject_block("s1", str(self.store), query="苹果")
+            finally:
+                engine_module.MemoryEngine = saved_engine
+        finally:
+            pipeline._inject_engine = saved
+        flags = (captured.get("config") or {}).get("flags") or {}
+        for key in ("mood_recall", "affect_encoding",
+                    "memory_feedback", "preference_drift"):
+            self.assertIs(flags.get(key), True)
+
+
+class EmbedStatsTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.dir = Path(self.tmp.name)
+        self.stats_file = self.dir / "embed_stats.json"
+        self.cache_dir = self.dir / "cache"
+        self._env = {k: os.environ.get(k) for k in
+                     ("BIOCORE_EMBED_STATS_FILE", "BIOCORE_EMBEDDING_CACHE_FILE")}
+        os.environ["BIOCORE_EMBED_STATS_FILE"] = str(self.stats_file)
+        os.environ["BIOCORE_EMBEDDING_CACHE_FILE"] = str(self.cache_dir)
+        embedding.embed_stats(reset=True)
+
+    def tearDown(self):
+        for key, value in self._env.items():
+            if value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = value
+        self.tmp.cleanup()
+
+    def test_miss_remote_then_disk_hit(self):
+        class Provider:
+            def embed_text(self, text):
+                return [0.1] * 8
+
+        self.assertIsNotNone(embedding.embed("hello world", Provider()))
+        first = json.loads(self.stats_file.read_text(encoding="utf-8"))
+        self.assertEqual(first["miss"], 1)
+        self.assertEqual(first["remote"], 1)
+        self.assertEqual(first["disk_hit"], 0)
+        embedding.embed("hello world", Provider())
+        second = json.loads(self.stats_file.read_text(encoding="utf-8"))
+        self.assertEqual(second["disk_hit"], 1)
+
+    def test_stats_action_reads_persisted_file(self):
+        self.stats_file.write_text(json.dumps(
+            {"hit": 2, "disk_hit": 3, "miss": 4, "remote": 5,
+             "local": 6, "fail": 7}), encoding="utf-8")
+        store = self.dir / "memory"
+        out = pipeline.dispatch("stats", {"store": str(store)})
+        self.assertEqual(out["embed"]["disk_hit"], 3)
+        self.assertEqual(out["embed"]["miss"], 4)
+        self.assertEqual(out["embed"]["remote"], 5)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/python/biocore_model/tests/test_retrieval_plus.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+"""task-62: normalized channels, chunks, shards, predictive exploration."""
+import json
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+from biocore_runtime import pipeline, retrieval_plus as rp
+
+
+def entry(i, text, ts=None):
+    return {"id": i, "text": text, "role": "user",
+            "ts": ts if ts is not None else time.time()}
+
+
+class Case(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.data_dir = Path(self.tmp.name) / "data"
+        self.store = self.data_dir / "memory"
+        (self.store / "sessions" / "s1").mkdir(parents=True)
+        self.now = time.time()
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+
+class NormalizeTest(Case):
+    def test_min_max(self):
+        self.assertEqual(rp.normalize([0.0, 5.0, 10.0]), [0.0, 0.5, 1.0])
+
+    def test_flat_positive_is_one(self):
+        self.assertEqual(rp.normalize([2.0, 2.0]), [1.0, 1.0])
+
+    def test_flat_zero_is_zero(self):
+        self.assertEqual(rp.normalize([0.0, 0.0]), [0.0, 0.0])
+
+    def test_channels_are_independent(self):
+        rows = [entry("a", "苹果 苹果 苹果"), entry("b", "香蕉")]
+        ranked = rp.rank(self.store, "s1", "苹果", rows, now=self.now)
+        by_id = {item["entry"]["id"]: item for item in ranked}
+        self.assertEqual(by_id["a"]["channels"]["bm25"], 1.0)
+        self.assertEqual(by_id["b"]["channels"]["bm25"], 0.0)
+        self.assertIn("vector", by_id["a"]["channels"])
+
+
+class ChunkTest(Case):
+    def test_sentences_split(self):
+        chunks = rp.split_chunks("第一句。第二句！第三句？")
+        self.assertEqual(len(chunks), 3)
+
+    def test_index_persists(self):
+        rp.build_chunks(self.store, [entry("a", "第一句。第二句。")])
+        doc = json.loads((self.data_dir / "retrieval" / "chunks.json")
+                         .read_text(encoding="utf-8"))
+        self.assertIn("a", doc["entries"])
+        self.assertEqual(len(doc["entries"]["a"]["chunks"]), 2)
+
+    def test_best_chunk_snippet(self):
+        rows = [entry("a", "今天很累。苹果很甜，想再买一次。")]
+        ranked = rp.rank(self.store, "s1", "苹果", rows, now=self.now)
+        self.assertIn("苹果", ranked[0]["chunk"])
+        self.assertGreaterEqual(ranked[0]["chunk_hits"], 1)
+
+
+class ShardTest(Case):
+    def test_session_shard(self):
+        shards = rp.assign_shards(self.store, [entry("a", "x")], session_id="s1")
+        self.assertEqual(shards["a"], "s-s1")
+        self.assertTrue((self.data_dir / "retrieval" / "shards"
+                         / "s-s1.json").exists())
+
+    def test_month_shard_without_session(self):
+        row = entry("a", "x", ts=time.mktime((2024, 3, 15, 12, 0, 0, 0, 0, -1)))
+        shards = rp.assign_shards(self.store, [row])
+        self.assertEqual(shards["a"], "2024-03")
+
+    def test_rollover_over_cap(self):
+        rows = [entry(str(index), "x") for index in range(5)]
+        shards = rp.assign_shards(self.store, rows, session_id="s1", limit=2)
+        names = list(shards.values())
+        self.assertEqual(names.count("s-s1"), 2)
+        self.assertIn("s-s1-r1", names)
+        self.assertIn("s-s1-r2", names)
+
+
+class PredictionTest(Case):
+    def test_expectation_weights(self):
+        rp.record_query(self.store, "苹果 咖啡")
+        rp.record_query(self.store, "苹果 咖啡 甜")
+        rows = [entry("expected", "苹果 咖啡"), entry("novel", "维修 螺丝")]
+        ranked = rp.rank(self.store, "s1", "苹果", rows, now=self.now)
+        by_id = {item["entry"]["id"]: item for item in ranked}
+        self.assertGreater(by_id["novel"]["expectation_weight"],
+                           by_id["expected"]["expectation_weight"])
+
+    def test_topic_history_bounded(self):
+        for index in range(25):
+            rp.record_query(self.store, "话题%d" % index)
+        doc = json.loads((self.data_dir / "topic_history.json")
+                         .read_text(encoding="utf-8"))
+        self.assertEqual(len(doc["turns"]), 20)
+
+    def test_off_fallback(self):
+        rows = [entry("a", "苹果"), entry("b", "香蕉")]
+        ranked = rp.rank(self.store, "s1", "苹果", rows, now=self.now,
+                         enabled=False)
+        self.assertEqual([item["entry"]["id"] for item in ranked], ["a", "b"])
+        self.assertEqual(ranked[0]["reason"], "fallback")
+
+
+class DebugTest(Case):
+    def test_debug_rows_and_state(self):
+        rows = [entry("a", "苹果很甜"), entry("b", "香蕉")]
+        ranked = rp.rank(self.store, "s1", "苹果", rows, now=self.now)
+        rp.record_debug(self.store, "苹果", ranked, now=self.now)
+        summary = rp.summary(self.store)
+        self.assertEqual(summary["query"], "苹果")
+        for key in ("channels", "reason", "chunk", "chunk_hits", "shard"):
+            self.assertIn(key, summary["rows"][0])
+        body = pipeline.dispatch("state", {
+            "store": str(self.store), "session_id": "s1",
+            "persona_profile": str(Path(self.tmp.name) / "missing.json")})["state"]
+        self.assertIn("recall_debug", body)
+
+    def test_switch_off_keeps_engine_order(self):
+        rows = [entry("old", "苹果 很甜", ts=self.now - 100),
+                entry("new", "苹果 今天", ts=self.now)]
+        (self.store / "sessions" / "s1" / "memory.json").write_text(
+            json.dumps({"entries": rows}), encoding="utf-8")
+        on = pipeline.inject_block("s1", str(self.store), query="苹果",
+                                   mode="query", top_k=2)
+        path = self.data_dir / "core_config.json"
+        path.write_text(json.dumps({"switches": {"retrieval_plus": False}}),
+                        encoding="utf-8")
+        off = pipeline.inject_block("s1", str(self.store), query="苹果",
+                                    mode="query", top_k=2)
+        self.assertEqual(len(on["ids"]), len(off["ids"]))
+
+
+if __name__ == "__main__":
+    unittest.main()