dsh-biocore 0.2.0 → 0.3.1

package/python/biocore_model/tests/test_memory_dedup.py

@@ -0,0 +1,316 @@
+# -*- coding: utf-8 -*-
+"""task-147: memory dedupe (text_hash unique key + semantic second pass)."""
+import json
+import sqlite3
+import tempfile
+import types
+import unittest
+from pathlib import Path
+
+from biocore_model import memory_store as ms
+
+VEC_A = [1.0, 0.0, 0.0, 0.0]
+VEC_NEAR = [0.92, 0.391918, 0.0, 0.0]      # cosine ~0.92 with VEC_A
+
+
+def _entry(text, ts=100.0, importance=0.5, hits=1, access=1,
+           vector=None, eid=None, **extra):
+    entry = {"text": text, "ts": ts, "importance": importance,
+             "hits": hits, "access_count": access,
+             "vector": list(vector) if vector else None, "role": "user"}
+    if eid:
+        entry["id"] = eid
+    entry.update(extra)
+    return entry
+
+
+def _append_save(store, session, entries):
+    """Full-doc save (what main does): keep existing rows, append new entries."""
+    doc = store.load_doc(session)
+    doc["entries"] = list(doc.get("entries") or []) + list(entries)
+    return store.save_doc(session, doc)
+
+
+def _legacy_db(path, session="s1"):
+    """Create a pre-dedupe (schema v4, no text_hash) SQLite file."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path))
+    conn.executescript("""
+CREATE TABLE entries(
+    id TEXT PRIMARY KEY, session TEXT NOT NULL, seq INTEGER NOT NULL DEFAULT 0,
+    text TEXT, role TEXT, ts REAL, importance REAL, vector BLOB,
+    storage_s REAL, retrieval_s REAL, access_count INTEGER, hits INTEGER,
+    inhibition REAL, suppression_count INTEGER, memory_tier TEXT,
+    memory_source TEXT, memory_confidence REAL, arousal_at_encoding REAL,
+    hidden INTEGER, tags TEXT, raw TEXT
+);
+CREATE TABLE session_docs(session TEXT PRIMARY KEY, doc TEXT, updated_at REAL);
+CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT);
+""")
+    return conn
+
+
+def _insert_legacy(conn, session, rows):
+    for seq, entry in enumerate(rows):
+        conn.execute(
+            "INSERT OR REPLACE INTO entries(id, session, seq, text, role, ts, "
+            "importance, vector, access_count, hits, memory_tier, "
+            "memory_source, raw) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            (entry.get("id") or "e%d" % seq, session, seq,
+             entry.get("text"), "user", entry.get("ts", 0.0),
+             entry.get("importance", 0.5),
+             ms.encode_vector(entry.get("vector")),
+             int(entry.get("access_count", 1)), int(entry.get("hits", 0)),
+             "warm", "direct",
+             json.dumps({k: v for k, v in entry.items() if k != "vector"})))
+    conn.commit()
+
+
+class TextHashTest(unittest.TestCase):
+    def test_normalize_and_hash(self):
+        self.assertEqual(ms.normalize_text(" 今天 很累！"), "今天很累")
+        self.assertEqual(ms.text_hash("今天 很累！"),
+                         ms.text_hash("今天很累"))
+        self.assertNotEqual(ms.text_hash("今天很累"),
+                            ms.text_hash("今天很开心"))
+
+    def test_same_text_written_twice_merges(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("今天很累", ts=100.0, importance=0.4, hits=1)]})
+                store.save_doc("s", {"entries": [
+                    _entry("今天很累", ts=50.0, importance=0.9, hits=2)]})
+                self.assertEqual(store.stats()["entries"], 1)
+                entry = store.load_doc("s")["entries"][0]
+                self.assertEqual(entry["ts"], 50.0)         # earliest
+                self.assertAlmostEqual(entry["importance"], 0.9)  # higher
+                self.assertEqual(entry["hits"], 3)          # summed
+                self.assertEqual(entry["access_count"], 2)
+                self.assertTrue(entry["text_hash"])
+            finally:
+                store.close()
+
+    def test_normalized_duplicate_merges(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("今天，很累！"), _entry(" 今天 很累 ") ]})
+                self.assertEqual(store.stats()["entries"], 1)
+            finally:
+                store.close()
+
+    def test_unique_index_blocks_direct_duplicate(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [_entry("同一句")]})
+                self.assertTrue(store._has_index(ms.TEXT_HASH_INDEX))
+                row = store._conn.execute(
+                    "SELECT id, text_hash FROM entries").fetchone()
+                with self.assertRaises(sqlite3.IntegrityError):
+                    store._conn.execute(
+                        "INSERT INTO entries(id, session, text_hash) "
+                        "VALUES(?,?,?)", ("dup", "s", row[1]))
+                    store._conn.commit()
+            finally:
+                store.close()
+
+
+class SemanticDedupTest(unittest.TestCase):
+    def test_semantic_paraphrase_merges(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("我很开心", ts=100.0, vector=[1.0, 0.001, 0, 0])]})
+                report = _append_save(store, "s", [
+                    _entry("我很开心呀", ts=200.0, vector=[0.999, 0.0, 0, 0])])
+                self.assertEqual(store.stats()["entries"], 1)
+                self.assertGreaterEqual(report["semantic_merged"], 1)
+                entry = store.load_doc("s")["entries"][0]
+                self.assertEqual(entry["ts"], 100.0)
+                self.assertEqual(entry["hits"], 2)
+            finally:
+                store.close()
+
+    def test_cosine_090_095_not_merged(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            captured = []
+            original = ms.logger
+            ms.logger = types.SimpleNamespace(
+                info=lambda *a, **k: captured.append(a),
+                warning=lambda *a, **k: None,
+                debug=lambda *a, **k: None)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("我有点开心", vector=VEC_A)]})
+                report = _append_save(store, "s", [
+                    _entry("我有点开心呢", vector=VEC_NEAR)])
+                self.assertEqual(store.stats()["entries"], 2)
+                self.assertEqual(report["semantic_merged"], 0)
+                self.assertGreaterEqual(report["near_merge"], 1)
+                self.assertEqual(ms.cosine(VEC_A, VEC_NEAR) >= 0.90, True)
+                self.assertEqual(ms.cosine(VEC_A, VEC_NEAR) < 0.95, True)
+                self.assertTrue(any("near-merge" in " ".join(map(str, row))
+                                    for row in captured))
+            finally:
+                ms.logger = original
+                store.close()
+
+    def test_doc_internal_duplicates_merge(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("同一句"), _entry("同一句。")]})
+                self.assertEqual(store.stats()["entries"], 1)
+                mirror = json.loads(
+                    (Path(tmp) / "s.json").read_text(encoding="utf-8"))
+                self.assertEqual(len(mirror["entries"]), 1)
+                self.assertIn("text_hash", mirror["entries"][0])
+            finally:
+                store.close()
+
+    def test_save_is_idempotent_for_same_doc(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                doc = {"entries": [_entry("一天", ts=10.0, hits=2,
+                                          access=3)]}
+                store.save_doc("s", doc)
+                first = store.load_doc("s")["entries"][0]
+                doc2 = {"entries": [_entry("一天", ts=10.0, hits=2,
+                                           access=3, id=first["id"])]}
+                store.save_doc("s", doc2)
+                second = store.load_doc("s")["entries"][0]
+                self.assertEqual(store.stats()["entries"], 1)
+                self.assertEqual(second["hits"], 2)        # no inflation
+                self.assertEqual(second["access_count"], 3)
+            finally:
+                store.close()
+
+
+class MergeFieldTest(unittest.TestCase):
+    def test_merge_field_values(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("同句", ts=300.0, importance=0.3, hits=1,
+                           access=1)]})
+                store.save_doc("s", {"entries": [
+                    _entry("同句", ts=100.0, importance=0.8, hits=4,
+                           access=2)]})
+                entry = store.load_doc("s")["entries"][0]
+                self.assertEqual(entry["ts"], 100.0)
+                self.assertAlmostEqual(entry["importance"], 0.8)
+                self.assertEqual(entry["hits"], 5)
+                self.assertEqual(entry["access_count"], 3)
+            finally:
+                store.close()
+
+
+class MigrationCleanupTest(unittest.TestCase):
+    def _rows(self):
+        return [
+            _entry("今天很累", ts=100.0, importance=0.4, hits=1,
+                   access=1, vector=VEC_A),
+            _entry("今天很累。", ts=50.0, importance=0.9, hits=2,
+                   access=1, vector=VEC_A),
+            _entry("我很开心", ts=200.0, importance=0.5, hits=1,
+                   access=1, vector=[0.0, 1.0, 0.0, 0.0]),
+            _entry("我很开心呀", ts=250.0, importance=0.7, hits=3,
+                   access=2, vector=[0.0, 0.999, 0.0, 0.0]),
+            _entry("我有点开心", ts=260.0, importance=0.5, hits=1,
+                   access=1, vector=VEC_NEAR),
+            _entry("别的事", ts=300.0, importance=0.5, hits=1,
+                   access=1, vector=[0.0, 0.0, 1.0, 0.0]),
+        ]
+
+    def test_migration_merges_and_reports(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db = Path(tmp) / ms.SQLITE_FILE
+            conn = _legacy_db(db)
+            try:
+                _insert_legacy(conn, "s1", self._rows())
+            finally:
+                conn.close()
+            store = ms.open_store(tmp)
+            try:
+                report = store.migration_report
+                self.assertTrue(report.get("ran"))
+                self.assertEqual(report.get("merged_pairs"), 2)
+                self.assertEqual(report.get("deleted_rows"), 2)
+                self.assertEqual(store.stats()["entries"], 4)
+                self.assertTrue(store._has_index(ms.TEXT_HASH_INDEX))
+                # rows are merged into the earliest row (ts=50 is e1)
+                merged = store.get_entry("s1", "e1")
+                self.assertEqual(merged["ts"], 50.0)
+                self.assertAlmostEqual(merged["importance"], 0.9)
+                self.assertEqual(merged["hits"], 3)
+                # 0.90-0.95 pair is kept separate
+                self.assertIsNotNone(store.get_entry("s1", "e4"))
+            finally:
+                store.close()
+
+    def test_migration_backup_exists(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db = Path(tmp) / ms.SQLITE_FILE
+            conn = _legacy_db(db)
+            try:
+                _insert_legacy(conn, "s1", self._rows())
+            finally:
+                conn.close()
+            store = ms.open_store(tmp)
+            try:
+                backup = store.migration_report.get("backup")
+                self.assertTrue(backup)
+                self.assertTrue(Path(backup).is_dir())
+                self.assertGreater(len(list(Path(backup).iterdir())), 0)
+            finally:
+                store.close()
+
+    def test_migration_idempotent(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            db = Path(tmp) / ms.SQLITE_FILE
+            conn = _legacy_db(db)
+            try:
+                _insert_legacy(conn, "s1", self._rows())
+            finally:
+                conn.close()
+            store = ms.open_store(tmp)
+            try:
+                first = store.stats()["entries"]
+            finally:
+                store.close()
+            store = ms.open_store(tmp)
+            try:
+                self.assertFalse(store.migration_report.get("ran"))
+                self.assertEqual(store.stats()["entries"], first)
+            finally:
+                store.close()
+
+
+class MirrorSyncTest(unittest.TestCase):
+    def test_json_mirror_matches_deduped_rows(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            store = ms.open_store(tmp)
+            try:
+                store.save_doc("s", {"entries": [
+                    _entry("镜像同步"), _entry("镜像同步！")]})
+                mirror = json.loads(
+                    (Path(tmp) / "s.json").read_text(encoding="utf-8"))
+                self.assertEqual(len(mirror["entries"]), 1)
+                self.assertEqual(mirror["entries"][0]["text"], "镜像同步")
+                self.assertTrue(mirror["entries"][0]["text_hash"])
+            finally:
+                store.close()
+
+
+if __name__ == "__main__":
+    unittest.main()

package/python/biocore_model/tests/test_memory_dual.py

@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*-
+"""MD-1 memory_dual tests: formulas, testing effect, selection, forgetting."""
+import math
+import unittest
+
+from biocore_model import memory_dual as md
+
+NOW = 1_700_000_000.0
+
+
+def entry(**kw):
+    base = {"id": "e1", "text": "hello", "storage_s": 0.5, "retrieval_s": 0.5,
+            "access_count": 0, "last_access_ts": NOW, "created_ts": NOW,
+            "arousal_at_encoding": 0.5, "open_loop": False}
+    base.update(kw)
+    return base
+
+
+class DecayTest(unittest.TestCase):
+    def test_power_law_1h(self):
+        self.assertAlmostEqual(md.decay(1.0, 1.0), math.pow(1 + 1 / 24.0, -0.5), places=12)
+        self.assertAlmostEqual(md.decay(1.0, 1.0), 0.979796, places=6)
+
+    def test_power_law_24h(self):
+        self.assertAlmostEqual(md.decay(1.0, 24.0), 0.707107, places=6)
+
+    def test_power_law_7d(self):
+        self.assertAlmostEqual(md.decay(1.0, 168.0), 0.353553, places=6)
+
+    def test_zero_dt_and_scaling(self):
+        self.assertEqual(md.decay(0.42, 0.0), 0.42)
+        self.assertAlmostEqual(md.decay(0.8, 24.0), 0.8 * 0.707107, places=6)
+        self.assertGreater(md.decay(0.8, 1.0), md.decay(0.8, 24.0))
+
+
+class EncodeTest(unittest.TestCase):
+    def test_defaults(self):
+        row = md.encode("海边", NOW)
+        self.assertEqual(row["storage_s"], 0.35)
+        self.assertEqual(row["retrieval_s"], 0.35)
+        self.assertEqual(row["access_count"], 0)
+        self.assertEqual(row["last_access_ts"], NOW)
+        self.assertFalse(row["open_loop"])
+        self.assertFalse(row["hidden"])
+
+    def test_arousal_raises_initial_storage(self):
+        low = md.encode("low", NOW, arousal=0.0)
+        high = md.encode("high", NOW, arousal=1.0)
+        self.assertAlmostEqual(high["storage_s"], 0.70, places=4)
+        self.assertGreater(high["storage_s"], low["storage_s"])
+        self.assertEqual(high["retrieval_s"], high["storage_s"])
+
+    def test_arousal_clamped(self):
+        self.assertAlmostEqual(md.encode("x", NOW, arousal=5.0)["storage_s"], 0.70, places=4)
+        self.assertAlmostEqual(md.encode("x", NOW, arousal=-3.0)["storage_s"], 0.35, places=4)
+
+    def test_open_loop_flag(self):
+        self.assertTrue(md.encode("todo", NOW, open_loop=True)["open_loop"])
+
+
+class RecallTest(unittest.TestCase):
+    def test_testing_effect_storage_gain(self):
+        row = entry(storage_s=0.4, retrieval_s=0.3, access_count=0, last_access_ts=0.0)
+        out = md.on_recall(row, NOW)
+        self.assertAlmostEqual(out["storage_s"], 0.46, places=4)   # +0.1*(1-0.4)
+        self.assertAlmostEqual(out["retrieval_s"], 0.65, places=4)  # +0.35
+        self.assertEqual(out["access_count"], 1)
+        self.assertEqual(out["last_access_ts"], NOW)
+
+    def test_retrieval_capped_at_one(self):
+        out = md.on_recall(entry(retrieval_s=0.9, last_access_ts=0.0), NOW)
+        self.assertEqual(out["retrieval_s"], 1.0)
+
+    def test_idempotent_same_now(self):
+        row = entry(storage_s=0.4, retrieval_s=0.3, last_access_ts=0.0)
+        first = md.on_recall(row, NOW)
+        second = md.on_recall(first, NOW)
+        self.assertEqual(second["storage_s"], first["storage_s"])
+        self.assertEqual(second["retrieval_s"], first["retrieval_s"])
+        self.assertEqual(second["access_count"], 1)
+
+    def test_does_not_mutate_input(self):
+        row = entry(storage_s=0.4, retrieval_s=0.3, last_access_ts=0.0)
+        before = dict(row)
+        md.on_recall(row, NOW)
+        self.assertEqual(row, before)
+
+    def test_new_now_applies_again(self):
+        first = md.on_recall(entry(last_access_ts=0.0), NOW)
+        second = md.on_recall(first, NOW + 60)
+        self.assertEqual(second["access_count"], 2)
+        self.assertGreater(second["storage_s"], first["storage_s"])
+
+
+class ScoreTest(unittest.TestCase):
+    def test_context_weight(self):
+        row = entry()
+        full = md.score(row, NOW, context_match=1.0)
+        half = md.score(row, NOW, context_match=0.25)
+        self.assertAlmostEqual(half / full, 0.5, places=6)  # 0.25 ** 0.5
+
+    def test_open_loop_bonus(self):
+        closed = md.score(entry(open_loop=False), NOW)
+        opened = md.score(entry(open_loop=True), NOW)
+        self.assertAlmostEqual(opened / closed, 1.15, places=6)
+
+    def test_frequency_term(self):
+        low = md.score(entry(access_count=0), NOW)
+        high = md.score(entry(access_count=3), NOW)
+        expected = 1.0 + 0.3 * math.log1p(3)
+        self.assertAlmostEqual(high / low, expected, places=6)
+
+    def test_decays_over_time(self):
+        row = entry(retrieval_s=0.6)
+        s0 = md.score(row, NOW)
+        s24 = md.score(row, NOW + 24 * 3600)
+        s72 = md.score(row, NOW + 72 * 3600)
+        self.assertGreater(s0, s24)
+        self.assertGreater(s24, s72)
+
+    def test_hidden_scores_zero(self):
+        self.assertEqual(md.score(entry(hidden=True), NOW), 0.0)
+
+
+class SelectTest(unittest.TestCase):
+    def _rows(self):
+        return [
+            entry(id="strong", text="strong", storage_s=0.9, retrieval_s=0.9,
+                  last_access_ts=NOW),
+            entry(id="recent", text="recent", storage_s=0.6, retrieval_s=0.6,
+                  last_access_ts=NOW - 3600),
+            entry(id="open", text="open", storage_s=0.5, retrieval_s=0.5,
+                  open_loop=True, last_access_ts=NOW - 24 * 3600),
+            entry(id="old", text="old", storage_s=0.4, retrieval_s=0.4,
+                  last_access_ts=NOW - 30 * 24 * 3600),
+            entry(id="hidden", text="hidden", retrieval_s=0.9, hidden=True),
+        ]
+
+    def test_sorted_and_limited(self):
+        out = md.select(self._rows(), NOW, limit=3)
+        ids = [row["id"] for row in out["picked"]]
+        self.assertEqual(len(ids), 3)
+        self.assertNotIn("hidden", ids)
+        scores = [row["score"] for row in out["picked"]]
+        self.assertEqual(scores, sorted(scores, reverse=True))
+        self.assertEqual(ids[0], "strong")
+
+    def test_min_score_filters(self):
+        rows = [entry(id="weak", retrieval_s=0.01, last_access_ts=NOW - 90 * 24 * 3600)]
+        out = md.select(rows, NOW, min_score=0.12)
+        self.assertEqual(out["picked"], [])
+        self.assertEqual(out["skipped"][0]["reason"], "below_min_score")
+
+    def test_skipped_reason_for_limit(self):
+        out = md.select(self._rows(), NOW, limit=1)
+        reasons = {row["id"]: row["reason"] for row in out["skipped"]}
+        self.assertEqual(reasons["hidden"], "hidden")
+        self.assertIn("limit", reasons.values())
+
+    def test_context_dict_per_entry(self):
+        rows = [entry(id="a", retrieval_s=0.5), entry(id="b", retrieval_s=0.5)]
+        out = md.select(rows, NOW, context={"a": 1.0, "b": 0.01})
+        self.assertEqual(out["picked"][0]["id"], "a")
+        self.assertNotIn("b", [row["id"] for row in out["picked"]])
+
+
+class ForgetTest(unittest.TestCase):
+    def test_marks_hidden_without_deleting(self):
+        rows = [entry(id="faint", retrieval_s=0.10),
+                entry(id="solid", retrieval_s=0.50)]
+        out = md.forget(rows, NOW)
+        self.assertEqual(len(out["entries"]), 2)          # nothing deleted
+        by_id = {row["id"]: row for row in out["entries"]}
+        self.assertTrue(by_id["faint"]["hidden"])
+        self.assertFalse(by_id["solid"].get("hidden"))
+        self.assertEqual(out["forgotten_ids"], ["faint"])
+        self.assertNotIn("hidden", rows[0])               # input untouched
+
+    def test_idempotent_second_call(self):
+        rows = [entry(id="faint", retrieval_s=0.10)]
+        first = md.forget(rows, NOW)
+        second = md.forget(first["entries"], NOW)
+        self.assertEqual(first["forgotten_ids"], ["faint"])
+        self.assertEqual(second["forgotten_ids"], [])
+
+
+class TimeEvolutionTest(unittest.TestCase):
+    def test_ten_memories_three_horizons(self):
+        rows = []
+        for index in range(10):
+            rows.append(entry(
+                id="m%02d" % (index + 1), text="memory-%d" % index,
+                storage_s=0.35 + 0.035 * index,
+                retrieval_s=0.35 + 0.035 * index,
+                last_access_ts=NOW - (index * 6) * 3600,
+                open_loop=(index % 3 == 0),
+            ))
+        early = md.select(rows, NOW, limit=4)
+        day = md.select(rows, NOW + 24 * 3600, limit=4)
+        three = md.select(rows, NOW + 72 * 3600, limit=4)
+        early_scores = {row["id"]: row["score"] for row in early["picked"]}
+        day_scores = {row["id"]: row["score"] for row in day["picked"]}
+        three_scores = {row["id"]: row["score"] for row in three["picked"]}
+        common = set(early_scores) & set(three_scores)
+        if common:
+            sample = sorted(common)[0]
+            self.assertGreater(early_scores[sample], three_scores[sample])
+        self.assertNotEqual(early_scores, day_scores)
+        self.assertLessEqual(max(three_scores.values()) if three_scores else 1.0,
+                             max(early_scores.values()))
+
+    def test_forget_after_long_gap(self):
+        rows = [entry(id="m1", retrieval_s=0.30, last_access_ts=NOW)]
+        out = md.forget(rows, NOW + 30 * 24 * 3600)
+        self.assertIn("m1", out["forgotten_ids"])
+
+
+if __name__ == "__main__":
+    unittest.main()