dsh-biocore 0.3.2 → 0.4.0

package/python/biocore_model/tests/test_standby.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+"""task-215: standby initiative -- outbox only, default off."""
+import json
+import tempfile
+import time
+import unittest
+from pathlib import Path
+
+from biocore_runtime import pipeline, standby
+
+
+def noon(offset_minutes=0):
+    local = time.localtime()
+    stamp = time.mktime((local.tm_year, local.tm_mon, local.tm_mday,
+                         12, 0, 0, 0, 0, -1))
+    return stamp + offset_minutes * 60
+
+
+def ok_transport(captured=None):
+    def call(url, payload, timeout, headers):
+        if captured is not None:
+            captured.append(payload)
+        body = json.dumps({"choices": [{"message": {"content": "最近还好吗？"}}]})
+        return 200, body.encode("utf-8")
+    return call
+
+
+def fail_transport(url, payload, timeout, headers):
+    raise OSError("network down")
+
+
+class Case(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.data_dir = Path(self.tmp.name) / "data"
+        self.store = self.data_dir / "memory"
+        (self.store / "sessions" / "s1").mkdir(parents=True)
+        self.cfg = {
+            "standby_enabled": True,
+            "standby_store": str(self.store),
+            "standby_session_id": "s1",
+            "standby_model_base_url": "http://127.0.0.1:9",
+            "standby_model": "test-model",
+            "standby_active_hours": "09:00-23:00",
+        }
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def outbox(self):
+        path = self.store / "proactive" / "outbox.jsonl"
+        if not path.exists():
+            return []
+        return [json.loads(line) for line in
+                path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+    def seed_state(self, **values):
+        path = self.store / "proactive" / "state.json"
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(values), encoding="utf-8")
+
+    def session_state(self, **values):
+        pipeline._save_session_state(str(self.store), "s1", values)
+
+
+class GatesTest(Case):
+    def test_default_off(self):
+        result = standby.tick(now=noon(), config={"standby_store": str(self.store)})
+        self.assertEqual(result["reason"], "disabled")
+        self.assertEqual(self.outbox(), [])
+
+    def test_outside_active_hours(self):
+        early = noon(-12 * 60)  # 00:00
+        result = standby.tick(now=early, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "outside_active_hours")
+        self.assertEqual(self.outbox(), [])
+
+    def test_daily_cap(self):
+        now = noon()
+        self.seed_state(sent=[now - 60, now - 120, now - 180],
+                        last_attempt_ts=0.0, last_sent_ts=0.0)
+        result = standby.tick(now=now, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "daily_cap")
+
+    def test_cooldown_blocks(self):
+        now = noon()
+        self.seed_state(last_attempt_ts=now - 10 * 60)
+        result = standby.tick(now=now, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "cooldown")
+
+    def test_no_reply_streak_stops(self):
+        now = noon()
+        self.seed_state(awaiting_reply=True, no_reply_streak=1,
+                        last_sent_ts=now - 3 * 3600)
+        self.session_state(last_human_ts=now - 5 * 3600)
+        result = standby.tick(now=now, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "no_reply_streak")
+        self.assertEqual(self.outbox(), [])
+
+    def test_reply_resets_streak(self):
+        now = noon()
+        self.seed_state(awaiting_reply=True, no_reply_streak=1,
+                        last_sent_ts=now - 3 * 3600,
+                        last_attempt_ts=now - 3 * 3600)
+        self.session_state(last_human_ts=now - 3600)
+        result = standby.tick(now=now, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "ok")
+        self.assertEqual(len(self.outbox()), 1)
+
+
+class OutputTest(Case):
+    def test_outbox_row_shape(self):
+        now = noon()
+        result = standby.tick(now=now, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(result["reason"], "ok")
+        row = self.outbox()[0]
+        for key in ("ts", "text", "reason", "state_snapshot", "prompt",
+                    "fallback"):
+            self.assertIn(key, row)
+        self.assertEqual(row["text"], "最近还好吗？")
+        self.assertFalse(row["fallback"])
+
+    def test_prompt_has_full_persona_memories_digest(self):
+        long_persona = "你是安吉拉。" + "必须保持礼貌、精确、克制。" * 40
+        (self.store / "persona_profile.json").write_text(
+            json.dumps({"persona_text": long_persona}), encoding="utf-8")
+        (self.store / "sessions" / "s1" / "memory.json").write_text(
+            json.dumps({"entries": [
+                {"id": "a", "text": "苹果很甜", "ts": 3},
+                {"id": "b", "text": "明天体检", "ts": 2},
+                {"id": "c", "text": "咖啡少糖", "ts": 1}]}), encoding="utf-8")
+        self.session_state(last_user_text="今天有点累", last_human_ts=noon() - 3600)
+        prompt = standby.build_prompt({}, ["苹果很甜", "明天体检", "咖啡少糖"],
+                                      long_persona, digest="今天有点累")
+        self.assertIn(long_persona, prompt)
+        self.assertIn("最近记得：", prompt)
+        self.assertIn("苹果很甜", prompt)
+        self.assertIn("上次对话：今天有点累", prompt)
+        captured = []
+        standby.tick(now=noon(), config=self.cfg,
+                     transport=ok_transport(captured))
+        tick_prompt = captured[0]["messages"][0]["content"]
+        self.assertIn("最近记得：", tick_prompt)
+        self.assertIn("上次对话：今天有点累", tick_prompt)
+
+    def test_missing_model_uses_template(self):
+        cfg = dict(self.cfg)
+        cfg["standby_model"] = ""
+        result = standby.tick(now=noon(), config=cfg)
+        self.assertEqual(result["reason"], "template_fallback")
+        self.assertTrue(result["fallback"])
+        self.assertTrue(self.outbox()[0]["text"])
+
+    def test_transport_failure_uses_template(self):
+        result = standby.tick(now=noon(), config=self.cfg,
+                              transport=fail_transport)
+        self.assertEqual(result["reason"], "template_fallback")
+        self.assertTrue(self.outbox()[0]["fallback"])
+
+    def test_second_tick_is_cooldown(self):
+        now = noon()
+        standby.tick(now=now, config=self.cfg, transport=ok_transport())
+        second = standby.tick(now=now + 60, config=self.cfg,
+                              transport=ok_transport())
+        self.assertEqual(second["reason"], "cooldown")
+        self.assertEqual(len(self.outbox()), 1)
+
+
+class StateTest(Case):
+    def test_summary_and_state_fields(self):
+        now = noon()
+        standby.tick(now=now, config=self.cfg, transport=ok_transport())
+        summary = standby.summary(str(self.store), now=now + 60, enabled=True)
+        self.assertTrue(summary["enabled"])
+        self.assertEqual(summary["sent_today"], 1)
+        self.assertGreater(summary["next_at"], 0)
+        self.assertEqual(summary["last_reason"], "ok")
+        body = pipeline.dispatch("state", {
+            "store": str(self.store), "session_id": "s1",
+            "persona_profile": str(Path(self.tmp.name) / "missing.json")})["state"]
+        self.assertIn("standby", body)
+        for key in ("enabled", "next_at", "pending", "sent_today",
+                    "last_reason"):
+            self.assertIn(key, body["standby"])
+
+    def test_action_is_gated_by_flag(self):
+        out = pipeline.dispatch("standby_tick", {
+            "store": str(self.store), "session_id": "s1", "now": noon()})
+        self.assertEqual(out["reason"], "disabled")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/python/biocore_runtime/__init__.py

@@ -2,6 +2,9 @@
 """BioCore runtime package: host-neutral policy shared by every adapter."""
 from __future__ import annotations
 
+__version__ = "0.1.0"
+VERSION = (0, 1, 0)
+
 from . import pipeline
 
 __all__ = ["pipeline"]

package/python/biocore_runtime/cache_stats.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+"""Cache/usage accounting in the core (contract migration 1).
+
+Usage events live in <dataDir>/usage_events.jsonl; the core is the only writer.
+The adapter may still read the legacy <dataDir>/cache_stats.json for a week as
+a read-only fallback, but it never writes or recomputes hit_rate.
+
+hit_rate = cache_read / (cache_read + cache_write), falling back to
+cache_read / prompt_tokens when no write tokens were reported; clamped to
+[0, 1]. The default window is 7 days and every event is bounded.
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+USAGE_FILE = "usage_events.jsonl"
+LEGACY_FILE = "cache_stats.json"
+SCHEMA = 1
+DEFAULT_WINDOW_HOURS = 24 * 7
+MAX_EVENTS = 5000
+
+
+def _root(store):
+    if isinstance(store, dict):
+        raw = store.get("root") or store.get("store") or store.get("path")
+    else:
+        raw = store
+    text = str(raw or "").strip()
+    return Path(text) if text else None
+
+
+def _data_dir(store):
+    root = _root(store)
+    return (root.parent if root.name == "memory" else root) if root else None
+
+
+def _num(value, default=0.0):
+    try:
+        number = float(value)
+    except Exception:
+        return default
+    return number if number == number else default
+
+
+def usage_path(store):
+    data_dir = _data_dir(store)
+    return (data_dir / USAGE_FILE) if data_dir is not None else None
+
+
+def legacy_path(store):
+    data_dir = _data_dir(store)
+    return (data_dir / LEGACY_FILE) if data_dir is not None else None
+
+
+def _read_events(store):
+    path = usage_path(store)
+    rows = []
+    try:
+        if path is None or not path.exists():
+            return rows
+        for line in path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                item = json.loads(line)
+            except Exception:
+                continue
+            if isinstance(item, dict):
+                rows.append(item)
+    except Exception:
+        return []
+    return rows
+
+
+def record_event(store, event, now=None):
+    """Append one usage event; bounded to MAX_EVENTS."""
+    path = usage_path(store)
+    if path is None:
+        return None
+    stamp = round(float(now or event.get("ts") or time.time()), 3)
+    row = {
+        "ts": stamp,
+        "prompt_tokens": max(0, int(_num(event.get("prompt_tokens")))),
+        "output_tokens": max(0, int(_num(event.get("output_tokens")))),
+        "cache_read": max(0, int(_num(event.get("cache_read")))),
+        "cache_write": max(0, int(_num(event.get("cache_write")))),
+    }
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        rows = _read_events(store)[-(MAX_EVENTS - 1):] + [row]
+        tmp = path.with_name(path.name + ".tmp")
+        tmp.write_text("\n".join(json.dumps(item, ensure_ascii=False)
+                                  for item in rows) + "\n", encoding="utf-8")
+        tmp.replace(path)
+    except Exception:
+        return None
+    return row
+
+
+def _legacy(store):
+    path = legacy_path(store)
+    try:
+        if path is None or not path.exists():
+            return None
+        doc = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    if not isinstance(doc, dict):
+        return None
+    return {
+        "turns": max(0, int(_num(doc.get("turns")))),
+        "prompt_tokens": max(0, int(_num(doc.get("prompt_tokens")))),
+        "output_tokens": max(0, int(_num(doc.get("output_tokens")))),
+        "cache_read": max(0, int(_num(doc.get("cache_read")))),
+        "cache_write": max(0, int(_num(doc.get("cache_write")))),
+        "hit_rate": round(min(1.0, max(0.0, _num(doc.get("hit_rate")))), 4),
+        "updated_at": round(_num(doc.get("updated_at")), 3),
+        "source": "legacy_file",
+    }
+
+
+def compute(store, now=None, window_hours=DEFAULT_WINDOW_HOURS):
+    """Aggregate usage events in the window; legacy file only if no events."""
+    stamp = float(now or time.time())
+    window = max(1.0, _num(window_hours, DEFAULT_WINDOW_HOURS))
+    cutoff = stamp - window * 3600.0
+    events = [row for row in _read_events(store)
+              if _num(row.get("ts"), 0.0) >= cutoff]
+    if not events:
+        legacy = _legacy(store)
+        if legacy is not None:
+            legacy["window_hours"] = round(window, 2)
+            return legacy
+        return {"turns": 0, "prompt_tokens": 0, "output_tokens": 0,
+                "cache_read": 0, "cache_write": 0, "hit_rate": 0.0,
+                "updated_at": 0.0, "window_hours": round(window, 2),
+                "source": "empty"}
+    prompt = sum(max(0, int(_num(row.get("prompt_tokens")))) for row in events)
+    output = sum(max(0, int(_num(row.get("output_tokens")))) for row in events)
+    cache_read = sum(max(0, int(_num(row.get("cache_read")))) for row in events)
+    cache_write = sum(max(0, int(_num(row.get("cache_write")))) for row in events)
+    denom = cache_read + cache_write
+    if prompt > 0:
+        hit_rate = cache_read / float(prompt)
+    elif denom > 0:
+        hit_rate = cache_read / float(denom)
+    else:
+        hit_rate = 0.0
+    hit_rate = round(min(1.0, max(0.0, hit_rate)), 4)
+    return {"turns": len(events), "prompt_tokens": prompt,
+            "output_tokens": output, "cache_read": cache_read,
+            "cache_write": cache_write, "hit_rate": hit_rate,
+            "updated_at": round(_num(events[-1].get("ts")), 3),
+            "window_hours": round(window, 2), "source": "usage_events"}
+
+
+def summary(store, now=None):
+    return compute(store, now=now, window_hours=DEFAULT_WINDOW_HOURS)

package/python/biocore_runtime/lifecycle.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+"""task-36 + task-78: tier hysteresis, reconsolidation, minimal repair loop.
+
+- tier_tick: call after recall; hot stays hot until hits < 3 (task-36).
+- record_recall/reconsolidate: 6h window; recall + new input marks the entry
+  regenerated_at and nudges its strength via biocore_model.feedback_memory.
+  It never deletes and is idempotent per recall batch (task-78).
+- repair: a blocked boundary arms one repair turn; prompt_block then lowers
+  confrontation / prefers repair wording, with a short cooldown (task-78).
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+from biocore_memory.store import tiering as _tiering
+
+try:
+    from biocore_model import feedback_memory as _feedback
+except Exception:  # pragma: no cover
+    _feedback = None
+
+TIERS_FILE = "tiers.json"
+RECALL_FILE = "recall_state.json"
+REPAIR_FILE = "repair.json"
+SCHEMA = 1
+RECONSOLIDATION_WINDOW_S = 6 * 3600.0
+REPAIR_TTL_S = 30 * 60.0
+REPAIR_COOLDOWN_S = 10 * 60.0
+IMPORTANCE_STEP = 0.02
+
+
+def _root(store):
+    if isinstance(store, dict):
+        raw = store.get("root") or store.get("store") or store.get("path")
+    else:
+        raw = store
+    text = str(raw or "").strip()
+    return Path(text) if text else None
+
+
+def _data_dir(store):
+    root = _root(store)
+    return (root.parent if root.name == "memory" else root) if root else None
+
+
+def _read(path):
+    try:
+        if path is None or not path.exists():
+            return {}
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return data if isinstance(data, dict) else {}
+    except Exception:
+        return {}
+
+
+def _write(path, doc):
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_name(path.name + ".tmp")
+        tmp.write_text(json.dumps(doc, ensure_ascii=False, indent=2),
+                       encoding="utf-8")
+        tmp.replace(path)
+        return True
+    except Exception:
+        return False
+
+
+def _num(value, default=0.0):
+    try:
+        number = float(value)
+    except Exception:
+        return default
+    return number if number == number else default
+
+
+# ---------------------------------------------------------------- tiers
+def tier_tick(store, entries, now=None):
+    """Classify the recalled entries with hysteresis; returns counters."""
+    root = _root(store)
+    if root is None:
+        return None
+    stamp = round(float(now or time.time()), 3)
+    doc = _read(root / TIERS_FILE)
+    by_id = doc.get("by_id") if isinstance(doc.get("by_id"), dict) else {}
+    counts = {"hot": 0, "warm": 0, "cold": 0}
+    entered = int(_num(doc.get("entered"), 0))
+    exited = int(_num(doc.get("exited"), 0))
+    for entry in entries or []:
+        if not isinstance(entry, dict):
+            continue
+        entry_id = str(entry.get("id") or "")
+        if not entry_id:
+            continue
+        previous = str((by_id.get(entry_id) or {}).get("tier") or "")
+        tier = _tiering.tier_of(entry, now=stamp, previous=previous)
+        if previous and previous != tier:
+            if tier == "hot":
+                entered += 1
+            elif previous == "hot":
+                exited += 1
+        by_id[entry_id] = {"tier": tier, "ts": stamp}
+        counts[tier] = counts.get(tier, 0) + 1
+    doc.update({"schema": SCHEMA, "by_id": by_id, "counts": counts,
+                "entered": entered, "exited": exited, "updated_at": stamp})
+    _write(root / TIERS_FILE, doc)
+    return {"counts": counts, "entered": entered, "exited": exited,
+            "total": sum(counts.values()), "updated_at": stamp}
+
+
+def tier_summary(store):
+    doc = _read((_root(store) or Path(".")) / TIERS_FILE)
+    counts = doc.get("counts") if isinstance(doc.get("counts"), dict) else {}
+    return {"hot": int(_num(counts.get("hot"))),
+            "warm": int(_num(counts.get("warm"))),
+            "cold": int(_num(counts.get("cold"))),
+            "entered": int(_num(doc.get("entered"))),
+            "exited": int(_num(doc.get("exited"))),
+            "updated_at": _num(doc.get("updated_at"))}
+
+
+# ---------------------------------------------------------- reconsolidation
+def record_recall(store, session_id, ids, now=None):
+    root = _root(store)
+    if root is None:
+        return False
+    stamp = round(float(now or time.time()), 3)
+    doc = {"schema": SCHEMA, "session_id": str(session_id or ""),
+           "ids": [str(item) for item in (ids or [])][:12], "ts": stamp}
+    return _write(root / RECALL_FILE, doc)
+
+
+def _load_session_entries(root, session_id):
+    path = root / "sessions" / str(session_id or "") / "memory.json"
+    doc = _read(path)
+    entries = doc.get("entries") if isinstance(doc.get("entries"), list) else []
+    return doc, entries
+
+
+def reconsolidate(store, now=None):
+    """Within 6h of a recall, a new input marks recalled entries regenerated."""
+    root = _root(store)
+    if root is None:
+        return {"enabled": False, "reason": "no_store"}
+    stamp = float(now or time.time())
+    state = _read(root / RECALL_FILE)
+    ids = [str(item) for item in (state.get("ids") or [])]
+    recalled_at = _num(state.get("ts"))
+    if not ids or recalled_at <= 0 or stamp - recalled_at > RECONSOLIDATION_WINDOW_S:
+        return {"enabled": True, "updated": 0, "skipped": len(ids),
+                "reason": "window"}
+    doc, entries = _load_session_entries(root, state.get("session_id"))
+    updated = 0
+    skipped = 0
+    for entry in entries:
+        if not isinstance(entry, dict) or str(entry.get("id") or "") not in ids:
+            continue
+        if _num(entry.get("regenerated_at")) >= recalled_at:
+            skipped += 1
+            continue
+        if _feedback is not None:
+            result = _feedback.touch([entry], entry.get("id"), stamp,
+                                     kind="reconsolidation")
+            touched = (result.get("entries") or [entry])[0]
+            entry.update(touched)
+        entry["regenerated_at"] = round(stamp, 3)
+        entry["reconsolidation_count"] = int(
+            _num(entry.get("reconsolidation_count"))) + 1
+        entry["importance"] = round(min(1.0, max(0.0,
+            _num(entry.get("importance"), 0.5) + IMPORTANCE_STEP)), 4)
+        updated += 1
+    if updated:
+        _write(root / "sessions" / str(state.get("session_id") or "")
+               / "memory.json", doc)
+    return {"enabled": True, "updated": updated, "skipped": skipped,
+            "ids": ids[:12], "ts": round(stamp, 3)}
+
+
+def reconsolidation_summary(store):
+    root = _root(store)
+    if root is None:
+        return {"ids": [], "recalled_at": 0.0, "window_s": RECONSOLIDATION_WINDOW_S}
+    state = _read(root / RECALL_FILE)
+    return {"ids": list(state.get("ids") or [])[:5],
+            "recalled_at": _num(state.get("ts")),
+            "window_s": RECONSOLIDATION_WINDOW_S}
+
+
+# ---------------------------------------------------------------- repair
+def arm_repair(store, reason="", now=None):
+    root = _root(store)
+    if root is None:
+        return False
+    stamp = float(now or time.time())
+    doc = _read(root / REPAIR_FILE)
+    if stamp < _num(doc.get("cooldown_until")):
+        return False
+    doc = {"schema": SCHEMA, "armed": True, "reason": str(reason or "")[:40],
+           "armed_at": round(stamp, 3),
+           "expires_at": round(stamp + REPAIR_TTL_S, 3),
+           "used_at": 0.0,
+           "cooldown_until": round(stamp + REPAIR_TTL_S + REPAIR_COOLDOWN_S, 3)}
+    return _write(root / REPAIR_FILE, doc)
+
+
+def repair_lines(store, now=None):
+    """Wording constraints for the next reply; one-shot per arming."""
+    root = _root(store)
+    if root is None:
+        return []
+    stamp = float(now or time.time())
+    doc = _read(root / REPAIR_FILE)
+    if not doc.get("armed") or _num(doc.get("used_at")) > 0:
+        return []
+    if stamp > _num(doc.get("expires_at")):
+        doc["armed"] = False
+        _write(root / REPAIR_FILE, doc)
+        return []
+    doc["used_at"] = round(stamp, 3)
+    doc["armed"] = False
+    _write(root / REPAIR_FILE, doc)
+    return ["先修复关系：承认刚才越界或语气不对，不辩解。",
+            "降低对抗：句子短、语气软，先问对方现在的感受。"]
+
+
+def repair_state(store):
+    doc = _read((_root(store) or Path(".")) / REPAIR_FILE)
+    return {"armed": bool(doc.get("armed")),
+            "reason": str(doc.get("reason") or ""),
+            "armed_at": _num(doc.get("armed_at")),
+            "used_at": _num(doc.get("used_at")),
+            "cooldown_until": _num(doc.get("cooldown_until"))}