dsh-biocore 0.1.0 → 0.1.2

package/python/biocore_memory/co_recall.py

@@ -0,0 +1,324 @@
+# -*- coding: utf-8 -*-
+"""task-187: co-recall statistics -- first step: count and mark, never merge.
+
+Two memories that keep being recalled in the same batch probably point at the
+same thing. This module only keeps the score; it never changes recall, never
+drops an entry, never merges anything and never calls a model:
+
+- pure functions: normalize_id / unique_ids / pair_key / pairs turn one recall
+  batch into unordered id pairs (a duplicate inside one batch counts once);
+- thin storage layer: table co_recall(pair_a, pair_b, count, first_ts, last_ts)
+  in memory/memory.sqlite3, upserted once per batch;
+- candidates(threshold=3, window_days=30): pairs that crossed the threshold
+  inside the recent window (read only);
+- mark_candidates(): writes co_recall_marks (a separate table) so the panel can
+  show "mergeable candidates", without touching entries;
+- read_marked() / stats() / readout(): read-only entry points for the panel or
+  a command. Wiring into main.py is intentionally left for the caller.
+
+Hard boundaries: no LLM call, no change to the recall output or its thresholds,
+no import of main.py. IO lives in the storage layer on purpose.
+"""
+from __future__ import annotations
+
+import sqlite3
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+DEFAULT_THRESHOLD = 3
+DEFAULT_WINDOW_DAYS = 30.0
+SQLITE_FILE = "memory.sqlite3"
+TABLE_RECALL = "co_recall"
+TABLE_MARKS = "co_recall_marks"
+DAY_SECONDS = 86400.0
+MAX_LIMIT = 500
+
+_PLUGIN_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_DB_PATH = _PLUGIN_ROOT / "memory" / SQLITE_FILE
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS co_recall(
+    pair_a TEXT NOT NULL,
+    pair_b TEXT NOT NULL,
+    count INTEGER NOT NULL DEFAULT 0,
+    first_ts REAL NOT NULL,
+    last_ts REAL NOT NULL,
+    PRIMARY KEY (pair_a, pair_b)
+);
+CREATE INDEX IF NOT EXISTS idx_co_recall_last ON co_recall(last_ts);
+CREATE INDEX IF NOT EXISTS idx_co_recall_count ON co_recall(count);
+CREATE TABLE IF NOT EXISTS co_recall_marks(
+    pair_a TEXT NOT NULL,
+    pair_b TEXT NOT NULL,
+    marked_ts REAL NOT NULL,
+    threshold INTEGER NOT NULL,
+    window_days REAL NOT NULL,
+    PRIMARY KEY (pair_a, pair_b)
+);
+"""
+
+
+# ------------------------------------------------------------ pure functions
+def _num(value, default=0.0):
+    try:
+        number = float(value)
+    except Exception:
+        return float(default)
+    if number != number or number in (float("inf"), float("-inf")):
+        return float(default)
+    return number
+
+
+def normalize_id(value):
+    """One memory id as a clean string; containers and blanks become ''."""
+    if value is None or isinstance(value, (dict, list, tuple, set)):
+        return ""
+    return str(value).strip()
+
+
+def unique_ids(ids):
+    """Preserve order, drop blanks and duplicates (one batch, one occurrence)."""
+    seen = []
+    if not isinstance(ids, (list, tuple, set)):
+        return seen
+    for raw in ids:
+        ident = normalize_id(raw)
+        if ident and ident not in seen:
+            seen.append(ident)
+    return seen
+
+
+def pair_key(a, b):
+    """Unordered key (a, b) with a < b; None when it is not a real pair."""
+    left = normalize_id(a)
+    right = normalize_id(b)
+    if not left or not right or left == right:
+        return None
+    return (left, right) if left < right else (right, left)
+
+
+def pairs(ids):
+    """All unordered pairs from one recall batch, sorted and de-duplicated."""
+    row = unique_ids(ids)
+    found = set()
+    for index in range(len(row)):
+        for other in range(index + 1, len(row)):
+            key = pair_key(row[index], row[other])
+            if key is not None:
+                found.add(key)
+    return sorted(found)
+
+
+def qualifies(count, last_ts, threshold=DEFAULT_THRESHOLD,
+              window_days=DEFAULT_WINDOW_DAYS, now=None):
+    """Pure version of the candidates rule (read-only, no storage)."""
+    limit = max(1, int(_num(threshold, DEFAULT_THRESHOLD)))
+    days = _num(window_days, DEFAULT_WINDOW_DAYS)
+    current = _now(now)
+    if int(_num(count, 0.0)) < limit:
+        return False
+    if days <= 0.0:
+        return True
+    return _num(last_ts, 0.0) >= current - days * DAY_SECONDS
+
+
+def _now(now=None):
+    try:
+        return float(now if now is not None else time.time())
+    except Exception:
+        return time.time()
+
+
+# ----------------------------------------------------------- storage layer
+def default_db_path():
+    """Where a plugin data dir keeps memory/memory.sqlite3."""
+    return DEFAULT_DB_PATH
+
+
+def ensure_schema(conn):
+    """Create the two additive tables; safe to call on every open."""
+    conn.executescript(SCHEMA_SQL)
+    return True
+
+
+def connect(db_path=None):
+    """Open (and initialize) the memory DB. Caller owns the connection."""
+    path = DEFAULT_DB_PATH if db_path is None else Path(db_path)
+    if str(path) != ":memory:":
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+        except Exception:
+            pass
+    conn = sqlite3.connect(str(path), timeout=5.0)
+    conn.row_factory = sqlite3.Row
+    try:
+        conn.execute("PRAGMA busy_timeout=5000")
+    except sqlite3.Error:
+        pass
+    ensure_schema(conn)
+    return conn
+
+
+@contextmanager
+def _session(db_path=None, conn=None):
+    owned = conn is None
+    db = conn if conn is not None else connect(db_path)
+    if not owned:
+        ensure_schema(db)
+    try:
+        yield db
+        if owned:
+            db.commit()
+    finally:
+        if owned:
+            db.close()
+
+
+def record_recall(ids, ts=None, db_path=None, conn=None):
+    """Count one recall batch. Unordered pairs; duplicate ids counted once."""
+    batch = pairs(ids)
+    stamp = _now(ts)
+    row = unique_ids(ids)
+    if not batch:
+        return {"pairs": 0, "ids": len(row), "ts": stamp}
+    with _session(db_path, conn) as db:
+        db.executemany(
+            "INSERT INTO co_recall(pair_a, pair_b, count, first_ts, last_ts) "
+            "VALUES(?, ?, 1, ?, ?) "
+            "ON CONFLICT(pair_a, pair_b) DO UPDATE SET "
+            "count = co_recall.count + 1, last_ts = excluded.last_ts",
+            [(left, right, stamp, stamp) for left, right in batch])
+    return {"pairs": len(batch), "ids": len(row), "ts": stamp}
+
+
+def _threshold(value):
+    return max(1, int(_num(value, DEFAULT_THRESHOLD)))
+
+
+def _window_days(value):
+    return max(0.0, _num(value, DEFAULT_WINDOW_DAYS))
+
+
+def _row_dict(row, current):
+    left = str(row["pair_a"])
+    right = str(row["pair_b"])
+    last_ts = _num(row["last_ts"], 0.0)
+    return {
+        "pair": (left, right),
+        "pair_a": left,
+        "pair_b": right,
+        "count": int(_num(row["count"], 0.0)),
+        "first_ts": round(_num(row["first_ts"], 0.0), 3),
+        "last_ts": round(last_ts, 3),
+        "age_days": round(max(0.0, (current - last_ts) / DAY_SECONDS), 2),
+        "marked": bool(row["marked"]),
+    }
+
+
+def candidates(threshold=DEFAULT_THRESHOLD,
+               window_days=DEFAULT_WINDOW_DAYS, now=None, db_path=None,
+               conn=None):
+    """Pairs with count >= threshold whose last co-recall is inside the window."""
+    limit = _threshold(threshold)
+    days = _window_days(window_days)
+    current = _now(now)
+    sql = ("SELECT r.pair_a, r.pair_b, r.count, r.first_ts, r.last_ts, "
+           "CASE WHEN m.pair_a IS NULL THEN 0 ELSE 1 END AS marked "
+           "FROM co_recall r LEFT JOIN co_recall_marks m "
+           "ON m.pair_a = r.pair_a AND m.pair_b = r.pair_b "
+           "WHERE r.count >= ?")
+    args = [limit]
+    if days > 0.0:
+        sql += " AND r.last_ts >= ?"
+        args.append(current - days * DAY_SECONDS)
+    sql += " ORDER BY r.count DESC, r.last_ts DESC, r.pair_a ASC, r.pair_b ASC"
+    with _session(db_path, conn) as db:
+        rows = db.execute(sql, args).fetchall()
+    return [_row_dict(row, current) for row in rows]
+
+
+def mark_candidates(threshold=DEFAULT_THRESHOLD,
+                    window_days=DEFAULT_WINDOW_DAYS, now=None, db_path=None,
+                    conn=None):
+    """Mark over-threshold pairs as mergeable; no deletion, no merging."""
+    limit = _threshold(threshold)
+    days = _window_days(window_days)
+    current = _now(now)
+    rows = candidates(limit, days, current, db_path, conn)
+    if not rows:
+        return {"marked": 0, "threshold": limit, "window_days": days,
+                "pairs": []}
+    with _session(db_path, conn) as db:
+        db.executemany(
+            "INSERT INTO co_recall_marks(pair_a, pair_b, marked_ts, threshold, "
+            "window_days) VALUES(?, ?, ?, ?, ?) "
+            "ON CONFLICT(pair_a, pair_b) DO UPDATE SET "
+            "marked_ts = excluded.marked_ts, threshold = excluded.threshold, "
+            "window_days = excluded.window_days",
+            [(row["pair_a"], row["pair_b"], current, limit, days)
+             for row in rows])
+    return {"marked": len(rows), "threshold": limit, "window_days": days,
+            "pairs": rows}
+
+
+def read_marked(db_path=None, conn=None, limit=None):
+    """Read-only: every marked pair, newest mark first."""
+    sql = ("SELECT m.pair_a, m.pair_b, m.marked_ts, m.threshold, "
+           "m.window_days, COALESCE(r.count, 0) AS count, "
+           "COALESCE(r.first_ts, 0.0) AS first_ts, "
+           "COALESCE(r.last_ts, 0.0) AS last_ts "
+           "FROM co_recall_marks m LEFT JOIN co_recall r "
+           "ON r.pair_a = m.pair_a AND r.pair_b = m.pair_b "
+           "ORDER BY m.marked_ts DESC, m.pair_a ASC, m.pair_b ASC")
+    args = []
+    if limit is not None:
+        size = int(_num(limit, 0.0))
+        if size > 0:
+            sql += " LIMIT ?"
+            args.append(min(size, MAX_LIMIT))
+    with _session(db_path, conn) as db:
+        rows = db.execute(sql, args).fetchall()
+    out = []
+    for row in rows:
+        left = str(row["pair_a"])
+        right = str(row["pair_b"])
+        out.append({
+            "pair": (left, right),
+            "pair_a": left,
+            "pair_b": right,
+            "count": int(_num(row["count"], 0.0)),
+            "marked_ts": round(_num(row["marked_ts"], 0.0), 3),
+            "threshold": int(_num(row["threshold"], DEFAULT_THRESHOLD)),
+            "window_days": round(_num(row["window_days"],
+                                      DEFAULT_WINDOW_DAYS), 2),
+        })
+    return out
+
+
+def stats(db_path=None, conn=None):
+    """Read-only totals: pairs, co-recall events, marked pairs."""
+    with _session(db_path, conn) as db:
+        row = db.execute("SELECT COUNT(*) AS pairs, "
+                         "COALESCE(SUM(count), 0) AS recalls "
+                         "FROM co_recall").fetchone()
+        marked = db.execute("SELECT COUNT(*) AS marked "
+                            "FROM co_recall_marks").fetchone()
+    return {
+        "pairs": int(_num(row["pairs"], 0.0)),
+        "recalls": int(_num(row["recalls"], 0.0)),
+        "marked": int(_num(marked["marked"], 0.0)),
+    }
+
+
+def readout(threshold=DEFAULT_THRESHOLD, window_days=DEFAULT_WINDOW_DAYS,
+            now=None, db_path=None, conn=None, limit=20):
+    """Bounded read-only view for a panel page or a command reply."""
+    limit = max(1, min(int(_num(limit, 20.0)), MAX_LIMIT))
+    rows = candidates(threshold, window_days, now, db_path, conn)
+    return {
+        "threshold": _threshold(threshold),
+        "window_days": round(_window_days(window_days), 2),
+        "stats": stats(db_path, conn),
+        "candidates": rows[:limit],
+    }

package/python/biocore_memory/consolidate/__init__.py

@@ -0,0 +1,11 @@
+# OWNER: engine-core
+# -*- coding: utf-8 -*-
+"""Consolidation mechanisms (default OFF, pure functions, no engine wiring)."""
+from .gated import should_consolidate
+from .tree import build_summary
+from .reflection import propose
+from .neighbors import strengthen
+from .writeback import apply_neighbors, apply_reflections, write_tree
+
+__all__ = ["should_consolidate", "build_summary", "propose", "strengthen",
+           "apply_reflections", "apply_neighbors", "write_tree"]

package/python/biocore_memory/consolidate/gated.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""RecMem-style consolidation gate (pure, read-only, default OFF)."""
+from __future__ import annotations
+
+import time
+
+GATED_ENABLED = False
+WINDOW_SECONDS = 7 * 86400.0
+MIN_TOPIC_REPEATS = 3
+MIN_GAP_SECONDS = 6 * 3600.0
+COSINE_MERGE = 0.80
+MAX_ITEMS = 200
+
+
+def _ts(entry):
+    try:
+        return float(entry.get("ts") or entry.get("created_ts") or 0.0)
+    except Exception:
+        return 0.0
+
+
+def _tags(entry):
+    raw = entry.get("tags")
+    if not isinstance(raw, list):
+        raw = (entry.get("metadata") or {}).get("tags") if isinstance(entry.get("metadata"), dict) else []
+    return {str(x).strip().lower() for x in (raw or []) if str(x).strip()}
+
+
+def _topic(entry):
+    for key in ("topic", "event_key", "event_name", "thread"):
+        value = str(entry.get(key) or "").strip().lower()
+        if value:
+            return value
+    return ""
+
+
+def _vector(entry):
+    vec = entry.get("vector")
+    if isinstance(vec, (list, tuple)) and vec:
+        return list(vec)
+    meta = entry.get("metadata")
+    if isinstance(meta, dict):
+        vec = meta.get("vector")
+        if isinstance(vec, (list, tuple)) and vec:
+            return list(vec)
+    return []
+
+
+def _cosine(a, b):
+    try:
+        if not a or not b or len(a) != len(b):
+            return 0.0
+        dot = sum(float(x) * float(y) for x, y in zip(a, b))
+        na = sum(float(x) ** 2 for x in a) ** 0.5
+        nb = sum(float(y) ** 2 for y in b) ** 0.5
+        return dot / (na * nb) if na and nb else 0.0
+    except Exception:
+        return 0.0
+
+
+def _same_topic(a, b):
+    ta, tb = _tags(a), _tags(b)
+    if ta and tb and (ta & tb):
+        return True
+    topic_a, topic_b = _topic(a), _topic(b)
+    if topic_a and topic_a == topic_b:
+        return True
+    return _cosine(_vector(a), _vector(b)) >= COSINE_MERGE
+
+
+def _last_consolidated(entries):
+    last = 0.0
+    for entry in entries:
+        for key in ("last_consolidated_ts", "consolidated_ts", "consolidate_ts"):
+            try:
+                last = max(last, float(entry.get(key) or 0.0))
+            except Exception:
+                continue
+    return last
+
+
+def should_consolidate(entries, now=None):
+    """True only when >=3 same-topic items in 7d and last pass was >=6h ago."""
+    if not GATED_ENABLED:
+        return False
+    try:
+        rows = [e for e in (entries or []) if isinstance(e, dict)][:MAX_ITEMS]
+        if len(rows) < MIN_TOPIC_REPEATS:
+            return False
+        stamp = float(now if now is not None else time.time())
+        recent = [e for e in rows if 0.0 <= stamp - _ts(e) <= WINDOW_SECONDS]
+        if len(recent) < MIN_TOPIC_REPEATS:
+            return False
+        last = _last_consolidated(recent)
+        if last and stamp - last < MIN_GAP_SECONDS:
+            return False
+        for i, entry in enumerate(recent):
+            same = 1 + sum(1 for j, other in enumerate(recent)
+                           if j != i and _same_topic(entry, other))
+            if same >= MIN_TOPIC_REPEATS:
+                return True
+        return False
+    except Exception:
+        return False

package/python/biocore_memory/consolidate/neighbors.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""A-MEM-style neighbor strengthening proposals (read-only)."""
+from __future__ import annotations
+
+NEIGHBORS_ENABLED = False
+COSINE_LINK = 0.80
+MAX_LINKS = 5
+
+
+def _tags(entry):
+    raw = entry.get("tags")
+    if not isinstance(raw, list):
+        raw = (entry.get("metadata") or {}).get("tags") if isinstance(entry.get("metadata"), dict) else []
+    return {str(x).strip().lower() for x in (raw or []) if str(x).strip()}
+
+
+def _topic(entry):
+    for key in ("topic", "event_key", "event_name"):
+        value = str(entry.get(key) or "").strip().lower()
+        if value:
+            return value
+    return ""
+
+
+def _cosine(a, b):
+    try:
+        if not a or not b or len(a) != len(b):
+            return 0.0
+        dot = sum(float(x) * float(y) for x, y in zip(a, b))
+        na = sum(float(x) ** 2 for x in a) ** 0.5
+        nb = sum(float(y) ** 2 for y in b) ** 0.5
+        return dot / (na * nb) if na and nb else 0.0
+    except Exception:
+        return 0.0
+
+
+def strengthen(entry, neighbors):
+    """Suggest bounded links for an entry; returns {entry_id, link_ids, weight}."""
+    result = {"entry_id": str((entry or {}).get("id") or "") if isinstance(entry, dict) else "",
+              "link_ids": [], "weight": 0.0}
+    if not NEIGHBORS_ENABLED or not isinstance(entry, dict):
+        return result
+    try:
+        base_tags, base_topic = _tags(entry), _topic(entry)
+        base_vec = entry.get("vector") if isinstance(entry.get("vector"), list) else []
+        scored = []
+        for other in neighbors or []:
+            if not isinstance(other, dict):
+                continue
+            oid = str(other.get("id") or "")
+            if not oid or oid == result["entry_id"]:
+                continue
+            sim = _cosine(base_vec, other.get("vector") or [])
+            if base_tags & _tags(other):
+                sim = max(sim, 0.80)
+            if base_topic and base_topic == _topic(other):
+                sim = max(sim, 0.80)
+            if sim >= COSINE_LINK:
+                scored.append((sim, oid))
+        scored.sort(key=lambda row: (-row[0], row[1]))
+        links = [oid for _sim, oid in scored[:MAX_LINKS]]
+        weight = round(max(0.0, min(1.0, sum(s for s, _ in scored[:MAX_LINKS])
+                                      / max(1, len(links)))), 4) if links else 0.0
+        result["link_ids"] = links
+        result["weight"] = weight
+        return result
+    except Exception:
+        return result

package/python/biocore_memory/consolidate/reflection.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""Reflection stage: propose experience candidates from repeated events (no writes)."""
+from __future__ import annotations
+
+REFLECTION_ENABLED = False
+MIN_REPEATS = 2
+MAX_PROPOSALS = 5
+MAX_TEXT = 120
+
+
+def _key(entry):
+    for key in ("event_key", "topic", "event_name"):
+        value = str(entry.get(key) or "").strip()
+        if value:
+            return value[:60]
+    return " ".join(str(entry.get("text") or "").split())[:60]
+
+
+def propose(entries):
+    """Return [{text, source_ids}] experience candidates; never writes storage."""
+    if not REFLECTION_ENABLED:
+        return []
+    try:
+        groups = {}
+        for entry in entries or []:
+            if not isinstance(entry, dict):
+                continue
+            key = _key(entry)
+            if key:
+                groups.setdefault(key, []).append(entry)
+        out = []
+        for key, items in sorted(groups.items(), key=lambda kv: -len(kv[1])):
+            if len(items) < MIN_REPEATS:
+                continue
+            sample = " ".join(str(items[-1].get("text") or "").split())[:60]
+            text = "经验：关于「%s」的重复经历共 %d 次；最近一次：%s" % (
+                key[:40], len(items), sample)
+            out.append({"text": text[:MAX_TEXT],
+                        "source_ids": [str(x.get("id") or "") for x in items[:8]
+                                       if str(x.get("id") or "")]})
+            if len(out) >= MAX_PROPOSALS:
+                break
+        return out
+    except Exception:
+        return []

package/python/biocore_memory/consolidate/tree.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""TiMem-style three-level temporal tree summaries (pure, no writes)."""
+from __future__ import annotations
+
+import time
+
+TREE_ENABLED = False
+THRESHOLD_L1 = 10
+THRESHOLD_L2 = 5
+THRESHOLD_L3 = 3
+MAX_SOURCE_IDS = 8
+
+
+def _ts(entry):
+    try:
+        return float(entry.get("ts") or 0.0)
+    except Exception:
+        return 0.0
+
+
+def _day(entry):
+    try:
+        return time.strftime("%Y-%m-%d", time.gmtime(_ts(entry)))
+    except Exception:
+        return ""
+
+
+def _key(entry):
+    for key in ("event_key", "topic", "event_name", "thread"):
+        value = str(entry.get(key) or "").strip()
+        if value:
+            return value[:60]
+    text = " ".join(str(entry.get("text") or "").split())
+    return text[:40] or "untitled"
+
+
+def _raw(rows):
+    return [{"id": str(e.get("id") or ""), "text": str(e.get("text") or "")[:120],
+             "ts": _ts(e)} for e in rows]
+
+
+def _group(rows, key_fn):
+    groups = {}
+    for entry in rows:
+        key = str(key_fn(entry) or "")
+        if not key:
+            continue
+        groups.setdefault(key, []).append(entry)
+    out = []
+    for key, items in groups.items():
+        items = sorted(items, key=_ts)
+        out.append({"key": key, "count": len(items),
+                    "source_ids": [str(x.get("id") or "") for x in items[:MAX_SOURCE_IDS]
+                                   if str(x.get("id") or "")],
+                    "ts_min": _ts(items[0]), "ts_max": _ts(items[-1]),
+                    "sample": str(items[-1].get("text") or "")[:120]})
+    out.sort(key=lambda row: (-row["count"], row["key"]))
+    return out
+
+
+def build_summary(entries, level=1):
+    """Build a level summary only when its threshold is exceeded.
+
+    Failure/disabled returns the original rows unchanged (items + generated=False).
+    """
+    try:
+        rows = [e for e in (entries or []) if isinstance(e, dict)]
+    except Exception:
+        rows = []
+    try:
+        lvl = int(level if level is not None else 1)
+    except Exception:
+        lvl = 1
+    base = {"level": lvl, "items": _raw(rows), "generated": False,
+            "enabled": bool(TREE_ENABLED)}
+    if not TREE_ENABLED:
+        return base
+    try:
+        if lvl <= 1:
+            if len(rows) <= THRESHOLD_L1:
+                return base
+            return {"level": 1, "items": _raw(rows)[:THRESHOLD_L1],
+                    "generated": True, "enabled": True, "truncated": len(rows)}
+        if lvl == 2:
+            if len(rows) <= THRESHOLD_L2:
+                return base
+            return {"level": 2, "items": _group(rows, _key),
+                    "generated": True, "enabled": True}
+        if len(rows) <= THRESHOLD_L3:
+            return base
+        return {"level": 3, "items": _group(rows, _day),
+                "generated": True, "enabled": True}
+    except Exception:
+        return {"level": lvl, "items": _raw(rows), "generated": False,
+                "enabled": bool(TREE_ENABLED), "error": "summary_failed"}