dsh-biocore 0.1.0 → 0.1.2

package/python/biocore_memory/consolidate/writeback.py

@@ -0,0 +1,181 @@
+"""Write-back bridge for consolidation suggestions (default OFF, idempotent).
+Pure/local: operates on an in-memory MemoryStore-like object (entries/add/update).
+It never saves to disk by itself; the caller decides when to call store.save().
+All failures degrade to skipped entries instead of raising.
+"""
+from __future__ import annotations
+import hashlib
+import json
+WRITEBACK_ENABLED = False
+MAX_RELATED = 8
+REFLECTION_SOURCE = "consolidate"
+TREE_SOURCE = "ti_mem"
+def _digest(text):
+    return hashlib.sha256(str(text or "").encode("utf-8")).hexdigest()[:12]
+def _entries(store):
+    try:
+        rows = getattr(store, "entries", None)
+        if isinstance(rows, list):
+            return rows
+        if isinstance(store, dict):
+            rows = store.get("entries")
+            return rows if isinstance(rows, list) else []
+        return list(store or [])
+    except Exception:
+        return []
+def _find(store, entry_id):
+    try:
+        fn = getattr(store, "find", None)
+        if callable(fn):
+            row = fn(entry_id)
+            if isinstance(row, dict):
+                return row
+    except Exception:
+        pass
+    for entry in _entries(store):
+        if isinstance(entry, dict) and str(entry.get("id") or "") == str(entry_id):
+            return entry
+    return None
+def _add(store, entry):
+    try:
+        fn = getattr(store, "add", None)
+        if callable(fn):
+            return fn(entry)
+        if isinstance(store, dict):
+            store.setdefault("entries", []).append(entry)
+            return entry
+        store.append(entry)
+        return entry
+    except Exception:
+        return None
+def _update(store, entry_id, **fields):
+    try:
+        fn = getattr(store, "update", None)
+        if callable(fn):
+            result = fn(entry_id, **fields)
+            if result is not None:
+                return result
+    except Exception:
+        pass
+    entry = _find(store, entry_id)
+    if isinstance(entry, dict):
+        entry.update(fields)
+        return entry
+    return None
+def _source_ids(proposal):
+    raw = proposal.get("source_ids") if isinstance(proposal, dict) else []
+    return sorted({str(x) for x in (raw or []) if str(x)})
+def _scope(entry):
+    fields = ("session", "scope", "user_id", "agent_id", "app_id", "run_id", "thread")
+    return {k: entry.get(k) for k in fields if entry.get(k) is not None}
+def _importance(store, ids, default=0.5):
+    values = []
+    for sid in ids:
+        row = _find(store, sid)
+        if isinstance(row, dict):
+            try:
+                values.append(float(row.get("importance", 0.5) or 0.5))
+            except Exception:
+                continue
+    return round(sum(values) / len(values), 4) if values else default
+def apply_reflections(store, proposals):
+    """Write reflection proposals as system entries; idempotent by source_ids."""
+    result = {"written": [], "skipped": []}
+    if not WRITEBACK_ENABLED:
+        result["skipped"].append({"reason": "disabled"})
+        return result
+    for idx, proposal in enumerate(proposals or []):
+        try:
+            text = str((proposal or {}).get("text") or "").strip() if isinstance(proposal, dict) else ""
+            ids = _source_ids(proposal)
+            if len(text) < 2 or not ids:
+                result["skipped"].append({"index": idx, "reason": "invalid"})
+                continue
+            duplicate = any(
+                str(e.get("source") or "") == REFLECTION_SOURCE
+                and sorted(str(x) for x in ((e.get("metadata") or {}).get("source_ids") or []))
+                == ids
+                for e in _entries(store) if isinstance(e, dict))
+            if duplicate:
+                result["skipped"].append({"index": idx, "reason": "duplicate"})
+                continue
+            base = {"id": "consolidate_" + _digest(json.dumps([text, ids]))}
+            base.update(_scope(_find(store, ids[0]) or {}))
+            base.update({"text": text, "role": "system", "source": REFLECTION_SOURCE,
+                         "importance": _importance(store, ids),
+                         "metadata": {"source_ids": ids, "origin": "reflection"}})
+            added = _add(store, base)
+            if isinstance(added, dict) and added.get("id"):
+                result["written"].append(str(added.get("id")))
+            else:
+                result["skipped"].append({"index": idx, "reason": "add_failed"})
+        except Exception:
+            result["skipped"].append({"index": idx, "reason": "error"})
+    return result
+def _suggestions(suggestions):
+    if isinstance(suggestions, dict):
+        return [suggestions]
+    return [s for s in (suggestions or []) if isinstance(s, dict)]
+def apply_neighbors(store, suggestions):
+    """Append strengthen() link_ids to related_ids (dedupe, total <=8)."""
+    result = {"linked": 0}
+    if not WRITEBACK_ENABLED:
+        return result
+    for suggestion in _suggestions(suggestions):
+        try:
+            eid = str(suggestion.get("entry_id") or "")
+            entry = _find(store, eid)
+            if not isinstance(entry, dict):
+                continue
+            current = entry.get("related_ids")
+            current = [str(x) for x in current if str(x)] if isinstance(current, list) else []
+            links = [str(x) for x in (suggestion.get("link_ids") or []) if str(x) and str(x) != eid]
+            merged = []
+            for value in current + links:
+                if value not in merged:
+                    merged.append(value)
+            merged = merged[:MAX_RELATED]
+            if merged == current:
+                continue
+            if _update(store, eid, related_ids=merged):
+                result["linked"] += 1
+        except Exception:
+            continue
+    return result
+def _day_of(items):
+    days = [str((x or {}).get("day") or (x or {}).get("key") or "") for x in (items or [])]
+    for value in days:
+        if len(value) >= 10 and value[:4].isdigit():
+            return value[:10]
+    return days[0][:10] if days and days[0] else ""
+def write_tree(store, summary):
+    """Write one tree summary entry (level/day); idempotent by source+level+day."""
+    level = int((summary or {}).get("level") or 1) if isinstance(summary, dict) else 1
+    result = {"written": 0, "level": level}
+    if not WRITEBACK_ENABLED or not isinstance(summary, dict):
+        return result
+    try:
+        items = list(summary.get("items") or [])
+        if not summary.get("generated") or not items:
+            return result
+        day = _day_of(items)
+        duplicate = any(
+            str(e.get("source") or "") == TREE_SOURCE
+            and int((e.get("metadata") or {}).get("level") or 0) == level
+            and str((e.get("metadata") or {}).get("day") or "") == day
+            for e in _entries(store) if isinstance(e, dict))
+        if duplicate:
+            return result
+        text = "时间树 L%d %s：%s" % (
+            level, day, json.dumps(items[:10], ensure_ascii=False, sort_keys=True)[:1000])
+        entry = {"id": TREE_SOURCE + "_" + _digest("%s|%s|%s" % (level, day, text)),
+                 "text": text[:1200], "role": "system", "source": TREE_SOURCE,
+                 "importance": 0.5,
+                 "metadata": {"level": level, "day": day,
+                              "source_ids": [str(x.get("key") or "") for x in items[:8]]}}
+        added = _add(store, entry)
+        if isinstance(added, dict):
+            result["written"] = 1
+        return result
+    except Exception:
+        return result

package/python/biocore_memory/core/__init__.py

@@ -0,0 +1,4 @@
+# OWNER: engine-core
+from .config import *  # noqa
+from .types import MemoryEntry, Cue, EventVector, ThreadCard, RelayCard  # noqa
+from .util import sha12, clamp01, normalize_text, now_ts  # noqa

package/python/biocore_memory/core/config.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+"""All memory-layer thresholds/defaults in one place."""
+from __future__ import annotations
+
+SCHEMA_VERSION = 1
+MAX_VERSIONS = 8
+TOMBSTONE_LIMIT = 100
+DEFAULT_BUDGET = 3
+MAX_BUDGET = 5
+
+# recall fusion / tiers (M2)
+CUE_WEIGHT = 0.50
+SEM_WEIGHT = 0.30
+IMPORTANCE_WEIGHT = 0.10
+TIME_BONUS = 0.10
+HIGH_TIER = 0.55
+MID_TIER = 0.35
+
+# source credibility gate (separate from importance)
+SOURCE_MIN_CONFIDENCE = 0.55
+SOURCE_CONFIDENCE_DEFAULT = 0.70
+
+# memory value formula (deterministic; trust is NOT part of value)
+VALUE_BASE = 0.20
+VALUE_W_INTENSITY = 0.30
+VALUE_W_VALENCE = 0.20
+VALUE_W_SELF = 0.15
+VALUE_W_MAINLINE = 0.15
+VALUE_W_ROUTINE = 0.20
+
+# eviction weights (read-only score)
+EVICT_W_SALIENCE = 0.30
+EVICT_W_POSITION = 0.20
+EVICT_W_REDUNDANCY = 0.20
+EVICT_W_CONFIDENCE = 0.15
+EVICT_W_STALENESS = 0.15
+
+# identity write-back gate
+SAME_DIRECTION_MIN = 3
+MAX_STEP = 0.02
+MAJOR_EVENT_STEP = 0.02
+
+# thread card
+THREAD_TTL = 21600.0
+THREAD_LIMIT = 3
+
+# memory-point gate (OR): value >= MIN_VALUE or importance >= MIN_IMPORTANCE or self >= MIN_SELF
+# (event centrality / emotional intensity; self-reference effect; engineering thresholds)
+MEMORY_POINT_MIN_VALUE = 0.60
+MEMORY_POINT_MIN_SELF = 0.45
+MEMORY_POINT_MIN_IMPORTANCE = 0.75
+
+# engine wiring knobs (all read by MemoryEngine only)
+RATE_LIMIT_PER_MIN = 30
+COMPRESS_MAX_CHARS = 200
+AUDIT_TAIL = 50
+COOC_MAX_ENTRIES = 400
+# after-turn auto consolidation: every N successful remember() calls
+CONSOLIDATE_AUTO_TURNS = 10
+
+
+# Per-mechanism flags: safety/read-path filters default ON; every optional
+# mechanism defaults OFF and must keep the old code path when disabled.
+# A flags.json in the data directory overrides these booleans at engine start
+# (and MemoryEngine.set_flags(..., persist=True) rewrites it).
+FLAGS = {
+    # safety / read-path (default ON)
+    # GA-03: keep engine default ON. DSH private mode disables it per-store via
+    # <dsh store>/flags.json; the shared engine must not be relaxed globally.
+    "security": True,          # store/security.should_reject before write
+    "provenance": True,        # provenance hash tag + chained tombstones
+    "source_filter": True,     # low-confidence candidates are counted/dropped
+    "temporal_filter": True,   # honour valid_at/invalid_at on recall
+    "dedupe": True,            # remember-side L0-L3 dedupe
+    "relay": True,             # relay card
+    "thread": True,            # thread card / TTL expiry
+    "embedding": True,         # optional embedding provider
+    # attention / model features (default OFF)
+    "slots": False,
+    "boundary": False,         # slots/boundary.py event-boundary switching
+    "three_channel": False,
+    "eviction_readonly": False,
+    "identity": False,
+    "identity_writeback": False,
+    "due_hints": False,
+    # recall extensions (default OFF)
+    "bm25": False,
+    "bm25_multi": False,
+    "ppr": False,
+    "mmr": False,
+    "complexity": False,
+    "constraints": False,
+    "tiering": False,
+    "entity_boost": False,
+    "inhibition": False,
+    "retrieval_practice": False,
+    "compress": False,         # output-only summary, never mutates the store
+    "source_consistency": False,  # source-rank marking/downweight on recall
+    # eviction extension modules (default OFF)
+    "heat": False,
+    "consequence": False,
+    "tiering_exec": False,
+    "apply_evictions": False,  # tick() may really soft-delete archived entries
+    "deletion_audit": False,   # derived-data coverage audit after forget()
+    "adaptive_evict": False,   # adaptive_score replaces the tick eviction score
+
+    # write-path extensions (default OFF)
+    "consistency": False,      # old/new fact conflict annotation on remember
+    "rate_limit": False,
+    "audit": False,
+    "permissions": False,
+    "notes": False,
+    "media": False,
+    "notes_link": False,
+    "ledger": False,
+    # index / graph
+    "graph": False,
+    "cue_writeback": True,      # store cue_index fields on each new entry (index-only, no recall change)
+    # consolidation modules (default OFF)
+    "consolidate_auto": False,  # after-turn auto consolidate every CONSOLIDATE_AUTO_TURNS remembers
+    "consolidate_gated": False,
+    "consolidate_tree": False,
+    "consolidate_reflection": False,
+    "consolidate_neighbors": False,
+    "consolidate_writeback": False,
+    "narrative": False,        # identity narrative update at consolidate end
+}

package/python/biocore_memory/core/permissions.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""Scope-based read/write permissions (default OFF; pure, no astrbot)."""
+from __future__ import annotations
+
+enabled = False
+ENABLED = False
+ADMIN_ROLES = ("admin", "owner", "system", "superuser", "root")
+READONLY_ROLES = ("guest", "readonly", "viewer")
+PUBLIC_SCOPES = ("*", "public", "all", "shared")
+
+
+def _principal(principal):
+    value = principal if isinstance(principal, dict) else {}
+    role = str(value.get("role") or "user").strip().lower()
+    identifier = str(value.get("id") or value.get("user_id") or "").strip()
+    scopes = value.get("scopes")
+    if not isinstance(scopes, list):
+        scopes = [value.get("scope")] if value.get("scope") is not None else []
+    return role, identifier, {str(s).strip() for s in scopes if str(s).strip()}
+
+
+def _scopes(entry):
+    if not isinstance(entry, dict):
+        return set()
+    raw = entry.get("scope")
+    if raw is None:
+        raw = entry.get("scopes")
+    if isinstance(raw, str):
+        raw = [raw]
+    if not isinstance(raw, (list, tuple, set)):
+        return set()
+    return {str(value).strip() for value in raw if str(value).strip()}
+
+
+def _can_access(entry, principal):
+    if not enabled:
+        return True
+    role, identifier, principal_scopes = _principal(principal)
+    if role in ADMIN_ROLES:
+        return True
+    scopes = _scopes(entry)
+    if not scopes:
+        return True
+    if scopes & set(PUBLIC_SCOPES):
+        return True
+    if identifier and (identifier in scopes or ("user:" + identifier) in scopes):
+        return True
+    return bool(principal_scopes & scopes)
+
+
+def can_read(entry, principal=None):
+    """True when principal may read entry (disabled -> always True)."""
+    try:
+        return _can_access(entry, principal)
+    except Exception:
+        return not enabled
+
+
+def can_write(entry, principal=None):
+    """True when principal may write entry; readonly roles are denied when on."""
+    try:
+        if not enabled:
+            return True
+        role, _identifier, _scopes_set = _principal(principal)
+        if role in READONLY_ROLES:
+            return False
+        return _can_access(entry, principal)
+    except Exception:
+        return not enabled

package/python/biocore_memory/core/types.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+"""Host-neutral data structures for the memory layer.
+
+Interface summary:
+  MemoryEntry(id, text, role, session, ts, vector, importance, pin, metadata, scope)
+  Cue(entities, place, time, event_name, id)
+  EventVector(intensity, self_relevance, schema_conflict, relation_weight,
+              novelty, unresolved, repetition)
+  ThreadCard(id, text, ts); RelayCard(identity, relationship, open_loops,
+              recent, pins)
+Each type exposes from_dict/to_dict (signatures below); no host imports.
+"""
+from __future__ import annotations
+from dataclasses import asdict, dataclass, field
+
+
+@dataclass
+class MemoryEntry:
+    id: str = ""
+    text: str = ""
+    role: str = "user"
+    session: str = ""
+    ts: float = 0.0
+    vector: list = field(default_factory=list)
+    importance: float = 0.5
+    pin: bool = False
+    metadata: dict = field(default_factory=dict)
+    scope: str = "default"
+
+    @classmethod
+    def from_dict(cls, value):
+        if not isinstance(value, dict):
+            return cls(text=str(value or ""))
+        return cls(**{k: value[k] for k in cls.__dataclass_fields__ if k in value})
+
+    def to_dict(self):
+        out = asdict(self)
+        metadata = out.pop("metadata", {}) or {}
+        out.update(metadata)
+        return out
+
+
+@dataclass
+class Cue:
+    entities: list = field(default_factory=list)
+    place: list = field(default_factory=list)
+    time: list = field(default_factory=list)
+    event_name: str = ""
+    id: str = ""
+
+    @classmethod
+    def from_dict(cls, value):
+        if not isinstance(value, dict):
+            return cls()
+        return cls(**{k: value[k] for k in cls.__dataclass_fields__ if k in value})
+
+    def to_dict(self):
+        return asdict(self)
+
+
+@dataclass
+class EventVector:
+    intensity: float = 0.0
+    self_relevance: float = 0.0
+    schema_conflict: float = 0.0
+    relation_weight: float = 0.0
+    novelty: float = 0.0
+    unresolved: float = 0.0
+    repetition: float = 0.0
+
+    @classmethod
+    def from_dict(cls, value):
+        if not isinstance(value, dict):
+            return cls()
+        return cls(**{k: value[k] for k in cls.__dataclass_fields__ if k in value})
+
+    def to_dict(self):
+        return asdict(self)
+
+    def as_dict(self):
+        """Compatibility alias for to_dict (older call sites)."""
+        return self.to_dict()
+
+
+@dataclass
+class ThreadCard:
+    id: str = ""
+    text: str = ""
+    ts: float = 0.0
+
+    @classmethod
+    def from_dict(cls, value):
+        if not isinstance(value, dict):
+            return cls()
+        return cls(**{k: value[k] for k in cls.__dataclass_fields__ if k in value})
+
+    def to_dict(self):
+        return asdict(self)
+
+
+@dataclass
+class RelayCard:
+    identity: list = field(default_factory=list)
+    relationship: list = field(default_factory=list)
+    open_loops: list = field(default_factory=list)
+    recent: list = field(default_factory=list)
+    pins: list = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, value):
+        if not isinstance(value, dict):
+            return cls()
+        return cls(**{k: value[k] for k in cls.__dataclass_fields__ if k in value})
+
+    def to_dict(self):
+        return asdict(self)

package/python/biocore_memory/core/util.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""Normalization / hash / time helpers (stdlib only)."""
+from __future__ import annotations
+import hashlib, re, time
+
+
+def sha12(text):
+    return hashlib.sha256(str(text).encode("utf-8")).hexdigest()[:12]
+
+
+def clamp01(value, default=0.0):
+    try:
+        return max(0.0, min(1.0, float(value)))
+    except Exception:
+        return default
+
+
+def normalize_text(text):
+    return re.sub(r"\s+", " ", str(text or "")).strip()
+
+
+def now_ts(now=None):
+    try:
+        return float(now if now is not None else time.time())
+    except Exception:
+        return time.time()