dsh-biocore 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/py/bridge.py +15 -7
- package/lib/py-bridge.js +4 -3
- package/package.json +3 -2
- package/python/biocore_memory/__init__.py +21 -0
- package/python/biocore_memory/adapters/__init__.py +2 -0
- package/python/biocore_memory/adapters/astrbot.py +24 -0
- package/python/biocore_memory/adapters/dsh_cli.py +335 -0
- package/python/biocore_memory/co_recall.py +324 -0
- package/python/biocore_memory/consolidate/__init__.py +11 -0
- package/python/biocore_memory/consolidate/gated.py +104 -0
- package/python/biocore_memory/consolidate/neighbors.py +68 -0
- package/python/biocore_memory/consolidate/reflection.py +45 -0
- package/python/biocore_memory/consolidate/tree.py +95 -0
- package/python/biocore_memory/consolidate/writeback.py +181 -0
- package/python/biocore_memory/core/__init__.py +4 -0
- package/python/biocore_memory/core/config.py +127 -0
- package/python/biocore_memory/core/permissions.py +69 -0
- package/python/biocore_memory/core/types.py +116 -0
- package/python/biocore_memory/core/util.py +26 -0
- package/python/biocore_memory/embedding_local.py +447 -0
- package/python/biocore_memory/embedding_mode.py +156 -0
- package/python/biocore_memory/engine.py +2476 -0
- package/python/biocore_memory/engine_helpers.py +266 -0
- package/python/biocore_memory/event/__init__.py +3 -0
- package/python/biocore_memory/event/_types.py +16 -0
- package/python/biocore_memory/event/consistency.py +117 -0
- package/python/biocore_memory/event/extract.py +48 -0
- package/python/biocore_memory/event/media.py +111 -0
- package/python/biocore_memory/event/notes.py +85 -0
- package/python/biocore_memory/event/notes_link.py +87 -0
- package/python/biocore_memory/event/vector.py +69 -0
- package/python/biocore_memory/evict/__init__.py +3 -0
- package/python/biocore_memory/evict/adaptive.py +73 -0
- package/python/biocore_memory/evict/consequence.py +58 -0
- package/python/biocore_memory/evict/heat.py +54 -0
- package/python/biocore_memory/evict/inhibition.py +105 -0
- package/python/biocore_memory/evict/policy.py +188 -0
- package/python/biocore_memory/evict/score.py +23 -0
- package/python/biocore_memory/identity/__init__.py +6 -0
- package/python/biocore_memory/identity/api.py +99 -0
- package/python/biocore_memory/identity/history.py +9 -0
- package/python/biocore_memory/identity/ledger.py +191 -0
- package/python/biocore_memory/identity/narrative.py +358 -0
- package/python/biocore_memory/identity/snapshot.py +13 -0
- package/python/biocore_memory/identity/writeback.py +61 -0
- package/python/biocore_memory/index/__init__.py +3 -0
- package/python/biocore_memory/index/cue_index.py +80 -0
- package/python/biocore_memory/index/entities.py +85 -0
- package/python/biocore_memory/index/entity_boost.py +104 -0
- package/python/biocore_memory/index/graph.py +172 -0
- package/python/biocore_memory/recall/__init__.py +5 -0
- package/python/biocore_memory/recall/api.py +65 -0
- package/python/biocore_memory/recall/bm25.py +118 -0
- package/python/biocore_memory/recall/channels.py +98 -0
- package/python/biocore_memory/recall/complexity.py +62 -0
- package/python/biocore_memory/recall/constraints.py +75 -0
- package/python/biocore_memory/recall/embedding.py +745 -0
- package/python/biocore_memory/recall/expand.py +26 -0
- package/python/biocore_memory/recall/filters.py +40 -0
- package/python/biocore_memory/recall/fusion.py +18 -0
- package/python/biocore_memory/recall/mmr.py +70 -0
- package/python/biocore_memory/recall/ppr.py +85 -0
- package/python/biocore_memory/recall/retrieval_practice.py +102 -0
- package/python/biocore_memory/relay/__init__.py +2 -0
- package/python/biocore_memory/relay/card.py +51 -0
- package/python/biocore_memory/slots/__init__.py +18 -0
- package/python/biocore_memory/slots/admission.py +64 -0
- package/python/biocore_memory/slots/boundary.py +128 -0
- package/python/biocore_memory/slots/evict_bridge.py +106 -0
- package/python/biocore_memory/slots/registry.py +135 -0
- package/python/biocore_memory/slots/switch.py +42 -0
- package/python/biocore_memory/store/__init__.py +4 -0
- package/python/biocore_memory/store/audit.py +74 -0
- package/python/biocore_memory/store/compress.py +81 -0
- package/python/biocore_memory/store/dedupe.py +178 -0
- package/python/biocore_memory/store/json_store.py +154 -0
- package/python/biocore_memory/store/media.py +145 -0
- package/python/biocore_memory/store/provenance.py +73 -0
- package/python/biocore_memory/store/rate_limit.py +93 -0
- package/python/biocore_memory/store/security.py +107 -0
- package/python/biocore_memory/store/source_consistency.py +123 -0
- package/python/biocore_memory/store/tiering.py +61 -0
- package/python/biocore_memory/store/tiering_exec.py +50 -0
- package/python/biocore_memory/store/tombstone.py +23 -0
- package/python/biocore_memory/store/versions.py +69 -0
- package/python/biocore_memory/thread/__init__.py +2 -0
- package/python/biocore_memory/thread/card.py +58 -0
- package/python/biocore_model/__init__.py +37 -0
- package/python/biocore_model/affective_labor.py +162 -0
- package/python/biocore_model/appraisal.py +257 -0
- package/python/biocore_model/appraisal_action.py +78 -0
- package/python/biocore_model/archive_queue.py +60 -0
- package/python/biocore_model/attachment_behavior.py +130 -0
- package/python/biocore_model/attachment_regulation.py +75 -0
- package/python/biocore_model/attachment_state.py +41 -0
- package/python/biocore_model/attribution.py +73 -0
- package/python/biocore_model/attributor.py +162 -0
- package/python/biocore_model/autonomic.py +91 -0
- package/python/biocore_model/autonomy_affect.py +51 -0
- package/python/biocore_model/autonomy_check.py +67 -0
- package/python/biocore_model/autonomy_loop.py +122 -0
- package/python/biocore_model/autonomy_mode.py +114 -0
- package/python/biocore_model/autonomy_model.py +733 -0
- package/python/biocore_model/autonomy_params.py +131 -0
- package/python/biocore_model/autonomy_reason.py +73 -0
- package/python/biocore_model/autonomy_tick.py +227 -0
- package/python/biocore_model/bio_appraisal.py +49 -0
- package/python/biocore_model/bio_autonomy_state.py +171 -0
- package/python/biocore_model/bio_config.py +221 -0
- package/python/biocore_model/bio_const.py +128 -0
- package/python/biocore_model/bio_daily.py +267 -0
- package/python/biocore_model/bio_emotion.py +538 -0
- package/python/biocore_model/bio_emotion2.py +214 -0
- package/python/biocore_model/bio_goal.py +220 -0
- package/python/biocore_model/bio_identity.py +82 -0
- package/python/biocore_model/bio_noise.py +119 -0
- package/python/biocore_model/bio_physiology2.py +114 -0
- package/python/biocore_model/bio_snapshot.py +324 -0
- package/python/biocore_model/bio_state.py +272 -0
- package/python/biocore_model/bio_tick.py +310 -0
- package/python/biocore_model/bio_util.py +82 -0
- package/python/biocore_model/blocks.py +264 -0
- package/python/biocore_model/boundary.py +251 -0
- package/python/biocore_model/boundary_gate.py +188 -0
- package/python/biocore_model/commitment_extractor.py +517 -0
- package/python/biocore_model/compassion_fatigue.py +84 -0
- package/python/biocore_model/connection_bids.py +120 -0
- package/python/biocore_model/core_bridge.py +177 -0
- package/python/biocore_model/core_client.py +127 -0
- package/python/biocore_model/core_commands.py +112 -0
- package/python/biocore_model/core_mode.py +104 -0
- package/python/biocore_model/core_mode_ui.mjs +104 -0
- package/python/biocore_model/criticality.py +81 -0
- package/python/biocore_model/defense_mechanisms.py +272 -0
- package/python/biocore_model/delta_budget.py +99 -0
- package/python/biocore_model/diag_trace.py +274 -0
- package/python/biocore_model/dissonance.py +134 -0
- package/python/biocore_model/emotion_geometry.py +108 -0
- package/python/biocore_model/emotion_probe.py +209 -0
- package/python/biocore_model/entity_graph.py +931 -0
- package/python/biocore_model/entity_resolve.py +260 -0
- package/python/biocore_model/envy.py +193 -0
- package/python/biocore_model/exit_or_silence.py +230 -0
- package/python/biocore_model/expr_guard.py +24 -0
- package/python/biocore_model/expr_intent.py +77 -0
- package/python/biocore_model/expr_state.py +204 -0
- package/python/biocore_model/expr_template.py +105 -0
- package/python/biocore_model/expr_tone.py +81 -0
- package/python/biocore_model/extreme_scenarios.py +205 -0
- package/python/biocore_model/feedback_flow.py +227 -0
- package/python/biocore_model/feedback_guard.py +129 -0
- package/python/biocore_model/feedback_llm.py +163 -0
- package/python/biocore_model/feedback_memory.py +96 -0
- package/python/biocore_model/feedback_user.py +93 -0
- package/python/biocore_model/frustration_curve.py +110 -0
- package/python/biocore_model/granularity.py +283 -0
- package/python/biocore_model/graph_extract.py +248 -0
- package/python/biocore_model/graph_recall.py +221 -0
- package/python/biocore_model/intent_render.py +91 -0
- package/python/biocore_model/intero_core.py +235 -0
- package/python/biocore_model/intero_style.py +116 -0
- package/python/biocore_model/layered_memory.py +297 -0
- package/python/biocore_model/learning_probe.py +120 -0
- package/python/biocore_model/memory_dual.py +224 -0
- package/python/biocore_model/memory_forgetting.py +125 -0
- package/python/biocore_model/memory_store.py +1399 -0
- package/python/biocore_model/metacog.py +153 -0
- package/python/biocore_model/migrate.py +733 -0
- package/python/biocore_model/migrate_store.py +337 -0
- package/python/biocore_model/mind_wander.py +111 -0
- package/python/biocore_model/mood.py +48 -0
- package/python/biocore_model/narrative.py +172 -0
- package/python/biocore_model/needs_esteem.py +116 -0
- package/python/biocore_model/needs_events.py +137 -0
- package/python/biocore_model/needs_sleep.py +72 -0
- package/python/biocore_model/noise_bank.py +95 -0
- package/python/biocore_model/persona_flow.mjs +25 -0
- package/python/biocore_model/pipeline.py +240 -0
- package/python/biocore_model/pipeline_api.py +200 -0
- package/python/biocore_model/plugin_config.py +83 -0
- package/python/biocore_model/plugin_routes.py +1435 -0
- package/python/biocore_model/policy_learning.py +132 -0
- package/python/biocore_model/possible_selves.py +110 -0
- package/python/biocore_model/predictor.py +178 -0
- package/python/biocore_model/preset.py +205 -0
- package/python/biocore_model/proactive_context.py +153 -0
- package/python/biocore_model/prompt_rules.py +90 -0
- package/python/biocore_model/prospective.py +183 -0
- package/python/biocore_model/provider_fallback.py +97 -0
- package/python/biocore_model/reconstruction.py +132 -0
- package/python/biocore_model/regulation_loop.py +179 -0
- package/python/biocore_model/regulation_strategies.py +294 -0
- package/python/biocore_model/relation_stage.py +130 -0
- package/python/biocore_model/relationship_stage.py +95 -0
- package/python/biocore_model/render.py +901 -0
- package/python/biocore_model/render_guard.mjs +34 -0
- package/python/biocore_model/repair_after_boundary.py +350 -0
- package/python/biocore_model/reply_limit.py +183 -0
- package/python/biocore_model/reply_split.py +203 -0
- package/python/biocore_model/rollback.py +143 -0
- package/python/biocore_model/runtime.py +479 -0
- package/python/biocore_model/sanitize.py +200 -0
- package/python/biocore_model/scheduler.py +123 -0
- package/python/biocore_model/schema.py +120 -0
- package/python/biocore_model/secrecy.py +145 -0
- package/python/biocore_model/self_esteem.py +50 -0
- package/python/biocore_model/self_intent.py +437 -0
- package/python/biocore_model/self_interest.py +263 -0
- package/python/biocore_model/self_verification.py +178 -0
- package/python/biocore_model/services.py +78 -0
- package/python/biocore_model/situation.py +254 -0
- package/python/biocore_model/sleep_circadian.py +116 -0
- package/python/biocore_model/social_pain.py +69 -0
- package/python/biocore_model/social_repair.py +111 -0
- package/python/biocore_model/social_theory.py +131 -0
- package/python/biocore_model/source_monitoring.py +292 -0
- package/python/biocore_model/stages.py +487 -0
- package/python/biocore_model/state_tag.py +231 -0
- package/python/biocore_model/switch_groups.mjs +201 -0
- package/python/biocore_model/tabs_state.mjs +37 -0
- package/python/biocore_model/template_expr.py +252 -0
- package/python/biocore_model/tools/caps_check.mjs +23 -0
- package/python/biocore_model/tools/init_symbols_check.mjs +11 -0
- package/python/biocore_model/tools/missing_ids.mjs +16 -0
- package/python/biocore_model/tools/scope_check.mjs +19 -0
- package/python/biocore_model/topic_control.py +268 -0
- package/python/biocore_model/turn_memory.py +152 -0
- package/python/biocore_model/ui_compat.mjs +48 -0
- package/python/biocore_model/ui_settings.py +266 -0
- package/python/biocore_model/ui_settings.py.bak_authority +108 -0
- package/python/biocore_model/value_order.py +186 -0
- package/python/biocore_model/view_mode.mjs +8 -0
- package/python/biocore_model/voice_bank.py +177 -0
- package/python/biocore_model/waiting_sense.py +229 -0
- package/python/biocore_model/x_directives.py +142 -0
- package/python/biocore_runtime/__init__.py +7 -0
- package/python/biocore_runtime/pipeline.py +1915 -0
- package/python/dsh_adapter/README.md +96 -0
- package/python/dsh_adapter/adapter.py +1077 -0
- package/python/dsh_adapter/cli.py +107 -0
- package/python/dsh_adapter/data/demo_state.json +164 -0
- package/python/dsh_adapter/data/verify_state.json +164 -0
- package/python/dsh_adapter/tests/__init__.py +1 -0
- package/python/dsh_adapter/tests/test_adapter.py +494 -0
- package/ui/panel-layout.js +3 -3
- package/ui/panel.css +53 -6
- package/ui/panel.html +33 -32
- package/ui/panel.js +9 -1
- package/lib/py/__pycache__/bridge.cpython-310.pyc +0 -0
|
@@ -0,0 +1,2476 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""MemoryEngine: orchestration only.
|
|
3
|
+
|
|
4
|
+
Owns the public API and wires every optional mechanism behind a boolean flag
|
|
5
|
+
declared in core/config.py FLAGS. A flags.json in the data directory overrides
|
|
6
|
+
those booleans at engine start, so "change the flag, change the behaviour".
|
|
7
|
+
|
|
8
|
+
Rules:
|
|
9
|
+
- orchestration only: no algorithm thresholds live here (core/config.py);
|
|
10
|
+
- never imports astrbot and never writes prompt text;
|
|
11
|
+
- every optional mechanism defaults OFF and its disabled path is the old code;
|
|
12
|
+
- optional sub-modules are imported lazily and degrade silently when missing,
|
|
13
|
+
so a partially landed tree still runs.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from .core.config import (COMPRESS_MAX_CHARS, CONSOLIDATE_AUTO_TURNS,
|
|
23
|
+
DEFAULT_BUDGET, FLAGS, MEMORY_POINT_MIN_IMPORTANCE,
|
|
24
|
+
MEMORY_POINT_MIN_SELF, MEMORY_POINT_MIN_VALUE,
|
|
25
|
+
RATE_LIMIT_PER_MIN)
|
|
26
|
+
from .core.util import clamp01, now_ts, sha12
|
|
27
|
+
from .engine_helpers import (annotate_tiers, check_after_remember,
|
|
28
|
+
cosine_vectors, explicit_number,
|
|
29
|
+
is_protected_entry, slots_ingest,
|
|
30
|
+
source_filtered_count, value_of, vector_for)
|
|
31
|
+
from .store.json_store import MemoryStore
|
|
32
|
+
from .event.extract import event_from_any
|
|
33
|
+
from .event._types import MemoryEvent
|
|
34
|
+
from .index import cue_score, cues as cue_extract
|
|
35
|
+
from .recall import recall_entries
|
|
36
|
+
from .evict import eviction_order, eviction_score
|
|
37
|
+
from .identity import identity_snapshot as _identity_snapshot, is_identity
|
|
38
|
+
from .identity.api import write as _identity_write
|
|
39
|
+
from .relay import relay_card as _relay_card
|
|
40
|
+
from .thread import expire_threads, thread_card as _thread_card, update_thread
|
|
41
|
+
|
|
42
|
+
_EMPTY_INDEX = ("entities", "place", "time_coarse", "event_name", "id")
|
|
43
|
+
_SUMMARY_SOURCES = ("consolidate", "ti_mem", "reflection")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def is_memory_point(value, importance, self_relevance):
|
|
47
|
+
"""Memory-point gate (OR semantics, literature-backed).
|
|
48
|
+
|
|
49
|
+
Any single strong signal is enough:
|
|
50
|
+
- value >= 0.60 -> composite value formula
|
|
51
|
+
- importance >= 0.75 -> event centrality / emotional intensity
|
|
52
|
+
(Berntsen & Rubin 2006; Talarico et al. 2004)
|
|
53
|
+
- self_relevance >= 0.45 -> self-reference effect
|
|
54
|
+
(Rogers et al. 1977; Symons & Johnson 1997)
|
|
55
|
+
Thresholds are engineering values, not paper parameters.
|
|
56
|
+
"""
|
|
57
|
+
def _num(raw):
|
|
58
|
+
try:
|
|
59
|
+
return float(raw or 0.0)
|
|
60
|
+
except Exception:
|
|
61
|
+
return 0.0
|
|
62
|
+
return bool(_num(value) >= MEMORY_POINT_MIN_VALUE
|
|
63
|
+
or _num(importance) >= MEMORY_POINT_MIN_IMPORTANCE
|
|
64
|
+
or _num(self_relevance) >= MEMORY_POINT_MIN_SELF)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class MemoryEngine:
|
|
68
|
+
def __init__(self, data_dir=None, affect=None, embedder=None, store=None,
|
|
69
|
+
path=None, now=None, three_channel=None,
|
|
70
|
+
eviction_readonly=None, identity_writeback=None,
|
|
71
|
+
config=None, slots_enabled=None, due_hints_enabled=None):
|
|
72
|
+
self.affect = affect
|
|
73
|
+
self.config = config if isinstance(config, dict) else {}
|
|
74
|
+
self.flags = dict(FLAGS)
|
|
75
|
+
overrides = (self.config.get("flags")
|
|
76
|
+
if isinstance(self.config.get("flags"), dict) else {})
|
|
77
|
+
for key, value in overrides.items():
|
|
78
|
+
if isinstance(value, bool) or (
|
|
79
|
+
str(key) == "consolidate_auto_turns"
|
|
80
|
+
and isinstance(value, (int, float))
|
|
81
|
+
and not isinstance(value, bool)):
|
|
82
|
+
self.flags[str(key)] = value
|
|
83
|
+
_dir = Path(data_dir) if data_dir is not None else None
|
|
84
|
+
if _dir is None and path is not None:
|
|
85
|
+
_dir = Path(path).parent
|
|
86
|
+
self._data_dir = _dir
|
|
87
|
+
self._flag_file = (_dir / "flags.json") if _dir is not None else None
|
|
88
|
+
self._load_flag_file()
|
|
89
|
+
self.principal = (self.config.get("principal")
|
|
90
|
+
if isinstance(self.config.get("principal"), dict) else None)
|
|
91
|
+
if embedder is None and self.flags.get("embedding", True):
|
|
92
|
+
try:
|
|
93
|
+
from .recall.embedding import load_provider
|
|
94
|
+
embedder = load_provider(self.config)
|
|
95
|
+
except Exception:
|
|
96
|
+
embedder = None
|
|
97
|
+
self.embedder = embedder
|
|
98
|
+
self.three_channel = (bool(three_channel) if three_channel is not None
|
|
99
|
+
else bool(self.flags.get("three_channel", False)))
|
|
100
|
+
self.eviction_readonly = (bool(eviction_readonly)
|
|
101
|
+
if eviction_readonly is not None
|
|
102
|
+
else bool(self.flags.get("eviction_readonly", False)))
|
|
103
|
+
self.identity_writeback_enabled = (
|
|
104
|
+
bool(identity_writeback) if identity_writeback is not None
|
|
105
|
+
else bool(self.flags.get("identity_writeback",
|
|
106
|
+
self.flags.get("identity", False))))
|
|
107
|
+
self.slots_enabled = (bool(slots_enabled) if slots_enabled is not None
|
|
108
|
+
else bool(self.flags.get("slots", False)))
|
|
109
|
+
self.due_hints_enabled = (bool(due_hints_enabled) if due_hints_enabled is not None
|
|
110
|
+
else bool(self.flags.get("due_hints", False)))
|
|
111
|
+
self.slot_registry = None
|
|
112
|
+
if self.slots_enabled:
|
|
113
|
+
try:
|
|
114
|
+
from .slots import SlotRegistry
|
|
115
|
+
self.slot_registry = SlotRegistry(enabled=True)
|
|
116
|
+
except Exception:
|
|
117
|
+
self.slot_registry = None
|
|
118
|
+
self._last_slots = None
|
|
119
|
+
self._last_recall_tiers = None
|
|
120
|
+
self._identity_state = {}
|
|
121
|
+
self._threads = {}
|
|
122
|
+
self._open_loops = []
|
|
123
|
+
self._last_dedupe = None
|
|
124
|
+
self._last_identity_check = None
|
|
125
|
+
self._last_security = None
|
|
126
|
+
self._last_compress = None
|
|
127
|
+
self._last_retrieval = None
|
|
128
|
+
self._last_ledger = None
|
|
129
|
+
self._last_constraint_hint = None
|
|
130
|
+
self._audit = None
|
|
131
|
+
self._rate_limiters = {}
|
|
132
|
+
self._cooc = {}
|
|
133
|
+
self._last_graph_edge_count = 0
|
|
134
|
+
self._last_slots_msg = ""
|
|
135
|
+
self._last_slots_ts = 0.0
|
|
136
|
+
self._last_evictions = None
|
|
137
|
+
self._last_deletion_audit = None
|
|
138
|
+
self._last_consistency = None
|
|
139
|
+
self._last_source_consistency = None
|
|
140
|
+
self._last_narrative = None
|
|
141
|
+
self._remember_turns = 0
|
|
142
|
+
self._last_auto_consolidate = None
|
|
143
|
+
if store is not None:
|
|
144
|
+
self.store = store
|
|
145
|
+
else:
|
|
146
|
+
if path is None and data_dir is not None:
|
|
147
|
+
path = Path(data_dir) / "memory.json"
|
|
148
|
+
self.store = MemoryStore.load(path) if path else MemoryStore()
|
|
149
|
+
self.now = now if callable(now) else (lambda: time.time())
|
|
150
|
+
|
|
151
|
+
# -- flags ------------------------------------------------------------
|
|
152
|
+
def _flag(self, name, default=False):
|
|
153
|
+
try:
|
|
154
|
+
return bool(self.flags.get(name, default))
|
|
155
|
+
except Exception:
|
|
156
|
+
return bool(default)
|
|
157
|
+
|
|
158
|
+
def _load_flag_file(self):
|
|
159
|
+
if self._flag_file is None:
|
|
160
|
+
return False
|
|
161
|
+
try:
|
|
162
|
+
import json as _json
|
|
163
|
+
if not self._flag_file.exists():
|
|
164
|
+
return False
|
|
165
|
+
data = _json.loads(self._flag_file.read_text(encoding="utf-8"))
|
|
166
|
+
except Exception:
|
|
167
|
+
return False
|
|
168
|
+
if not isinstance(data, dict):
|
|
169
|
+
return False
|
|
170
|
+
changed = False
|
|
171
|
+
for key, value in data.items():
|
|
172
|
+
if isinstance(value, bool) or (
|
|
173
|
+
str(key) == "consolidate_auto_turns"
|
|
174
|
+
and isinstance(value, (int, float))
|
|
175
|
+
and not isinstance(value, bool)):
|
|
176
|
+
self.flags[str(key)] = value
|
|
177
|
+
changed = True
|
|
178
|
+
return changed
|
|
179
|
+
|
|
180
|
+
def reload_flags(self):
|
|
181
|
+
"""Re-read flags.json and refresh derived switches at runtime."""
|
|
182
|
+
self._load_flag_file()
|
|
183
|
+
self._sync_attributes()
|
|
184
|
+
return dict(self.flags)
|
|
185
|
+
|
|
186
|
+
def set_flags(self, overrides, persist=False):
|
|
187
|
+
"""Apply boolean flag overrides; optionally rewrite flags.json."""
|
|
188
|
+
changed = {}
|
|
189
|
+
if isinstance(overrides, dict):
|
|
190
|
+
for key, value in overrides.items():
|
|
191
|
+
if isinstance(value, bool) or (
|
|
192
|
+
str(key) == "consolidate_auto_turns"
|
|
193
|
+
and isinstance(value, (int, float))
|
|
194
|
+
and not isinstance(value, bool)):
|
|
195
|
+
self.flags[str(key)] = value
|
|
196
|
+
changed[str(key)] = value
|
|
197
|
+
self._sync_attributes()
|
|
198
|
+
if persist and self._flag_file is not None and changed:
|
|
199
|
+
try:
|
|
200
|
+
import json as _json
|
|
201
|
+
data = {}
|
|
202
|
+
if self._flag_file.exists():
|
|
203
|
+
existing = _json.loads(self._flag_file.read_text(encoding="utf-8"))
|
|
204
|
+
if isinstance(existing, dict):
|
|
205
|
+
data = existing
|
|
206
|
+
data.update(changed)
|
|
207
|
+
self._flag_file.parent.mkdir(parents=True, exist_ok=True)
|
|
208
|
+
self._flag_file.write_text(
|
|
209
|
+
_json.dumps(data, ensure_ascii=False, indent=2),
|
|
210
|
+
encoding="utf-8")
|
|
211
|
+
except Exception:
|
|
212
|
+
pass
|
|
213
|
+
return dict(self.flags)
|
|
214
|
+
|
|
215
|
+
def _sync_attributes(self):
|
|
216
|
+
self.three_channel = bool(self.flags.get("three_channel", False))
|
|
217
|
+
self.eviction_readonly = bool(self.flags.get("eviction_readonly", False))
|
|
218
|
+
self.identity_writeback_enabled = bool(
|
|
219
|
+
self.flags.get("identity_writeback", self.flags.get("identity", False)))
|
|
220
|
+
self.slots_enabled = bool(self.flags.get("slots", False))
|
|
221
|
+
self.due_hints_enabled = bool(self.flags.get("due_hints", False))
|
|
222
|
+
if self.slots_enabled and self.slot_registry is None:
|
|
223
|
+
try:
|
|
224
|
+
from .slots import SlotRegistry
|
|
225
|
+
self.slot_registry = SlotRegistry(enabled=True)
|
|
226
|
+
except Exception:
|
|
227
|
+
self.slot_registry = None
|
|
228
|
+
elif not self.slots_enabled:
|
|
229
|
+
self.slot_registry = None
|
|
230
|
+
|
|
231
|
+
def _auto_turn_limit(self):
|
|
232
|
+
"""Resolve the auto-consolidate interval (default 10, flags/config)."""
|
|
233
|
+
for raw in (self.flags.get("consolidate_auto_turns"),
|
|
234
|
+
self.config.get("consolidate_auto_turns")):
|
|
235
|
+
try:
|
|
236
|
+
limit = int(raw)
|
|
237
|
+
except Exception:
|
|
238
|
+
continue
|
|
239
|
+
if limit > 0:
|
|
240
|
+
return limit
|
|
241
|
+
return int(CONSOLIDATE_AUTO_TURNS)
|
|
242
|
+
|
|
243
|
+
def touch(self, now=None):
|
|
244
|
+
"""Count one successful remember turn; auto-consolidate when enabled.
|
|
245
|
+
|
|
246
|
+
Default OFF: with consolidate_auto False (shipped default) this only
|
|
247
|
+
increments a counter and the old remember path is unchanged. When
|
|
248
|
+
consolidate_auto is True and consolidate_gated is on, every N remember
|
|
249
|
+
calls trigger consolidate(auto=True). Never raises.
|
|
250
|
+
"""
|
|
251
|
+
self._remember_turns = int(getattr(self, "_remember_turns", 0)) + 1
|
|
252
|
+
info = {"turn": self._remember_turns, "triggered": False}
|
|
253
|
+
try:
|
|
254
|
+
if not self._flag("consolidate_auto"):
|
|
255
|
+
return info
|
|
256
|
+
limit = self._auto_turn_limit()
|
|
257
|
+
info["every"] = limit
|
|
258
|
+
if self._remember_turns % limit != 0:
|
|
259
|
+
return info
|
|
260
|
+
if not self._flag("consolidate_gated"):
|
|
261
|
+
info["reason"] = "consolidate_gated_off"
|
|
262
|
+
return info
|
|
263
|
+
result = self.consolidate(auto=True, now=now)
|
|
264
|
+
self._last_auto_consolidate = result
|
|
265
|
+
info["triggered"] = bool((result or {}).get("enabled"))
|
|
266
|
+
info["consolidate"] = result
|
|
267
|
+
except Exception:
|
|
268
|
+
info["error"] = "auto_consolidate_failed"
|
|
269
|
+
return info
|
|
270
|
+
|
|
271
|
+
def after_turn(self, now=None):
|
|
272
|
+
"""Alias for touch(): after-turn counter used by the caller loop."""
|
|
273
|
+
return self.touch(now=now)
|
|
274
|
+
|
|
275
|
+
def _bump(self, key, amount=1):
|
|
276
|
+
try:
|
|
277
|
+
return int(self.store.bump_stats(key, amount) or 0)
|
|
278
|
+
except Exception:
|
|
279
|
+
return 0
|
|
280
|
+
|
|
281
|
+
def _set_last_op(self, entry, action, when=None):
|
|
282
|
+
"""Record the last write/dedupe action on one entry (UI read-only)."""
|
|
283
|
+
if not isinstance(entry, dict):
|
|
284
|
+
return entry
|
|
285
|
+
meta = entry.get("metadata")
|
|
286
|
+
if not isinstance(meta, dict):
|
|
287
|
+
meta = {}
|
|
288
|
+
entry["metadata"] = meta
|
|
289
|
+
meta["last_op"] = str(action or "noop")[:16]
|
|
290
|
+
try:
|
|
291
|
+
meta["last_op_ts"] = round(
|
|
292
|
+
float(when if when is not None else self.now()), 3)
|
|
293
|
+
except Exception:
|
|
294
|
+
meta["last_op_ts"] = 0.0
|
|
295
|
+
return entry
|
|
296
|
+
|
|
297
|
+
# -- persistence ------------------------------------------------------
|
|
298
|
+
def save(self):
|
|
299
|
+
return self.store.save()
|
|
300
|
+
|
|
301
|
+
def export(self):
|
|
302
|
+
return self.store.export()
|
|
303
|
+
|
|
304
|
+
def _flush_stats(self):
|
|
305
|
+
if self.store.path is not None:
|
|
306
|
+
try:
|
|
307
|
+
self.store.save()
|
|
308
|
+
except Exception:
|
|
309
|
+
pass
|
|
310
|
+
|
|
311
|
+
def write_snapshot(self, path=None):
|
|
312
|
+
"""Write the read-only UI snapshot JSON (never raises).
|
|
313
|
+
|
|
314
|
+
Merges timeline / slots_snapshot / dedupe_actions / graph_snapshot /
|
|
315
|
+
monitor into one document. Default target is <store>/snapshot.json;
|
|
316
|
+
the write is atomic (temp file + os.replace) and a failure returns
|
|
317
|
+
None without touching the caller's main flow.
|
|
318
|
+
"""
|
|
319
|
+
try:
|
|
320
|
+
target = Path(path) if path is not None else None
|
|
321
|
+
if target is None:
|
|
322
|
+
base = self._data_dir
|
|
323
|
+
if base is None and getattr(self.store, "path", None) is not None:
|
|
324
|
+
base = Path(self.store.path).parent
|
|
325
|
+
if base is None:
|
|
326
|
+
return None
|
|
327
|
+
target = Path(base) / "snapshot.json"
|
|
328
|
+
elif target.is_dir():
|
|
329
|
+
target = target / "snapshot.json"
|
|
330
|
+
payload = {
|
|
331
|
+
"schema": "bc.memory.snapshot.v1",
|
|
332
|
+
"ts": round(now_ts(self.now()), 3),
|
|
333
|
+
"timeline": self.timeline(),
|
|
334
|
+
"slots": self.slots_snapshot(),
|
|
335
|
+
"dedupe_actions": self.dedupe_actions(),
|
|
336
|
+
"graph": self.graph_snapshot(),
|
|
337
|
+
"monitor": self.monitor(),
|
|
338
|
+
"memory_points": self.memory_points_snapshot(),
|
|
339
|
+
"index": self.index_snapshot(),
|
|
340
|
+
"loops": self.loops_snapshot(),
|
|
341
|
+
"flags_all": dict(self.flags),
|
|
342
|
+
}
|
|
343
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
344
|
+
tmp = target.with_name(target.name + ".tmp")
|
|
345
|
+
try:
|
|
346
|
+
tmp.write_text(json.dumps(payload, ensure_ascii=False,
|
|
347
|
+
indent=1, default=str),
|
|
348
|
+
encoding="utf-8")
|
|
349
|
+
os.replace(str(tmp), str(target))
|
|
350
|
+
finally:
|
|
351
|
+
if tmp.exists():
|
|
352
|
+
try:
|
|
353
|
+
tmp.unlink()
|
|
354
|
+
except Exception:
|
|
355
|
+
pass
|
|
356
|
+
return target
|
|
357
|
+
except Exception:
|
|
358
|
+
return None
|
|
359
|
+
|
|
360
|
+
def _write_snapshot_safe(self, path=None):
|
|
361
|
+
try:
|
|
362
|
+
return self.write_snapshot(path)
|
|
363
|
+
except Exception:
|
|
364
|
+
return None
|
|
365
|
+
|
|
366
|
+
# -- optional mechanism adapters --------------------------------------
|
|
367
|
+
def _audit_path(self):
|
|
368
|
+
base = self._data_dir
|
|
369
|
+
if base is None and getattr(self.store, "path", None) is not None:
|
|
370
|
+
base = Path(self.store.path).parent
|
|
371
|
+
if base is None:
|
|
372
|
+
return None
|
|
373
|
+
return Path(base) / "audit.jsonl"
|
|
374
|
+
|
|
375
|
+
def _audit_write(self, action, entry_id="", event="", detail=None):
|
|
376
|
+
if not self._flag("audit"):
|
|
377
|
+
return False
|
|
378
|
+
path = self._audit_path()
|
|
379
|
+
if path is None:
|
|
380
|
+
return False
|
|
381
|
+
try:
|
|
382
|
+
if self._audit is None:
|
|
383
|
+
from .store.audit import audit_file
|
|
384
|
+
self._audit = audit_file(path)
|
|
385
|
+
row = {"schema": "bc.memory.audit.v1",
|
|
386
|
+
"action": str(action or "")[:40],
|
|
387
|
+
"entry_id": str(entry_id or "")[:80],
|
|
388
|
+
"event": str(event or "")[:40]}
|
|
389
|
+
if detail is not None:
|
|
390
|
+
row["detail"] = str(detail)[:200]
|
|
391
|
+
if self._audit.write(row):
|
|
392
|
+
self._bump("audit_writes")
|
|
393
|
+
return True
|
|
394
|
+
except Exception:
|
|
395
|
+
pass
|
|
396
|
+
return False
|
|
397
|
+
|
|
398
|
+
def _rate_limit_ok(self, scope):
|
|
399
|
+
if not self._flag("rate_limit"):
|
|
400
|
+
return True
|
|
401
|
+
base = self._data_dir
|
|
402
|
+
if base is None and getattr(self.store, "path", None) is not None:
|
|
403
|
+
base = Path(self.store.path).parent
|
|
404
|
+
if base is None:
|
|
405
|
+
return True
|
|
406
|
+
key = sha12(str(scope or "default"))
|
|
407
|
+
limiter = self._rate_limiters.get(key)
|
|
408
|
+
if limiter is None:
|
|
409
|
+
try:
|
|
410
|
+
from .store.rate_limit import rate_limiter
|
|
411
|
+
limiter = rate_limiter(Path(base) / ("rate_limit_%s.json" % key),
|
|
412
|
+
max_per_min=int(RATE_LIMIT_PER_MIN))
|
|
413
|
+
except Exception:
|
|
414
|
+
return True
|
|
415
|
+
self._rate_limiters[key] = limiter
|
|
416
|
+
try:
|
|
417
|
+
allowed = bool(limiter.check(now_ts(self.now())))
|
|
418
|
+
except Exception:
|
|
419
|
+
allowed = True
|
|
420
|
+
if not allowed:
|
|
421
|
+
self._bump("rate_limited")
|
|
422
|
+
return allowed
|
|
423
|
+
|
|
424
|
+
def _permission_ok(self, entry, scope, action="write"):
|
|
425
|
+
if not self._flag("permissions"):
|
|
426
|
+
return True
|
|
427
|
+
try:
|
|
428
|
+
from .core import permissions
|
|
429
|
+
except Exception:
|
|
430
|
+
return True
|
|
431
|
+
principal = self.principal
|
|
432
|
+
if not isinstance(principal, dict):
|
|
433
|
+
principal = {"role": "user", "scopes": [str(scope or "default")]}
|
|
434
|
+
previous = permissions.enabled
|
|
435
|
+
permissions.enabled = True
|
|
436
|
+
try:
|
|
437
|
+
if action == "read":
|
|
438
|
+
allowed = bool(permissions.can_read(entry, principal))
|
|
439
|
+
else:
|
|
440
|
+
allowed = bool(permissions.can_write(entry, principal))
|
|
441
|
+
except Exception:
|
|
442
|
+
allowed = True
|
|
443
|
+
finally:
|
|
444
|
+
permissions.enabled = previous
|
|
445
|
+
if not allowed:
|
|
446
|
+
self._bump("permissions_denied")
|
|
447
|
+
self._audit_write("permission_denied", (entry or {}).get("id")
|
|
448
|
+
if isinstance(entry, dict) else "", action)
|
|
449
|
+
return allowed
|
|
450
|
+
|
|
451
|
+
def _security_check(self, entry):
|
|
452
|
+
if not self._flag("security"):
|
|
453
|
+
return False, "security_disabled"
|
|
454
|
+
try:
|
|
455
|
+
from .store import security
|
|
456
|
+
except Exception:
|
|
457
|
+
return False, "security_unavailable"
|
|
458
|
+
previous = security.SECURITY_ENABLED
|
|
459
|
+
security.SECURITY_ENABLED = True
|
|
460
|
+
try:
|
|
461
|
+
reject, reason = security.should_reject(entry)
|
|
462
|
+
except Exception:
|
|
463
|
+
return False, "security_fallback"
|
|
464
|
+
finally:
|
|
465
|
+
security.SECURITY_ENABLED = previous
|
|
466
|
+
return bool(reject), str(reason)
|
|
467
|
+
|
|
468
|
+
def _apply_media(self, entry):
|
|
469
|
+
if not self._flag("media") or not isinstance(entry, dict):
|
|
470
|
+
return entry
|
|
471
|
+
media = entry.get("media")
|
|
472
|
+
if not isinstance(media, dict) or not media:
|
|
473
|
+
return entry
|
|
474
|
+
try:
|
|
475
|
+
from .store.media import attach
|
|
476
|
+
except Exception:
|
|
477
|
+
return entry
|
|
478
|
+
try:
|
|
479
|
+
result = attach(entry)
|
|
480
|
+
except Exception:
|
|
481
|
+
return entry
|
|
482
|
+
if isinstance(result, dict):
|
|
483
|
+
stored = result.get("media") if isinstance(result.get("media"), dict) else {}
|
|
484
|
+
if stored and (stored != media or stored.get("redacted")):
|
|
485
|
+
self._bump("media_attached")
|
|
486
|
+
return result
|
|
487
|
+
return entry
|
|
488
|
+
|
|
489
|
+
def _apply_notes(self, entry):
|
|
490
|
+
if not self._flag("notes") or not isinstance(entry, dict):
|
|
491
|
+
return entry
|
|
492
|
+
try:
|
|
493
|
+
from .event.notes import make_note
|
|
494
|
+
except Exception:
|
|
495
|
+
return entry
|
|
496
|
+
try:
|
|
497
|
+
tags = entry.get("tags") if isinstance(entry.get("tags"), list) else None
|
|
498
|
+
links = (entry.get("related_ids")
|
|
499
|
+
if isinstance(entry.get("related_ids"), list) else None)
|
|
500
|
+
note = make_note(entry.get("text"), tags=tags, links=links)
|
|
501
|
+
except Exception:
|
|
502
|
+
return entry
|
|
503
|
+
if isinstance(note, dict):
|
|
504
|
+
entry["note"] = note
|
|
505
|
+
meta = entry.get("metadata")
|
|
506
|
+
if isinstance(meta, dict):
|
|
507
|
+
meta["note"] = note
|
|
508
|
+
self._bump("notes_written")
|
|
509
|
+
return entry
|
|
510
|
+
|
|
511
|
+
def _apply_notes_link(self, entry):
|
|
512
|
+
if not self._flag("notes_link") or not isinstance(entry, dict):
|
|
513
|
+
return entry
|
|
514
|
+
try:
|
|
515
|
+
from .event import notes_link as notes_link_mod
|
|
516
|
+
except Exception:
|
|
517
|
+
return entry
|
|
518
|
+
try:
|
|
519
|
+
result = notes_link_mod.enrich_entry(entry, enabled=True)
|
|
520
|
+
except Exception:
|
|
521
|
+
return entry
|
|
522
|
+
if isinstance(result, dict):
|
|
523
|
+
meta = result.get("metadata")
|
|
524
|
+
if isinstance(meta, dict) and isinstance(meta.get("note"), dict):
|
|
525
|
+
self._bump("notes_linked")
|
|
526
|
+
return result
|
|
527
|
+
return entry
|
|
528
|
+
|
|
529
|
+
def _apply_ledger(self, event, now=None):
|
|
530
|
+
if not self._flag("ledger"):
|
|
531
|
+
return None
|
|
532
|
+
meta = getattr(event, "metadata", None)
|
|
533
|
+
if not isinstance(meta, dict) and isinstance(event, dict):
|
|
534
|
+
meta = event
|
|
535
|
+
if not isinstance(meta, dict):
|
|
536
|
+
return None
|
|
537
|
+
signal = any(meta.get(key) is not None for key in
|
|
538
|
+
("trust_delta", "conflict_delta", "relationship_kind"))
|
|
539
|
+
if not signal:
|
|
540
|
+
return None
|
|
541
|
+
try:
|
|
542
|
+
from .identity import ledger
|
|
543
|
+
except Exception:
|
|
544
|
+
return None
|
|
545
|
+
try:
|
|
546
|
+
result = ledger.append_entry(self.store, dict(meta),
|
|
547
|
+
now=now_ts(now or self.now()), enabled=True)
|
|
548
|
+
except Exception:
|
|
549
|
+
return None
|
|
550
|
+
written = len(result.get("written") or []) if isinstance(result, dict) else 0
|
|
551
|
+
if written:
|
|
552
|
+
self._bump("ledger_writes", written)
|
|
553
|
+
self._last_ledger = result
|
|
554
|
+
return result
|
|
555
|
+
|
|
556
|
+
def _cooc_refresh(self):
|
|
557
|
+
if not self._flag("entity_boost"):
|
|
558
|
+
return
|
|
559
|
+
try:
|
|
560
|
+
from .index.entity_boost import build_cooccurrence
|
|
561
|
+
self._cooc = build_cooccurrence(self.store.entries)
|
|
562
|
+
except Exception:
|
|
563
|
+
self._cooc = {}
|
|
564
|
+
|
|
565
|
+
def _apply_entity_boost(self, rows, query):
|
|
566
|
+
if not self._flag("entity_boost"):
|
|
567
|
+
return rows
|
|
568
|
+
try:
|
|
569
|
+
from .index import entity_boost as eb_mod
|
|
570
|
+
from .index.entities import cues as cue_entities
|
|
571
|
+
except Exception:
|
|
572
|
+
return rows
|
|
573
|
+
if not self._cooc:
|
|
574
|
+
self._cooc_refresh()
|
|
575
|
+
if not isinstance(self._cooc, dict) or not self._cooc:
|
|
576
|
+
return rows
|
|
577
|
+
try:
|
|
578
|
+
query_entities = cue_entities(str(query or "")).get("entities") or []
|
|
579
|
+
except Exception:
|
|
580
|
+
query_entities = []
|
|
581
|
+
if not query_entities:
|
|
582
|
+
return rows
|
|
583
|
+
boosted = 0
|
|
584
|
+
for row in rows:
|
|
585
|
+
entry = row.get("entry") if isinstance(row, dict) else None
|
|
586
|
+
if not isinstance(entry, dict):
|
|
587
|
+
continue
|
|
588
|
+
entry_entities = entry.get("entities")
|
|
589
|
+
if not isinstance(entry_entities, list) or not entry_entities:
|
|
590
|
+
try:
|
|
591
|
+
entry_entities = cue_entities(str(entry.get("text") or "")
|
|
592
|
+
).get("entities") or []
|
|
593
|
+
except Exception:
|
|
594
|
+
entry_entities = []
|
|
595
|
+
names = [name for name in query_entities if name in entry_entities]
|
|
596
|
+
value = 0.0
|
|
597
|
+
for name in names:
|
|
598
|
+
try:
|
|
599
|
+
value = max(value, float(eb_mod.boost(name, self._cooc) or 0.0))
|
|
600
|
+
except Exception:
|
|
601
|
+
continue
|
|
602
|
+
if value > 0:
|
|
603
|
+
row["score"] = round(min(1.0, float(row.get("score") or 0.0) + value), 6)
|
|
604
|
+
reasons = row.get("reasons")
|
|
605
|
+
if isinstance(reasons, list):
|
|
606
|
+
reasons.append("entity_boost")
|
|
607
|
+
else:
|
|
608
|
+
row["reasons"] = ["entity_boost"]
|
|
609
|
+
boosted += 1
|
|
610
|
+
if boosted:
|
|
611
|
+
self._bump("entity_boost_hits", boosted)
|
|
612
|
+
rows.sort(key=lambda item: float(item.get("score") or 0.0), reverse=True)
|
|
613
|
+
return rows
|
|
614
|
+
|
|
615
|
+
def _graph_edges(self):
|
|
616
|
+
if not self._flag("graph"):
|
|
617
|
+
return []
|
|
618
|
+
try:
|
|
619
|
+
from .index import graph as graph_mod
|
|
620
|
+
raw = graph_mod.build_links(self.store.entries)
|
|
621
|
+
except Exception:
|
|
622
|
+
return []
|
|
623
|
+
edges = self._normalize_edges(raw)
|
|
624
|
+
if edges and len(edges) != self._last_graph_edge_count:
|
|
625
|
+
self._bump("graph_edges", max(0, len(edges) - self._last_graph_edge_count))
|
|
626
|
+
self._last_graph_edge_count = len(edges)
|
|
627
|
+
return edges
|
|
628
|
+
|
|
629
|
+
@staticmethod
|
|
630
|
+
def _normalize_edges(raw):
|
|
631
|
+
edges = []
|
|
632
|
+
if isinstance(raw, dict):
|
|
633
|
+
for src, dsts in raw.items():
|
|
634
|
+
if isinstance(dsts, (list, tuple, set)):
|
|
635
|
+
for dst in dsts:
|
|
636
|
+
edges.append((str(src), str(dst)))
|
|
637
|
+
elif dsts is not None:
|
|
638
|
+
edges.append((str(src), str(dsts)))
|
|
639
|
+
return edges
|
|
640
|
+
for item in raw or []:
|
|
641
|
+
if isinstance(item, dict):
|
|
642
|
+
src = item.get("src", item.get("source", item.get("from")))
|
|
643
|
+
dst = item.get("dst", item.get("target", item.get("to")))
|
|
644
|
+
if src is not None and dst is not None:
|
|
645
|
+
edges.append((str(src), str(dst)))
|
|
646
|
+
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
|
647
|
+
edges.append((str(item[0]), str(item[1])))
|
|
648
|
+
return edges
|
|
649
|
+
|
|
650
|
+
def _apply_inhibition_filter(self, entries):
|
|
651
|
+
if not self._flag("inhibition"):
|
|
652
|
+
return entries
|
|
653
|
+
try:
|
|
654
|
+
from .evict import inhibition as inhibition_mod
|
|
655
|
+
return inhibition_mod.filter_inhibited(entries)
|
|
656
|
+
except Exception:
|
|
657
|
+
return entries
|
|
658
|
+
|
|
659
|
+
def _mark_inhibition(self, entries, selected_ids):
|
|
660
|
+
if not self._flag("inhibition"):
|
|
661
|
+
return 0
|
|
662
|
+
try:
|
|
663
|
+
from .evict import inhibition as inhibition_mod
|
|
664
|
+
touched = inhibition_mod.mark_inhibited(entries, selected_ids)
|
|
665
|
+
except Exception:
|
|
666
|
+
return 0
|
|
667
|
+
if touched:
|
|
668
|
+
self._bump("inhibition_applied", len(touched))
|
|
669
|
+
return len(touched or [])
|
|
670
|
+
|
|
671
|
+
def _apply_retrieval_practice(self, rows, stamp):
|
|
672
|
+
if not self._flag("retrieval_practice"):
|
|
673
|
+
return 0
|
|
674
|
+
try:
|
|
675
|
+
from .recall import retrieval_practice as rp_mod
|
|
676
|
+
except Exception:
|
|
677
|
+
return 0
|
|
678
|
+
updated = 0
|
|
679
|
+
for row in rows:
|
|
680
|
+
entry = row.get("entry") if isinstance(row, dict) else None
|
|
681
|
+
if not isinstance(entry, dict):
|
|
682
|
+
continue
|
|
683
|
+
entry_id = str(entry.get("id") or "")
|
|
684
|
+
try:
|
|
685
|
+
patch = rp_mod.update_stability(entry, success=True, now=stamp,
|
|
686
|
+
enabled=True)
|
|
687
|
+
except Exception:
|
|
688
|
+
patch = {}
|
|
689
|
+
if not isinstance(patch, dict) or not patch:
|
|
690
|
+
continue
|
|
691
|
+
try:
|
|
692
|
+
if entry_id and self.store.find(entry_id) is not None:
|
|
693
|
+
self.store.update(entry_id, **patch)
|
|
694
|
+
else:
|
|
695
|
+
entry.update(patch)
|
|
696
|
+
updated += 1
|
|
697
|
+
except Exception:
|
|
698
|
+
continue
|
|
699
|
+
if updated:
|
|
700
|
+
self._bump("retrieval_practice", updated)
|
|
701
|
+
self._last_retrieval = {"updated": updated}
|
|
702
|
+
return updated
|
|
703
|
+
|
|
704
|
+
def _boundary_signal(self, entry, stamp):
|
|
705
|
+
if not (self._flag("boundary") and self._flag("slots")):
|
|
706
|
+
return 0.0, False
|
|
707
|
+
try:
|
|
708
|
+
from .slots import boundary as boundary_mod
|
|
709
|
+
except Exception:
|
|
710
|
+
return 0.0, False
|
|
711
|
+
text = str((entry or {}).get("text") or "") if isinstance(entry, dict) else ""
|
|
712
|
+
previous = str(getattr(self, "_last_slots_msg", "") or "")
|
|
713
|
+
previous_ts = float(getattr(self, "_last_slots_ts", 0.0) or 0.0)
|
|
714
|
+
gap = max(0.0, float(stamp) - previous_ts) if previous_ts else 0.0
|
|
715
|
+
try:
|
|
716
|
+
is_boundary, score = boundary_mod.is_boundary(
|
|
717
|
+
{"prev_msg": previous, "cur_msg": text, "dt": gap})
|
|
718
|
+
except Exception:
|
|
719
|
+
return 0.0, False
|
|
720
|
+
self._last_slots_msg = text
|
|
721
|
+
self._last_slots_ts = float(stamp)
|
|
722
|
+
if is_boundary:
|
|
723
|
+
self._bump("boundary_switches")
|
|
724
|
+
return float(score or 0.0), bool(is_boundary)
|
|
725
|
+
|
|
726
|
+
def _compress_entries(self, entries):
|
|
727
|
+
if not self._flag("compress"):
|
|
728
|
+
self._last_compress = None
|
|
729
|
+
return None
|
|
730
|
+
try:
|
|
731
|
+
from .store import compress as compress_mod
|
|
732
|
+
except Exception:
|
|
733
|
+
return None
|
|
734
|
+
previous = compress_mod.enabled
|
|
735
|
+
compress_mod.enabled = True
|
|
736
|
+
try:
|
|
737
|
+
result = compress_mod.compress(entries,
|
|
738
|
+
max_chars=int(COMPRESS_MAX_CHARS))
|
|
739
|
+
except Exception:
|
|
740
|
+
result = {"summary": "", "ids": []}
|
|
741
|
+
finally:
|
|
742
|
+
compress_mod.enabled = previous
|
|
743
|
+
if isinstance(result, dict) and result.get("summary"):
|
|
744
|
+
self._bump("compress_runs")
|
|
745
|
+
self._last_compress = result
|
|
746
|
+
return result
|
|
747
|
+
|
|
748
|
+
def _apply_tiering_exec(self, entries, stamp):
|
|
749
|
+
if not self._flag("tiering_exec"):
|
|
750
|
+
return None
|
|
751
|
+
try:
|
|
752
|
+
from .store import tiering as tiering_mod
|
|
753
|
+
from .store import tiering_exec as tiering_exec_mod
|
|
754
|
+
except Exception:
|
|
755
|
+
return None
|
|
756
|
+
previous = tiering_mod.TIERING_ENABLED
|
|
757
|
+
tiering_mod.TIERING_ENABLED = True
|
|
758
|
+
try:
|
|
759
|
+
result = tiering_exec_mod.apply_tiers(entries, now=stamp)
|
|
760
|
+
except Exception:
|
|
761
|
+
return None
|
|
762
|
+
finally:
|
|
763
|
+
tiering_mod.TIERING_ENABLED = previous
|
|
764
|
+
archived = result.get("archived") if isinstance(result, dict) else None
|
|
765
|
+
if archived:
|
|
766
|
+
self._bump("tiering_archived", len(archived))
|
|
767
|
+
self._tiering_exec_applied = result
|
|
768
|
+
return result
|
|
769
|
+
|
|
770
|
+
def _store_load(self):
|
|
771
|
+
"""Normalized store load (0..1) used by the adaptive eviction flag."""
|
|
772
|
+
try:
|
|
773
|
+
from .store.tiering import WARM_BUDGET
|
|
774
|
+
budget = float(WARM_BUDGET or 0)
|
|
775
|
+
except Exception:
|
|
776
|
+
budget = 200.0
|
|
777
|
+
if budget <= 0:
|
|
778
|
+
return 0.0
|
|
779
|
+
try:
|
|
780
|
+
return max(0.0, min(1.0, len(self.store.entries) / budget))
|
|
781
|
+
except Exception:
|
|
782
|
+
return 0.0
|
|
783
|
+
|
|
784
|
+
def _adaptive_eviction_score(self, entry, base, load):
|
|
785
|
+
"""Replace one base eviction score via evict/adaptive.adaptive_score."""
|
|
786
|
+
if not self._flag("adaptive_evict"):
|
|
787
|
+
return base
|
|
788
|
+
try:
|
|
789
|
+
from .evict import adaptive as adaptive_mod
|
|
790
|
+
importance = float((entry or {}).get("importance", 0.5) or 0.5)
|
|
791
|
+
except Exception:
|
|
792
|
+
return base
|
|
793
|
+
try:
|
|
794
|
+
return float(adaptive_mod.adaptive_score(
|
|
795
|
+
base, load=load, importance=importance) or 0.0)
|
|
796
|
+
except Exception:
|
|
797
|
+
return base
|
|
798
|
+
|
|
799
|
+
def _apply_consistency(self, entry):
|
|
800
|
+
"""Annotate old/new fact conflicts on the new entry (never overwrites)."""
|
|
801
|
+
if not self._flag("consistency") or not isinstance(entry, dict):
|
|
802
|
+
return entry
|
|
803
|
+
try:
|
|
804
|
+
from .event import consistency as consistency_mod
|
|
805
|
+
except Exception:
|
|
806
|
+
return entry
|
|
807
|
+
previous = consistency_mod.CONSISTENCY_ENABLED
|
|
808
|
+
consistency_mod.CONSISTENCY_ENABLED = True
|
|
809
|
+
try:
|
|
810
|
+
old_entries = [x for x in self.store.entries
|
|
811
|
+
if isinstance(x, dict)
|
|
812
|
+
and str(x.get("id") or "") != str(entry.get("id") or "")]
|
|
813
|
+
result = consistency_mod.detect_conflicts(entry, old_entries)
|
|
814
|
+
except Exception:
|
|
815
|
+
result = None
|
|
816
|
+
finally:
|
|
817
|
+
consistency_mod.CONSISTENCY_ENABLED = previous
|
|
818
|
+
if isinstance(result, dict) and result.get("conflict"):
|
|
819
|
+
meta = entry.get("metadata")
|
|
820
|
+
if not isinstance(meta, dict):
|
|
821
|
+
meta = {}
|
|
822
|
+
entry["metadata"] = meta
|
|
823
|
+
targets = [str(x) for x in (result.get("target_ids") or []) if str(x)]
|
|
824
|
+
meta["conflict"] = {
|
|
825
|
+
"flag": True,
|
|
826
|
+
"against_id": targets[0] if targets else "",
|
|
827
|
+
"against_ids": targets[:8],
|
|
828
|
+
"reasons": [str((item or {}).get("reason") or "")
|
|
829
|
+
for item in (result.get("details") or [])][:8],
|
|
830
|
+
"detected_ts": round(now_ts(self.now()), 3)}
|
|
831
|
+
meta["uncertainty"] = float(result.get("uncertainty") or 0.0)
|
|
832
|
+
self._bump("consistency_conflicts")
|
|
833
|
+
self._audit_write("consistency_conflict", entry.get("id"),
|
|
834
|
+
"remember",
|
|
835
|
+
"against=%s" % (targets[0] if targets else ""))
|
|
836
|
+
self._last_consistency = result
|
|
837
|
+
return entry
|
|
838
|
+
|
|
839
|
+
def _apply_source_consistency(self, rows):
|
|
840
|
+
"""Mark source_rank per candidate and downweight conflicting sources."""
|
|
841
|
+
if not self._flag("source_consistency"):
|
|
842
|
+
return rows
|
|
843
|
+
try:
|
|
844
|
+
from .store import source_consistency as source_mod
|
|
845
|
+
except Exception:
|
|
846
|
+
return rows
|
|
847
|
+
previous = source_mod.SOURCE_CONSISTENCY_ENABLED
|
|
848
|
+
source_mod.SOURCE_CONSISTENCY_ENABLED = True
|
|
849
|
+
ranked = 0
|
|
850
|
+
conflicts = 0
|
|
851
|
+
try:
|
|
852
|
+
candidates = [row.get("entry") for row in rows
|
|
853
|
+
if isinstance(row, dict)
|
|
854
|
+
and isinstance(row.get("entry"), dict)]
|
|
855
|
+
for row in rows:
|
|
856
|
+
entry = row.get("entry") if isinstance(row, dict) else None
|
|
857
|
+
if not isinstance(entry, dict):
|
|
858
|
+
continue
|
|
859
|
+
try:
|
|
860
|
+
rank = float(source_mod.source_rank(entry) or 0.0)
|
|
861
|
+
except Exception:
|
|
862
|
+
rank = 0.0
|
|
863
|
+
try:
|
|
864
|
+
check = source_mod.detect_conflicts(entry, candidates)
|
|
865
|
+
except Exception:
|
|
866
|
+
check = None
|
|
867
|
+
row["source_rank"] = round(rank, 4)
|
|
868
|
+
if isinstance(check, dict) and check.get("conflict"):
|
|
869
|
+
row["source_conflict"] = {
|
|
870
|
+
"flag": True,
|
|
871
|
+
"confidence": float(check.get("confidence") or 0.0),
|
|
872
|
+
"source_rank": float(check.get("source_rank") or rank),
|
|
873
|
+
"target_ids": [str(x) for x in
|
|
874
|
+
(check.get("target_ids") or [])][:8]}
|
|
875
|
+
row["score"] = round(
|
|
876
|
+
float(row.get("score") or 0.0) * (0.85 + 0.15 * rank), 6)
|
|
877
|
+
conflicts += 1
|
|
878
|
+
else:
|
|
879
|
+
row["source_conflict"] = {"flag": False}
|
|
880
|
+
ranked += 1
|
|
881
|
+
finally:
|
|
882
|
+
source_mod.SOURCE_CONSISTENCY_ENABLED = previous
|
|
883
|
+
if ranked:
|
|
884
|
+
self._bump("source_consistency_ranked", ranked)
|
|
885
|
+
if conflicts:
|
|
886
|
+
self._bump("source_consistency_conflicts", conflicts)
|
|
887
|
+
rows.sort(key=lambda item: float(item.get("score") or 0.0),
|
|
888
|
+
reverse=True)
|
|
889
|
+
self._last_source_consistency = {"ranked": ranked, "conflicts": conflicts}
|
|
890
|
+
return rows
|
|
891
|
+
|
|
892
|
+
def _apply_narrative(self, now=None):
|
|
893
|
+
"""Run one identity-narrative update at the end of consolidation."""
|
|
894
|
+
if not self._flag("narrative"):
|
|
895
|
+
return None
|
|
896
|
+
try:
|
|
897
|
+
from .identity import narrative as narrative_mod
|
|
898
|
+
except Exception:
|
|
899
|
+
return None
|
|
900
|
+
previous = narrative_mod.NARRATIVE_ENABLED
|
|
901
|
+
narrative_mod.NARRATIVE_ENABLED = True
|
|
902
|
+
try:
|
|
903
|
+
entries = [x for x in self.store.entries
|
|
904
|
+
if isinstance(x, dict) and not x.get("deleted")]
|
|
905
|
+
outcome = narrative_mod.apply_update(
|
|
906
|
+
self.store, entries=entries, now=now_ts(now or self.now()),
|
|
907
|
+
enabled=True)
|
|
908
|
+
except Exception:
|
|
909
|
+
outcome = None
|
|
910
|
+
finally:
|
|
911
|
+
narrative_mod.NARRATIVE_ENABLED = previous
|
|
912
|
+
if not isinstance(outcome, dict):
|
|
913
|
+
return None
|
|
914
|
+
written = 1 if outcome.get("updated") else 0
|
|
915
|
+
try:
|
|
916
|
+
version = int(outcome.get("version") or 0)
|
|
917
|
+
except Exception:
|
|
918
|
+
version = 0
|
|
919
|
+
report = {"written": written, "version": version,
|
|
920
|
+
"reason": str(outcome.get("reason") or ""),
|
|
921
|
+
"identity_id": str(outcome.get("id") or "")}
|
|
922
|
+
if written:
|
|
923
|
+
self._bump("narrative_writes")
|
|
924
|
+
self._audit_write("narrative", report["identity_id"],
|
|
925
|
+
"consolidate", "version=%d" % version)
|
|
926
|
+
self._last_narrative = report
|
|
927
|
+
return report
|
|
928
|
+
|
|
929
|
+
# -- remember ---------------------------------------------------------
|
|
930
|
+
def remember(self, event, scope=None):
|
|
931
|
+
raw_event = event
|
|
932
|
+
ev = event_from_any(event)
|
|
933
|
+
text = str(getattr(ev, "text", "") or "").strip()
|
|
934
|
+
if len(text) < 2:
|
|
935
|
+
return None
|
|
936
|
+
stamp = now_ts(getattr(ev, "ts", 0.0) or self.now())
|
|
937
|
+
vector = vector_for(self, ev)
|
|
938
|
+
meta = getattr(ev, "metadata", {}) or {}
|
|
939
|
+
if not isinstance(meta, dict):
|
|
940
|
+
meta = {}
|
|
941
|
+
if isinstance(raw_event, dict):
|
|
942
|
+
# event_from_any() only keeps dataclass fields; dict payloads may
|
|
943
|
+
# carry tags/media/topic/scope/etc. Keep them as metadata extras.
|
|
944
|
+
merged = dict(meta)
|
|
945
|
+
for key, value in raw_event.items():
|
|
946
|
+
if key == "metadata" or key in MemoryEvent.__dataclass_fields__:
|
|
947
|
+
continue
|
|
948
|
+
merged.setdefault(key, value)
|
|
949
|
+
meta = merged
|
|
950
|
+
ev.metadata = meta
|
|
951
|
+
entry_scope = str(scope or getattr(ev, "scope", "")
|
|
952
|
+
or meta.get("scope") or "default")
|
|
953
|
+
explicit_importance = explicit_number(event, "importance")
|
|
954
|
+
explicit_self = explicit_number(event, "self_relevance")
|
|
955
|
+
probe = {"text": text, "scope": entry_scope,
|
|
956
|
+
"role": str(getattr(ev, "role", "user") or "user")}
|
|
957
|
+
if not self._permission_ok(probe, entry_scope, "write"):
|
|
958
|
+
return None
|
|
959
|
+
if not self._rate_limit_ok(entry_scope):
|
|
960
|
+
return None
|
|
961
|
+
for existing in self.store.entries:
|
|
962
|
+
if (str(existing.get("text") or "") == text
|
|
963
|
+
and str(existing.get("role") or "user")
|
|
964
|
+
== str(getattr(ev, "role", "user") or "user")):
|
|
965
|
+
existing["hits"] = int(existing.get("hits", 1) or 1) + 1
|
|
966
|
+
if vector:
|
|
967
|
+
existing["vector"] = vector
|
|
968
|
+
self._last_dedupe, self._last_identity_check = None, None
|
|
969
|
+
_chk = (check_after_remember(self.store, existing,
|
|
970
|
+
embedder=self.embedder)
|
|
971
|
+
if self._flag("dedupe")
|
|
972
|
+
else {"dedupe": {"action": "keep",
|
|
973
|
+
"reason": "dedupe_disabled"},
|
|
974
|
+
"identity": None})
|
|
975
|
+
self._last_dedupe = _chk.get("dedupe")
|
|
976
|
+
self._last_identity_check = _chk.get("identity")
|
|
977
|
+
if self._flag("dedupe"):
|
|
978
|
+
# The exact-text branch is the engine's L0 merge.
|
|
979
|
+
self._set_last_op(existing, "merge", stamp)
|
|
980
|
+
self._bump("dedupe_exact_merge_count")
|
|
981
|
+
if self._last_dedupe and self._last_dedupe.get("action") in ("merge", "update"):
|
|
982
|
+
self._bump("dedupe_merge_count")
|
|
983
|
+
else:
|
|
984
|
+
self._set_last_op(existing, "noop", stamp)
|
|
985
|
+
self._bump("dedupe_noop_count")
|
|
986
|
+
self._slots_ingest(existing)
|
|
987
|
+
self._cooc_refresh()
|
|
988
|
+
self._audit_write("remember_merge", existing.get("id"), "remember")
|
|
989
|
+
self._flush_stats()
|
|
990
|
+
self._write_snapshot_safe()
|
|
991
|
+
self.touch(now=stamp)
|
|
992
|
+
return existing
|
|
993
|
+
entry = {
|
|
994
|
+
"id": (str(getattr(ev, "id", "") or "") or sha12(text + str(stamp))),
|
|
995
|
+
"text": text,
|
|
996
|
+
"role": str(getattr(ev, "role", "user") or "user"),
|
|
997
|
+
"session": str(getattr(ev, "session", "") or ""),
|
|
998
|
+
"ts": stamp,
|
|
999
|
+
"valid_at": stamp,
|
|
1000
|
+
"vector": vector,
|
|
1001
|
+
"hits": 1,
|
|
1002
|
+
"importance": (explicit_importance if explicit_importance is not None
|
|
1003
|
+
else value_of(ev)),
|
|
1004
|
+
"self_relevance": (explicit_self if explicit_self is not None
|
|
1005
|
+
else clamp01(getattr(ev, "self_relevance", 0.0))),
|
|
1006
|
+
"memory_source": "direct",
|
|
1007
|
+
"memory_confidence": 0.7,
|
|
1008
|
+
"pin": False,
|
|
1009
|
+
"previous_versions": [],
|
|
1010
|
+
"module": "biocore_memory",
|
|
1011
|
+
"scope": entry_scope,
|
|
1012
|
+
}
|
|
1013
|
+
entry.update({k: v for k, v in meta.items() if k not in entry})
|
|
1014
|
+
try:
|
|
1015
|
+
memory_value = float(value_of(ev))
|
|
1016
|
+
except Exception:
|
|
1017
|
+
memory_value = clamp01(entry.get("importance", 0.5), 0.5)
|
|
1018
|
+
entry["value"] = round(memory_value, 6)
|
|
1019
|
+
entry["memory_point"] = bool(is_memory_point(
|
|
1020
|
+
entry["value"], entry.get("importance", memory_value),
|
|
1021
|
+
entry.get("self_relevance", 0.0)))
|
|
1022
|
+
_role = str(getattr(ev, "role", "user") or "user")
|
|
1023
|
+
if "memory_confidence" in meta:
|
|
1024
|
+
entry["memory_confidence"] = clamp01(meta.get("memory_confidence"), 0.7)
|
|
1025
|
+
try:
|
|
1026
|
+
confidence = clamp01(entry.get("memory_confidence", 0.7), 0.7)
|
|
1027
|
+
except Exception:
|
|
1028
|
+
confidence = 0.7
|
|
1029
|
+
entry["provenance"] = {
|
|
1030
|
+
"source": str(("user" if _role == "user"
|
|
1031
|
+
else (entry.get("source") or "assistant"))),
|
|
1032
|
+
"ts": stamp, "user_direct": _role == "user",
|
|
1033
|
+
"confidence": round(confidence, 4)}
|
|
1034
|
+
entry.setdefault("source", "biocore_memory:" + str(entry.get("id") or ""))
|
|
1035
|
+
if self._flag("provenance"):
|
|
1036
|
+
try:
|
|
1037
|
+
from .store.provenance import source_tag
|
|
1038
|
+
entry["provenance"].update(source_tag(
|
|
1039
|
+
text, entry["provenance"].get("source"), stamp))
|
|
1040
|
+
except Exception:
|
|
1041
|
+
pass
|
|
1042
|
+
rejected, reason = self._security_check(entry)
|
|
1043
|
+
if rejected:
|
|
1044
|
+
self._bump("security_rejected")
|
|
1045
|
+
self._last_security = {"rejected": True, "reason": reason}
|
|
1046
|
+
self._audit_write("security_rejected", entry.get("id"), "remember", reason)
|
|
1047
|
+
self._flush_stats()
|
|
1048
|
+
return None
|
|
1049
|
+
entry = self._apply_media(entry)
|
|
1050
|
+
entry = self._apply_notes(entry)
|
|
1051
|
+
entry = self._apply_notes_link(entry)
|
|
1052
|
+
if self._flag("cue_writeback", True):
|
|
1053
|
+
try:
|
|
1054
|
+
_cue = cue_extract(text, ref_ts=stamp)
|
|
1055
|
+
entry["cues"] = {key: _cue.get(key) for key in
|
|
1056
|
+
("event_name", "entities", "place",
|
|
1057
|
+
"time_coarse")}
|
|
1058
|
+
except Exception:
|
|
1059
|
+
pass
|
|
1060
|
+
added = self.store.add(entry)
|
|
1061
|
+
if added is None:
|
|
1062
|
+
return None
|
|
1063
|
+
_chk = (check_after_remember(self.store, added, embedder=self.embedder)
|
|
1064
|
+
if self._flag("dedupe")
|
|
1065
|
+
else {"dedupe": {"action": "keep", "reason": "dedupe_disabled"},
|
|
1066
|
+
"identity": None})
|
|
1067
|
+
self._last_dedupe = _chk.get("dedupe")
|
|
1068
|
+
self._last_identity_check = _chk.get("identity")
|
|
1069
|
+
action = (str((self._last_dedupe or {}).get("action") or "keep")
|
|
1070
|
+
if self._flag("dedupe") else "noop")
|
|
1071
|
+
if action in ("merge", "update"):
|
|
1072
|
+
target_id = (self._last_dedupe or {}).get("target_id")
|
|
1073
|
+
target = self.store.find(target_id) if target_id else None
|
|
1074
|
+
self._set_last_op(target if isinstance(target, dict) else added,
|
|
1075
|
+
action, stamp)
|
|
1076
|
+
self._bump("dedupe_merge_count" if action == "merge"
|
|
1077
|
+
else "dedupe_update_count")
|
|
1078
|
+
elif action == "flag":
|
|
1079
|
+
self._set_last_op(added, "flag", stamp)
|
|
1080
|
+
self._bump("dedupe_flag_count")
|
|
1081
|
+
elif action == "keep":
|
|
1082
|
+
self._set_last_op(added, "add", stamp)
|
|
1083
|
+
self._bump("dedupe_add_count")
|
|
1084
|
+
else:
|
|
1085
|
+
self._set_last_op(added, "noop", stamp)
|
|
1086
|
+
self._bump("dedupe_noop_count")
|
|
1087
|
+
self._apply_ledger(ev, stamp)
|
|
1088
|
+
self._apply_consistency(added)
|
|
1089
|
+
self._slots_ingest(added)
|
|
1090
|
+
self._cooc_refresh()
|
|
1091
|
+
self._audit_write("remember", added.get("id"), "remember",
|
|
1092
|
+
"source=%s" % str(added.get("memory_source") or ""))
|
|
1093
|
+
self._flush_stats()
|
|
1094
|
+
self._write_snapshot_safe()
|
|
1095
|
+
self.touch(now=stamp)
|
|
1096
|
+
return added
|
|
1097
|
+
|
|
1098
|
+
# -- recall -----------------------------------------------------------
|
|
1099
|
+
def recall(self, query, budget=DEFAULT_BUDGET, vector=None, scope="default",
|
|
1100
|
+
mode="now", principal=None):
|
|
1101
|
+
entries = [e for e in self.store.entries
|
|
1102
|
+
if isinstance(e, dict) and not e.get("deleted")
|
|
1103
|
+
and str(e.get("scope") or "default") == str(scope or "default")
|
|
1104
|
+
and not e.get("archived")]
|
|
1105
|
+
if self._flag("permissions"):
|
|
1106
|
+
checked = []
|
|
1107
|
+
for entry in entries:
|
|
1108
|
+
if self._permission_ok(entry, scope, "read"):
|
|
1109
|
+
checked.append(entry)
|
|
1110
|
+
entries = checked
|
|
1111
|
+
filtered = 0
|
|
1112
|
+
if self._flag("source_filter"):
|
|
1113
|
+
filtered = source_filtered_count(entries)
|
|
1114
|
+
if filtered:
|
|
1115
|
+
self._bump("source_filtered", filtered)
|
|
1116
|
+
if self._flag("temporal_filter"):
|
|
1117
|
+
try:
|
|
1118
|
+
from .recall.filters import filter_valid
|
|
1119
|
+
entries = filter_valid(entries, at=float(self.now()), mode=mode)
|
|
1120
|
+
except Exception:
|
|
1121
|
+
pass
|
|
1122
|
+
if self._flag("inhibition"):
|
|
1123
|
+
entries = self._apply_inhibition_filter(entries)
|
|
1124
|
+
if self._flag("complexity"):
|
|
1125
|
+
try:
|
|
1126
|
+
from .recall.complexity import suggest
|
|
1127
|
+
budget = max(1, min(5, int(suggest(query).get("budget", budget))))
|
|
1128
|
+
except Exception:
|
|
1129
|
+
pass
|
|
1130
|
+
self._last_constraint_hint = None
|
|
1131
|
+
if self._flag("constraints"):
|
|
1132
|
+
try:
|
|
1133
|
+
from .recall.constraints import check_constraints
|
|
1134
|
+
self._last_constraint_hint = check_constraints(
|
|
1135
|
+
query, open_loops=self._open_loops,
|
|
1136
|
+
commitments=[x for x in self.store.entries
|
|
1137
|
+
if isinstance(x, dict) and x.get("promise")])
|
|
1138
|
+
except Exception:
|
|
1139
|
+
self._last_constraint_hint = None
|
|
1140
|
+
rows = recall_entries(entries, query, budget=budget, vector=vector,
|
|
1141
|
+
embedder=self.embedder,
|
|
1142
|
+
three_channel=self.three_channel,
|
|
1143
|
+
now=float(self.now()), cue_scorer=cue_score)
|
|
1144
|
+
if self._flag("bm25"):
|
|
1145
|
+
try:
|
|
1146
|
+
from .recall.bm25 import score as _bm25
|
|
1147
|
+
from .recall.bm25 import score_multi as _bm25_multi
|
|
1148
|
+
scorer = _bm25_multi if self._flag("bm25_multi") else _bm25
|
|
1149
|
+
for row in rows:
|
|
1150
|
+
text = str((row.get("entry") or {}).get("text") or "")
|
|
1151
|
+
row["score"] = round(0.7 * float(row.get("score") or 0.0)
|
|
1152
|
+
+ 0.3 * scorer(query, text), 6)
|
|
1153
|
+
rows.sort(key=lambda item: item.get("score") or 0.0, reverse=True)
|
|
1154
|
+
except Exception:
|
|
1155
|
+
pass
|
|
1156
|
+
rows = self._apply_entity_boost(rows, query)
|
|
1157
|
+
if self._flag("ppr"):
|
|
1158
|
+
self._apply_ppr(rows, entries)
|
|
1159
|
+
rows = self._apply_source_consistency(rows)
|
|
1160
|
+
if self._flag("mmr") and len(rows) > 1:
|
|
1161
|
+
try:
|
|
1162
|
+
from .recall import mmr as mmr_mod
|
|
1163
|
+
# MMR diversity needs candidate vectors: expose entry.vector on
|
|
1164
|
+
# the row (mmr_select reads row["vector"] via _vec()).
|
|
1165
|
+
attached = []
|
|
1166
|
+
for row in rows:
|
|
1167
|
+
if not isinstance(row, dict) or "vector" in row:
|
|
1168
|
+
continue
|
|
1169
|
+
entry = row.get("entry")
|
|
1170
|
+
vector = entry.get("vector") if isinstance(entry, dict) else None
|
|
1171
|
+
if isinstance(vector, list) and vector:
|
|
1172
|
+
row["vector"] = list(vector)
|
|
1173
|
+
attached.append(row)
|
|
1174
|
+
previous = mmr_mod.MMR_ENABLED
|
|
1175
|
+
mmr_mod.MMR_ENABLED = True
|
|
1176
|
+
try:
|
|
1177
|
+
rows = mmr_mod.mmr_select(rows, k=budget,
|
|
1178
|
+
cosine_fn=cosine_vectors)
|
|
1179
|
+
finally:
|
|
1180
|
+
mmr_mod.MMR_ENABLED = previous
|
|
1181
|
+
for row in attached:
|
|
1182
|
+
row.pop("vector", None)
|
|
1183
|
+
except Exception:
|
|
1184
|
+
pass
|
|
1185
|
+
if self._flag("tiering"):
|
|
1186
|
+
annotated = annotate_tiers(rows)
|
|
1187
|
+
self._last_recall_tiers = {
|
|
1188
|
+
"high": sum(1 for r in annotated if r.get("tier") == "high"),
|
|
1189
|
+
"mid": sum(1 for r in annotated if r.get("tier") == "mid"),
|
|
1190
|
+
"low": sum(1 for r in annotated if r.get("tier") == "low")}
|
|
1191
|
+
for key, value in self._last_recall_tiers.items():
|
|
1192
|
+
if value:
|
|
1193
|
+
self._bump("tier_" + key, value)
|
|
1194
|
+
rows = [r for r in annotated if r.get("usage") != "drop"]
|
|
1195
|
+
else:
|
|
1196
|
+
self._last_recall_tiers = None
|
|
1197
|
+
stamp = float(self.now())
|
|
1198
|
+
self._apply_retrieval_practice(rows, stamp)
|
|
1199
|
+
if self.slot_registry is not None:
|
|
1200
|
+
for row in rows:
|
|
1201
|
+
self._slots_ingest(row.get("entry") or {}, cue=row.get("score", 0.0))
|
|
1202
|
+
if self._flag("inhibition"):
|
|
1203
|
+
selected = [str((r.get("entry") or {}).get("id") or "") for r in rows]
|
|
1204
|
+
self._mark_inhibition(entries, selected)
|
|
1205
|
+
if self._flag("compress"):
|
|
1206
|
+
self._compress_entries(entries)
|
|
1207
|
+
self._bump("recall_count")
|
|
1208
|
+
self._audit_write("recall", "", "recall",
|
|
1209
|
+
"query=%s rows=%d mode=%s" % (str(query)[:40], len(rows), mode))
|
|
1210
|
+
self._flush_stats()
|
|
1211
|
+
return rows
|
|
1212
|
+
|
|
1213
|
+
def _apply_ppr(self, rows, entries):
|
|
1214
|
+
try:
|
|
1215
|
+
from .recall.ppr import expand_ppr
|
|
1216
|
+
except Exception:
|
|
1217
|
+
return rows
|
|
1218
|
+
edges = self._graph_edges() if self._flag("graph") else []
|
|
1219
|
+
if not edges:
|
|
1220
|
+
for entry in entries:
|
|
1221
|
+
for rid in (entry.get("related_ids") or []):
|
|
1222
|
+
edges.append((str(entry.get("id") or ""), str(rid)))
|
|
1223
|
+
if not edges:
|
|
1224
|
+
return rows
|
|
1225
|
+
seeds = [str((row.get("entry") or {}).get("id") or "")
|
|
1226
|
+
for row in rows[:2]]
|
|
1227
|
+
seeds = [seed for seed in seeds if seed]
|
|
1228
|
+
if not seeds:
|
|
1229
|
+
return rows
|
|
1230
|
+
try:
|
|
1231
|
+
extra = expand_ppr(seeds, edges, hops=2, damping=0.5)
|
|
1232
|
+
except Exception:
|
|
1233
|
+
return rows
|
|
1234
|
+
by_id = {str(x.get("id") or ""): x for x in entries}
|
|
1235
|
+
added = 0
|
|
1236
|
+
for entry_id, ppr_score in extra[:2]:
|
|
1237
|
+
entry = by_id.get(entry_id)
|
|
1238
|
+
if entry is None:
|
|
1239
|
+
continue
|
|
1240
|
+
rows.append({"entry": entry, "score": round(float(ppr_score), 6),
|
|
1241
|
+
"mode": "ppr", "reasons": ["ppr"]})
|
|
1242
|
+
added += 1
|
|
1243
|
+
if added:
|
|
1244
|
+
self._bump("ppr_expansions", added)
|
|
1245
|
+
return rows
|
|
1246
|
+
|
|
1247
|
+
# -- forget / invalidate ----------------------------------------------
|
|
1248
|
+
def forget(self, entry_id, hard=False, reason="deleted", now=None, scope=None):
|
|
1249
|
+
entry = self.store.find(entry_id)
|
|
1250
|
+
if entry is None:
|
|
1251
|
+
return False
|
|
1252
|
+
if scope is not None and str(entry.get("scope") or "default") != str(scope):
|
|
1253
|
+
return False
|
|
1254
|
+
stamp = now_ts(now or self.now())
|
|
1255
|
+
before = len(self.store.tombstones)
|
|
1256
|
+
removed = self.store.remove(entry_id, now=stamp, reason=reason, hard=hard)
|
|
1257
|
+
if removed:
|
|
1258
|
+
if not hard and self._flag("provenance"):
|
|
1259
|
+
try:
|
|
1260
|
+
from .store.provenance import append_tombstone
|
|
1261
|
+
del self.store.tombstones[before:]
|
|
1262
|
+
append_tombstone(self.store.tombstones, entry, now=stamp,
|
|
1263
|
+
reason=reason)
|
|
1264
|
+
except Exception:
|
|
1265
|
+
pass
|
|
1266
|
+
self._propagate_delete(entry, now=stamp)
|
|
1267
|
+
self._tombstone_chain = None
|
|
1268
|
+
if self._flag("deletion_audit"):
|
|
1269
|
+
try:
|
|
1270
|
+
self._clean_derived_refs(entry_id, entry.get("text"))
|
|
1271
|
+
except Exception:
|
|
1272
|
+
pass
|
|
1273
|
+
try:
|
|
1274
|
+
self._last_deletion_audit = self.deletion_audit(
|
|
1275
|
+
entry_id, text=str(entry.get("text") or ""))
|
|
1276
|
+
self._bump("deletion_audits")
|
|
1277
|
+
self._audit_write(
|
|
1278
|
+
"deletion_audit", entry_id, "forget",
|
|
1279
|
+
"residue=%d" % (self._last_deletion_audit.get("residue") or 0))
|
|
1280
|
+
except Exception:
|
|
1281
|
+
self._last_deletion_audit = None
|
|
1282
|
+
self._audit_write("forget", entry_id, "forget", reason)
|
|
1283
|
+
self._flush_stats()
|
|
1284
|
+
self._write_snapshot_safe()
|
|
1285
|
+
return removed
|
|
1286
|
+
|
|
1287
|
+
def invalidate(self, entry_id, now=None):
|
|
1288
|
+
"""Mark an entry invalid at ts (dual-temporal); it stays stored."""
|
|
1289
|
+
entry = self.store.find(entry_id)
|
|
1290
|
+
if entry is None:
|
|
1291
|
+
return None
|
|
1292
|
+
stamp = now_ts(now or self.now())
|
|
1293
|
+
try:
|
|
1294
|
+
from .store.versions import mark_invalid
|
|
1295
|
+
patch = mark_invalid(entry_id, at=stamp)
|
|
1296
|
+
if isinstance(patch, dict):
|
|
1297
|
+
patch = dict(patch)
|
|
1298
|
+
patch.pop("id", None)
|
|
1299
|
+
self.store.update(entry_id, **patch)
|
|
1300
|
+
except Exception:
|
|
1301
|
+
return None
|
|
1302
|
+
self._audit_write("invalidate", entry_id, "invalidate")
|
|
1303
|
+
self._flush_stats()
|
|
1304
|
+
return self.store.find(entry_id)
|
|
1305
|
+
|
|
1306
|
+
def pin(self, entry_id, pinned=True):
|
|
1307
|
+
result = self.store.pin(entry_id, pinned)
|
|
1308
|
+
if result:
|
|
1309
|
+
self._audit_write("pin" if pinned else "unpin", entry_id, "pin")
|
|
1310
|
+
self._flush_stats()
|
|
1311
|
+
return result
|
|
1312
|
+
|
|
1313
|
+
def _propagate_delete(self, entry, now=None):
|
|
1314
|
+
"""Mark related anchors, drop identity evidence, clean thread refs."""
|
|
1315
|
+
target = str(entry.get("id") or "")
|
|
1316
|
+
text = str(entry.get("text") or "")
|
|
1317
|
+
count = 0
|
|
1318
|
+
for other in self.store.entries:
|
|
1319
|
+
if not isinstance(other, dict):
|
|
1320
|
+
continue
|
|
1321
|
+
related = (str(other.get("anchor") or "") == target
|
|
1322
|
+
or target in [str(x) for x in (other.get("related_ids") or [])]
|
|
1323
|
+
or str(other.get("event_anchor") or "") == target)
|
|
1324
|
+
evidence = other.get("identity_evidence")
|
|
1325
|
+
if isinstance(evidence, list):
|
|
1326
|
+
kept = [x for x in evidence
|
|
1327
|
+
if not (isinstance(x, dict) and target in
|
|
1328
|
+
(str(x.get("id") or ""), str(x.get("entry_id") or "")))]
|
|
1329
|
+
if len(kept) != len(evidence):
|
|
1330
|
+
self.store.update(other.get("id"), identity_evidence=kept)
|
|
1331
|
+
count += 1
|
|
1332
|
+
if related:
|
|
1333
|
+
self.store.update(other.get("id"), related_deleted=True,
|
|
1334
|
+
deleted_anchor=target)
|
|
1335
|
+
count += 1
|
|
1336
|
+
for key, card in list(self._threads.items()):
|
|
1337
|
+
if isinstance(card, dict) and (target in str(card.get("id") or "")
|
|
1338
|
+
or text[:20] in str(card.get("topic") or "")):
|
|
1339
|
+
self._threads.pop(key, None)
|
|
1340
|
+
count += 1
|
|
1341
|
+
self._open_loops = [x for x in self._open_loops
|
|
1342
|
+
if target not in str(x.get("id") or "")
|
|
1343
|
+
and text[:20] not in str(x.get("topic") or "")]
|
|
1344
|
+
if count:
|
|
1345
|
+
self._bump("delete_propagation", count)
|
|
1346
|
+
return count
|
|
1347
|
+
|
|
1348
|
+
def _clean_derived_refs(self, target_id, text=None):
|
|
1349
|
+
"""Drop a deleted id from derived summary entries' source_ids."""
|
|
1350
|
+
target = str(target_id or "")
|
|
1351
|
+
if not target:
|
|
1352
|
+
return 0
|
|
1353
|
+
cleaned = 0
|
|
1354
|
+
for entry in self.store.entries:
|
|
1355
|
+
if not isinstance(entry, dict):
|
|
1356
|
+
continue
|
|
1357
|
+
source = str(entry.get("source") or entry.get("memory_source") or "")
|
|
1358
|
+
if source not in _SUMMARY_SOURCES:
|
|
1359
|
+
continue
|
|
1360
|
+
changed = False
|
|
1361
|
+
refs = entry.get("source_ids")
|
|
1362
|
+
if isinstance(refs, list):
|
|
1363
|
+
kept = [str(x) for x in refs if str(x) != target]
|
|
1364
|
+
if len(kept) != len(refs):
|
|
1365
|
+
entry["source_ids"] = kept
|
|
1366
|
+
changed = True
|
|
1367
|
+
meta = entry.get("metadata")
|
|
1368
|
+
if isinstance(meta, dict) and isinstance(meta.get("source_ids"), list):
|
|
1369
|
+
kept = [str(x) for x in meta["source_ids"] if str(x) != target]
|
|
1370
|
+
if len(kept) != len(meta["source_ids"]):
|
|
1371
|
+
meta["source_ids"] = kept
|
|
1372
|
+
changed = True
|
|
1373
|
+
if changed:
|
|
1374
|
+
entry["derived_stale"] = True
|
|
1375
|
+
entry["derived_updated_ts"] = round(now_ts(self.now()), 3)
|
|
1376
|
+
cleaned += 1
|
|
1377
|
+
return cleaned
|
|
1378
|
+
|
|
1379
|
+
def deletion_audit(self, entry_id, text=None):
|
|
1380
|
+
"""Read-only coverage audit of derived data after one deletion.
|
|
1381
|
+
|
|
1382
|
+
Returns {target, covered, missed, residue}: covered lists the derived
|
|
1383
|
+
areas with no residual reference, missed lists residual areas (with a
|
|
1384
|
+
reason). Tombstone rows are intentionally retained (audit chain) and
|
|
1385
|
+
are reported as covered, never as missed.
|
|
1386
|
+
"""
|
|
1387
|
+
target = str(entry_id or "")
|
|
1388
|
+
text = str(text or "")
|
|
1389
|
+
report = {"target": target, "covered": [], "missed": [], "residue": 0}
|
|
1390
|
+
if not target:
|
|
1391
|
+
report["missed"].append({"area": "target", "count": 1,
|
|
1392
|
+
"reason": "empty_entry_id"})
|
|
1393
|
+
report["residue"] = 1
|
|
1394
|
+
return report
|
|
1395
|
+
entries = [x for x in self.store.entries if isinstance(x, dict)]
|
|
1396
|
+
|
|
1397
|
+
def _mark(area, ids, reason):
|
|
1398
|
+
if ids:
|
|
1399
|
+
report["missed"].append({"area": area, "count": len(ids),
|
|
1400
|
+
"reason": reason, "ids": ids[:8]})
|
|
1401
|
+
report["residue"] += len(ids)
|
|
1402
|
+
else:
|
|
1403
|
+
report["covered"].append(area)
|
|
1404
|
+
|
|
1405
|
+
# 1) consolidation summaries (role=system/source=consolidate etc.)
|
|
1406
|
+
summary_ids = []
|
|
1407
|
+
for entry in entries:
|
|
1408
|
+
source = str(entry.get("source") or entry.get("memory_source") or "")
|
|
1409
|
+
if source not in _SUMMARY_SOURCES:
|
|
1410
|
+
continue
|
|
1411
|
+
refs = entry.get("source_ids")
|
|
1412
|
+
refs = list(refs) if isinstance(refs, list) else []
|
|
1413
|
+
meta = entry.get("metadata")
|
|
1414
|
+
if isinstance(meta, dict) and isinstance(meta.get("source_ids"), list):
|
|
1415
|
+
refs += meta["source_ids"]
|
|
1416
|
+
if target in [str(x) for x in refs]:
|
|
1417
|
+
summary_ids.append(str(entry.get("id") or ""))
|
|
1418
|
+
_mark("summary", summary_ids, "summary source_ids still reference target")
|
|
1419
|
+
|
|
1420
|
+
# 2) identity evidence / anchors (deleted_anchor is a cleanup marker)
|
|
1421
|
+
identity_ids = []
|
|
1422
|
+
for other in entries:
|
|
1423
|
+
evidence = other.get("identity_evidence")
|
|
1424
|
+
if isinstance(evidence, list) and any(
|
|
1425
|
+
isinstance(x, dict) and target in
|
|
1426
|
+
(str(x.get("id") or ""), str(x.get("entry_id") or ""))
|
|
1427
|
+
for x in evidence):
|
|
1428
|
+
identity_ids.append(str(other.get("id") or ""))
|
|
1429
|
+
continue
|
|
1430
|
+
if (str(other.get("anchor") or "") == target
|
|
1431
|
+
or str(other.get("event_anchor") or "") == target):
|
|
1432
|
+
identity_ids.append(str(other.get("id") or ""))
|
|
1433
|
+
_mark("identity_evidence", identity_ids,
|
|
1434
|
+
"identity evidence/anchor still references target")
|
|
1435
|
+
|
|
1436
|
+
# 3) index/graph edges (derived from live, non-deleted entries)
|
|
1437
|
+
graph_ids = []
|
|
1438
|
+
try:
|
|
1439
|
+
from .index.graph import build_links
|
|
1440
|
+
links = build_links([x for x in entries if not x.get("deleted")]) or {}
|
|
1441
|
+
if target in links:
|
|
1442
|
+
graph_ids.append(target)
|
|
1443
|
+
graph_ids.extend(str(key) for key, values in links.items()
|
|
1444
|
+
if target in [str(x) for x in (values or [])])
|
|
1445
|
+
except Exception:
|
|
1446
|
+
links = {}
|
|
1447
|
+
_mark("graph", graph_ids[:8], "graph still contains target node/edge")
|
|
1448
|
+
|
|
1449
|
+
# 4) thread cards / open loops
|
|
1450
|
+
thread_ids = []
|
|
1451
|
+
for key, card in self._threads.items():
|
|
1452
|
+
if not isinstance(card, dict):
|
|
1453
|
+
continue
|
|
1454
|
+
topic = str(card.get("topic") or "")
|
|
1455
|
+
if (target in str(card.get("id") or "")
|
|
1456
|
+
or (text and text[:20] and text[:20] in topic)):
|
|
1457
|
+
thread_ids.append(str(card.get("id") or key))
|
|
1458
|
+
for loop in self._open_loops:
|
|
1459
|
+
label = str((loop or {}).get("id") or (loop or {}).get("topic") or "") if isinstance(loop, dict) else str(loop)
|
|
1460
|
+
if target in label or (text and text[:20] and text[:20] in label):
|
|
1461
|
+
thread_ids.append(label[:80])
|
|
1462
|
+
_mark("thread", thread_ids[:8], "thread/open-loop still references target")
|
|
1463
|
+
|
|
1464
|
+
# 5) tombstones are retained on purpose (hash-chain audit trail)
|
|
1465
|
+
tomb_count = sum(1 for tomb in self.store.tombstones
|
|
1466
|
+
if isinstance(tomb, dict)
|
|
1467
|
+
and str(tomb.get("id") or "") == target)
|
|
1468
|
+
if tomb_count:
|
|
1469
|
+
report["covered"].append("tombstone_chain(intentional, retained)")
|
|
1470
|
+
else:
|
|
1471
|
+
report["covered"].append("tombstone_chain(absent)")
|
|
1472
|
+
return report
|
|
1473
|
+
|
|
1474
|
+
# -- slots ------------------------------------------------------------
|
|
1475
|
+
def _slots_ingest(self, entry, cue=None):
|
|
1476
|
+
if self.slot_registry is None or not isinstance(entry, dict):
|
|
1477
|
+
return None
|
|
1478
|
+
stamp = now_ts(self.now())
|
|
1479
|
+
ctx = {"now": stamp,
|
|
1480
|
+
"cue": cue if cue is not None else 1.0,
|
|
1481
|
+
"goal": float(entry.get("goal_relevance", 0.5) or 0.5),
|
|
1482
|
+
"boundary_score": float(entry.get("boundary_score", 0.0) or 0.0),
|
|
1483
|
+
"salience": float(entry.get("importance", 0.5) or 0.5),
|
|
1484
|
+
"slot_count": len(getattr(self.slot_registry, "slots", []) or []),
|
|
1485
|
+
"total_chars": self.slot_registry.total_chars(),
|
|
1486
|
+
"budget_chars": 360,
|
|
1487
|
+
"preempt_used_ts": stamp}
|
|
1488
|
+
boundary_score, is_boundary = self._boundary_signal(entry, stamp)
|
|
1489
|
+
if boundary_score:
|
|
1490
|
+
ctx["boundary_score"] = max(ctx["boundary_score"], boundary_score)
|
|
1491
|
+
if is_boundary:
|
|
1492
|
+
ctx["boundary_score"] = 1.0
|
|
1493
|
+
try:
|
|
1494
|
+
self._last_slots = slots_ingest(self.slot_registry, entry, ctx)
|
|
1495
|
+
if self._last_slots.get("evicted"):
|
|
1496
|
+
self._bump("slots_evicted", len(self._last_slots["evicted"]))
|
|
1497
|
+
if self._last_slots.get("decision") == "admit":
|
|
1498
|
+
self._bump("slots_admitted")
|
|
1499
|
+
return self._last_slots
|
|
1500
|
+
except Exception:
|
|
1501
|
+
self._last_slots = None
|
|
1502
|
+
return None
|
|
1503
|
+
|
|
1504
|
+
# -- tick / eviction --------------------------------------------------
|
|
1505
|
+
def tick(self, now=None, apply_evictions=None):
|
|
1506
|
+
stamp = now_ts(now or self.now())
|
|
1507
|
+
entries = [x for x in self.store.entries if isinstance(x, dict)]
|
|
1508
|
+
evictions_requested = (bool(apply_evictions)
|
|
1509
|
+
if apply_evictions is not None
|
|
1510
|
+
else self._flag("apply_evictions"))
|
|
1511
|
+
tiering_exec_report = self._apply_tiering_exec(entries, stamp)
|
|
1512
|
+
total = max(1, len(entries))
|
|
1513
|
+
adaptive_on = self._flag("adaptive_evict")
|
|
1514
|
+
load = self._store_load() if adaptive_on else 0.0
|
|
1515
|
+
adaptive_scored = 0
|
|
1516
|
+
rows = []
|
|
1517
|
+
for position, entry in enumerate(entries):
|
|
1518
|
+
redundancy = 1.0 if any(
|
|
1519
|
+
other is not entry
|
|
1520
|
+
and str(other.get("text") or "") == str(entry.get("text") or "")
|
|
1521
|
+
for other in entries) else 0.0
|
|
1522
|
+
base = eviction_score(entry, now=stamp,
|
|
1523
|
+
position=position / float(total),
|
|
1524
|
+
redundancy=redundancy)
|
|
1525
|
+
score = base
|
|
1526
|
+
if adaptive_on:
|
|
1527
|
+
score = self._adaptive_eviction_score(entry, base, load)
|
|
1528
|
+
adaptive_scored += 1
|
|
1529
|
+
row = {"entry": entry, "score": score,
|
|
1530
|
+
"id": str(entry.get("id") or "")}
|
|
1531
|
+
if adaptive_on:
|
|
1532
|
+
row["base_score"] = base
|
|
1533
|
+
rows.append(row)
|
|
1534
|
+
ordered = eviction_order(rows)
|
|
1535
|
+
result = {"now": stamp, "candidates": ordered[:20], "count": len(ordered),
|
|
1536
|
+
"deleted": 0, "readonly": True, "policy": ordered[:20],
|
|
1537
|
+
"stats": self._stats_snapshot(ordered)}
|
|
1538
|
+
if adaptive_on:
|
|
1539
|
+
result["adaptive"] = {"load": round(load, 4),
|
|
1540
|
+
"scored": adaptive_scored}
|
|
1541
|
+
if adaptive_scored:
|
|
1542
|
+
self._bump("adaptive_scored", adaptive_scored)
|
|
1543
|
+
eviction_report = None
|
|
1544
|
+
if evictions_requested:
|
|
1545
|
+
eviction_report = self._apply_evictions(stamp)
|
|
1546
|
+
if isinstance(eviction_report, dict):
|
|
1547
|
+
result["apply_evictions"] = eviction_report
|
|
1548
|
+
result["deleted"] = len(eviction_report.get("deleted") or [])
|
|
1549
|
+
result["readonly"] = False
|
|
1550
|
+
if self._flag("heat"):
|
|
1551
|
+
try:
|
|
1552
|
+
from .evict import heat as heat_mod
|
|
1553
|
+
previous = heat_mod.HEAT_ENABLED
|
|
1554
|
+
heat_mod.HEAT_ENABLED = True
|
|
1555
|
+
try:
|
|
1556
|
+
result["heat"] = {str(x.get("id") or ""): heat_mod.heat(x, now=stamp)
|
|
1557
|
+
for x in entries}
|
|
1558
|
+
finally:
|
|
1559
|
+
heat_mod.HEAT_ENABLED = previous
|
|
1560
|
+
except Exception:
|
|
1561
|
+
result["heat"] = {}
|
|
1562
|
+
if self._flag("consequence"):
|
|
1563
|
+
try:
|
|
1564
|
+
from .evict import consequence as consequence_mod
|
|
1565
|
+
previous = consequence_mod.CONSEQUENCE_ENABLED
|
|
1566
|
+
consequence_mod.CONSEQUENCE_ENABLED = True
|
|
1567
|
+
try:
|
|
1568
|
+
result["consequence"] = {
|
|
1569
|
+
str(x.get("id") or ""): consequence_mod.consequence(x)
|
|
1570
|
+
for x in entries}
|
|
1571
|
+
finally:
|
|
1572
|
+
consequence_mod.CONSEQUENCE_ENABLED = previous
|
|
1573
|
+
except Exception:
|
|
1574
|
+
result["consequence"] = {}
|
|
1575
|
+
if tiering_exec_report is not None:
|
|
1576
|
+
result["tiering_exec"] = tiering_exec_report
|
|
1577
|
+
compressed = self._compress_entries(entries)
|
|
1578
|
+
if compressed is not None:
|
|
1579
|
+
result["compress"] = compressed
|
|
1580
|
+
result["stats"] = self._stats_snapshot(ordered)
|
|
1581
|
+
self._audit_write("tick", "", "tick",
|
|
1582
|
+
"entries=%d candidates=%d" % (len(entries), len(ordered)))
|
|
1583
|
+
if ((tiering_exec_report or {}).get("archived") or compressed
|
|
1584
|
+
or (eviction_report or {}).get("deleted")):
|
|
1585
|
+
self._flush_stats()
|
|
1586
|
+
self._write_snapshot_safe()
|
|
1587
|
+
return result
|
|
1588
|
+
|
|
1589
|
+
def _apply_evictions(self, now=None):
|
|
1590
|
+
"""Soft-delete archived entries (never removes data; never touches pins).
|
|
1591
|
+
|
|
1592
|
+
Uses the read-only evict/policy partition to find low-value candidates,
|
|
1593
|
+
then marks each one deleted=true with a tombstone. Protected entries
|
|
1594
|
+
(pin / identity anchor+evidence / promise / P0-P1) are always skipped.
|
|
1595
|
+
"""
|
|
1596
|
+
stamp = now_ts(now or self.now())
|
|
1597
|
+
try:
|
|
1598
|
+
from .evict.policy import tick as policy_tick
|
|
1599
|
+
plan = policy_tick(self.store.entries, now=stamp)
|
|
1600
|
+
except Exception:
|
|
1601
|
+
return {"applied": False, "reason": "policy_unavailable",
|
|
1602
|
+
"deleted": [], "protected": 0, "candidates": 0,
|
|
1603
|
+
"tombstones": 0, "now": stamp}
|
|
1604
|
+
candidates = [str(x) for x in (plan.get("archive") or []) if str(x)]
|
|
1605
|
+
deleted = []
|
|
1606
|
+
protected = 0
|
|
1607
|
+
tombstones_before = len(self.store.tombstones)
|
|
1608
|
+
for entry_id in candidates:
|
|
1609
|
+
entry = self.store.find(entry_id)
|
|
1610
|
+
if not isinstance(entry, dict) or entry.get("deleted"):
|
|
1611
|
+
continue
|
|
1612
|
+
if is_protected_entry(entry):
|
|
1613
|
+
protected += 1
|
|
1614
|
+
continue
|
|
1615
|
+
entry["deleted"] = True
|
|
1616
|
+
entry["deleted_ts"] = round(stamp, 3)
|
|
1617
|
+
entry["deleted_reason"] = "evicted"
|
|
1618
|
+
try:
|
|
1619
|
+
if self._flag("provenance"):
|
|
1620
|
+
from .store.provenance import append_tombstone
|
|
1621
|
+
else:
|
|
1622
|
+
from .store.tombstone import append_tombstone
|
|
1623
|
+
append_tombstone(self.store.tombstones, entry, now=stamp,
|
|
1624
|
+
reason="evicted")
|
|
1625
|
+
except Exception:
|
|
1626
|
+
pass
|
|
1627
|
+
deleted.append(entry_id)
|
|
1628
|
+
report = {"applied": True, "deleted": deleted, "protected": protected,
|
|
1629
|
+
"candidates": len(candidates),
|
|
1630
|
+
"tombstones": max(0, len(self.store.tombstones) - tombstones_before),
|
|
1631
|
+
"now": stamp}
|
|
1632
|
+
if deleted:
|
|
1633
|
+
self._bump("evictions_applied", len(deleted))
|
|
1634
|
+
self._bump("evict_tombstones", report["tombstones"])
|
|
1635
|
+
self._audit_write("evictions", "", "tick",
|
|
1636
|
+
"deleted=%d protected=%d" % (len(deleted), protected))
|
|
1637
|
+
self._last_evictions = report
|
|
1638
|
+
return report
|
|
1639
|
+
|
|
1640
|
+
def restore_evictions(self, entry_ids=None, now=None):
|
|
1641
|
+
"""Roll back soft evictions: deleted=false, all original fields kept."""
|
|
1642
|
+
stamp = now_ts(now or self.now())
|
|
1643
|
+
if entry_ids is not None:
|
|
1644
|
+
try:
|
|
1645
|
+
ids = [str(x) for x in entry_ids]
|
|
1646
|
+
except Exception:
|
|
1647
|
+
ids = []
|
|
1648
|
+
else:
|
|
1649
|
+
ids = [str(entry.get("id") or "") for entry in self.store.entries
|
|
1650
|
+
if isinstance(entry, dict)
|
|
1651
|
+
and entry.get("deleted_reason") == "evicted"]
|
|
1652
|
+
restored = []
|
|
1653
|
+
for entry_id in ids:
|
|
1654
|
+
entry = self.store.find(entry_id)
|
|
1655
|
+
if not isinstance(entry, dict):
|
|
1656
|
+
continue
|
|
1657
|
+
if entry.get("deleted_reason") != "evicted":
|
|
1658
|
+
continue
|
|
1659
|
+
entry["deleted"] = False
|
|
1660
|
+
entry.pop("deleted_ts", None)
|
|
1661
|
+
entry.pop("deleted_reason", None)
|
|
1662
|
+
entry["restored_ts"] = round(stamp, 3)
|
|
1663
|
+
restored.append(entry_id)
|
|
1664
|
+
if restored:
|
|
1665
|
+
self._bump("evictions_restored", len(restored))
|
|
1666
|
+
self._audit_write("evictions_restored", "", "restore",
|
|
1667
|
+
"restored=%d" % len(restored))
|
|
1668
|
+
self._flush_stats()
|
|
1669
|
+
self._write_snapshot_safe()
|
|
1670
|
+
return {"restored": restored, "count": len(restored), "now": stamp}
|
|
1671
|
+
|
|
1672
|
+
# -- identity / relay / thread ---------------------------------------
|
|
1673
|
+
def identity_snapshot(self):
|
|
1674
|
+
return _identity_snapshot(self.store.entries)
|
|
1675
|
+
|
|
1676
|
+
def relay_card(self):
|
|
1677
|
+
if not self._flag("relay"):
|
|
1678
|
+
return {"identity": [], "relationship": [], "recent_changes": [],
|
|
1679
|
+
"promises": [], "open_loops": []}
|
|
1680
|
+
card = _relay_card(self.store.entries, now=self.now(),
|
|
1681
|
+
open_loops=self._open_loops,
|
|
1682
|
+
due_enabled=self.due_hints_enabled)
|
|
1683
|
+
self._bump("relay_count")
|
|
1684
|
+
return card
|
|
1685
|
+
|
|
1686
|
+
def thread_update(self, topic, status="active", now=None):
|
|
1687
|
+
if not self._flag("thread"):
|
|
1688
|
+
return {}
|
|
1689
|
+
self._threads, card = update_thread(self._threads, topic, status=status,
|
|
1690
|
+
now=now or self.now())
|
|
1691
|
+
self._expire_threads(now=now)
|
|
1692
|
+
return card or {}
|
|
1693
|
+
|
|
1694
|
+
def thread_card(self, now=None, ttl=None):
|
|
1695
|
+
if not self._flag("thread"):
|
|
1696
|
+
return []
|
|
1697
|
+
self._expire_threads(now=now, ttl=ttl)
|
|
1698
|
+
kwargs = {"threads": self._threads,
|
|
1699
|
+
"now": now_ts(now or self.now())}
|
|
1700
|
+
if ttl is not None:
|
|
1701
|
+
kwargs["ttl"] = ttl
|
|
1702
|
+
return _thread_card(self.store.entries, **kwargs)
|
|
1703
|
+
|
|
1704
|
+
def _expire_threads(self, now=None, ttl=None):
|
|
1705
|
+
args = {"now": now or self.now()}
|
|
1706
|
+
if ttl is not None:
|
|
1707
|
+
args["ttl"] = ttl
|
|
1708
|
+
active, loops = expire_threads(self._threads, **args)
|
|
1709
|
+
changed = False
|
|
1710
|
+
for card in loops:
|
|
1711
|
+
topic = card.get("topic")
|
|
1712
|
+
if topic and topic not in [x.get("topic") for x in self._open_loops]:
|
|
1713
|
+
self._open_loops.append({"topic": topic, "ts": card.get("updated_ts")})
|
|
1714
|
+
changed = True
|
|
1715
|
+
if changed:
|
|
1716
|
+
self._open_loops = self._open_loops[-20:]
|
|
1717
|
+
return active, loops
|
|
1718
|
+
|
|
1719
|
+
# -- import / export --------------------------------------------------
|
|
1720
|
+
def import_data(self, payload, scope=None):
|
|
1721
|
+
"""Symmetric with export: entries/tombstones/stats; failure is atomic."""
|
|
1722
|
+
if not isinstance(payload, dict):
|
|
1723
|
+
return {"ok": False, "reason": "invalid_payload"}
|
|
1724
|
+
old_doc = None
|
|
1725
|
+
try:
|
|
1726
|
+
entries = payload.get("entries")
|
|
1727
|
+
if not isinstance(entries, list):
|
|
1728
|
+
raise ValueError("entries must be a list")
|
|
1729
|
+
tombstones = payload.get("tombstones", [])
|
|
1730
|
+
if not isinstance(tombstones, list):
|
|
1731
|
+
raise ValueError("tombstones must be a list")
|
|
1732
|
+
stats = payload.get("stats")
|
|
1733
|
+
if not isinstance(stats, dict):
|
|
1734
|
+
meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
|
|
1735
|
+
stats = meta.get("stats", {})
|
|
1736
|
+
if not isinstance(stats, dict):
|
|
1737
|
+
raise ValueError("stats must be an object")
|
|
1738
|
+
selected = []
|
|
1739
|
+
for entry in entries:
|
|
1740
|
+
if not isinstance(entry, dict):
|
|
1741
|
+
raise ValueError("entry must be an object")
|
|
1742
|
+
if scope is not None and str(entry.get("scope") or "default") != str(scope):
|
|
1743
|
+
continue
|
|
1744
|
+
selected.append(dict(entry))
|
|
1745
|
+
old_doc = self.store.export()
|
|
1746
|
+
old_meta = dict(self.store.meta)
|
|
1747
|
+
new_meta = dict(old_meta)
|
|
1748
|
+
new_meta["stats"] = {str(k): int(v or 0) for k, v in stats.items()}
|
|
1749
|
+
doc = dict(old_doc)
|
|
1750
|
+
doc["entries"] = selected
|
|
1751
|
+
doc["tombstones"] = [dict(x) for x in tombstones if isinstance(x, dict)]
|
|
1752
|
+
doc["meta"] = new_meta
|
|
1753
|
+
self.store.import_doc(doc)
|
|
1754
|
+
if self.store.path:
|
|
1755
|
+
self.store.save()
|
|
1756
|
+
self._cooc_refresh()
|
|
1757
|
+
self._audit_write("import", "", "import",
|
|
1758
|
+
"entries=%d" % len(selected))
|
|
1759
|
+
return {"ok": True, "entries": len(selected),
|
|
1760
|
+
"tombstones": len(doc["tombstones"]), "stats": len(new_meta["stats"])}
|
|
1761
|
+
except Exception as exc:
|
|
1762
|
+
if old_doc is not None:
|
|
1763
|
+
try:
|
|
1764
|
+
self.store.import_doc(old_doc)
|
|
1765
|
+
except Exception:
|
|
1766
|
+
pass
|
|
1767
|
+
return {"ok": False, "reason": str(exc)[:120]}
|
|
1768
|
+
|
|
1769
|
+
# -- consolidation ----------------------------------------------------
|
|
1770
|
+
def consolidate(self, level=1, now=None):
|
|
1771
|
+
"""Gated consolidation pipeline; writes only with the *_writeback flag."""
|
|
1772
|
+
stage_flags = ("consolidate_gated", "consolidate_tree",
|
|
1773
|
+
"consolidate_reflection", "consolidate_neighbors",
|
|
1774
|
+
"consolidate_writeback")
|
|
1775
|
+
if not any(self._flag(key) for key in stage_flags):
|
|
1776
|
+
return {"enabled": False}
|
|
1777
|
+
entries = [x for x in self.store.entries
|
|
1778
|
+
if isinstance(x, dict) and not x.get("summarized")
|
|
1779
|
+
and not x.get("archived")]
|
|
1780
|
+
stamp = now_ts(now or self.now())
|
|
1781
|
+
result = {"enabled": True, "level": int(level or 1), "written": 0,
|
|
1782
|
+
"marked": 0}
|
|
1783
|
+
try:
|
|
1784
|
+
from .consolidate import gated as gated_mod
|
|
1785
|
+
from .consolidate import neighbors as neighbors_mod
|
|
1786
|
+
from .consolidate import reflection as reflection_mod
|
|
1787
|
+
from .consolidate import tree as tree_mod
|
|
1788
|
+
except Exception as exc:
|
|
1789
|
+
result["error"] = str(exc)[:120]
|
|
1790
|
+
return result
|
|
1791
|
+
previous = {
|
|
1792
|
+
"gated": gated_mod.GATED_ENABLED,
|
|
1793
|
+
"tree": tree_mod.TREE_ENABLED,
|
|
1794
|
+
"reflection": reflection_mod.REFLECTION_ENABLED,
|
|
1795
|
+
"neighbors": neighbors_mod.NEIGHBORS_ENABLED,
|
|
1796
|
+
}
|
|
1797
|
+
gated_mod.GATED_ENABLED = self._flag("consolidate_gated")
|
|
1798
|
+
tree_mod.TREE_ENABLED = self._flag("consolidate_tree")
|
|
1799
|
+
reflection_mod.REFLECTION_ENABLED = self._flag("consolidate_reflection")
|
|
1800
|
+
neighbors_mod.NEIGHBORS_ENABLED = self._flag("consolidate_neighbors")
|
|
1801
|
+
try:
|
|
1802
|
+
if self._flag("consolidate_gated"):
|
|
1803
|
+
result["gated"] = bool(gated_mod.should_consolidate(entries,
|
|
1804
|
+
now=stamp))
|
|
1805
|
+
if not result["gated"]:
|
|
1806
|
+
result["reason"] = "gate_closed"
|
|
1807
|
+
self._write_snapshot_safe()
|
|
1808
|
+
return result
|
|
1809
|
+
summary_ids = []
|
|
1810
|
+
tree_data = {}
|
|
1811
|
+
if self._flag("consolidate_tree"):
|
|
1812
|
+
result["tree"] = tree_mod.build_summary(entries, level=level)
|
|
1813
|
+
tree_data = (result["tree"]
|
|
1814
|
+
if isinstance(result["tree"], dict) else {})
|
|
1815
|
+
# Level items are only real summaries when generated is true.
|
|
1816
|
+
if tree_data.get("generated"):
|
|
1817
|
+
for item in (tree_data.get("items") or []):
|
|
1818
|
+
summary_ids.extend(item.get("source_ids") or [])
|
|
1819
|
+
if item.get("id"):
|
|
1820
|
+
summary_ids.append(item.get("id"))
|
|
1821
|
+
if self._flag("consolidate_reflection"):
|
|
1822
|
+
result["reflections"] = reflection_mod.propose(entries)
|
|
1823
|
+
for item in (result["reflections"] or []):
|
|
1824
|
+
summary_ids.extend(item.get("source_ids") or [])
|
|
1825
|
+
if self._flag("consolidate_neighbors"):
|
|
1826
|
+
result["neighbors"] = [neighbors_mod.strengthen(x, entries)
|
|
1827
|
+
for x in entries[:5]]
|
|
1828
|
+
summary_ids = sorted({str(x) for x in summary_ids if str(x)})
|
|
1829
|
+
real_summary = (bool(result.get("reflections"))
|
|
1830
|
+
or bool(tree_data.get("generated")))
|
|
1831
|
+
written = 0
|
|
1832
|
+
if self._flag("consolidate_writeback"):
|
|
1833
|
+
written += self._consolidate_writeback(result, stamp)
|
|
1834
|
+
if real_summary and written == 0 and summary_ids:
|
|
1835
|
+
summary = self._consolidate_summary(result, summary_ids, stamp)
|
|
1836
|
+
if summary is not None:
|
|
1837
|
+
written += 1
|
|
1838
|
+
result["summary_id"] = summary.get("id")
|
|
1839
|
+
result["summary"] = summary.get("text")
|
|
1840
|
+
if written == 0:
|
|
1841
|
+
# No real tree/reflection content: never write a placeholder
|
|
1842
|
+
# summary and never mark the source entries.
|
|
1843
|
+
result["noop"] = True
|
|
1844
|
+
if real_summary and written:
|
|
1845
|
+
for entry_id in summary_ids:
|
|
1846
|
+
if self.store.find(entry_id) is not None:
|
|
1847
|
+
self.store.update(entry_id, summarized=True)
|
|
1848
|
+
result["marked"] += 1
|
|
1849
|
+
narrative_report = self._apply_narrative(stamp)
|
|
1850
|
+
if narrative_report is not None:
|
|
1851
|
+
result["narrative"] = narrative_report
|
|
1852
|
+
result["written"] = written
|
|
1853
|
+
self._bump("consolidate_writes", written)
|
|
1854
|
+
self._bump("consolidate_runs")
|
|
1855
|
+
self._audit_write("consolidate", result.get("summary_id") or "",
|
|
1856
|
+
"consolidate",
|
|
1857
|
+
"written=%d marked=%d" % (result["written"],
|
|
1858
|
+
result["marked"]))
|
|
1859
|
+
self._flush_stats()
|
|
1860
|
+
self._write_snapshot_safe()
|
|
1861
|
+
return result
|
|
1862
|
+
finally:
|
|
1863
|
+
gated_mod.GATED_ENABLED = previous["gated"]
|
|
1864
|
+
tree_mod.TREE_ENABLED = previous["tree"]
|
|
1865
|
+
reflection_mod.REFLECTION_ENABLED = previous["reflection"]
|
|
1866
|
+
neighbors_mod.NEIGHBORS_ENABLED = previous["neighbors"]
|
|
1867
|
+
|
|
1868
|
+
def _consolidate_summary(self, result, source_ids, stamp):
|
|
1869
|
+
pieces = []
|
|
1870
|
+
reflections = result.get("reflections") or []
|
|
1871
|
+
if reflections:
|
|
1872
|
+
pieces.append(str(reflections[0].get("text") or ""))
|
|
1873
|
+
tree_data = result.get("tree") or {}
|
|
1874
|
+
if tree_data.get("generated"):
|
|
1875
|
+
pieces.append("时间树 L%s 已整理 %d 项" % (
|
|
1876
|
+
tree_data.get("level"),
|
|
1877
|
+
len(tree_data.get("items") or [])))
|
|
1878
|
+
if not pieces:
|
|
1879
|
+
pieces.append("已整理 %d 条记忆" % len(source_ids))
|
|
1880
|
+
text = ";".join(piece for piece in pieces if piece)[:200]
|
|
1881
|
+
if not text:
|
|
1882
|
+
return None
|
|
1883
|
+
try:
|
|
1884
|
+
summary = self.store.add({
|
|
1885
|
+
"id": sha12("consolidate" + text + str(stamp)),
|
|
1886
|
+
"text": text, "role": "system", "ts": stamp,
|
|
1887
|
+
"valid_at": stamp, "importance": 0.6,
|
|
1888
|
+
"memory_confidence": 0.8, "memory_source": "consolidate",
|
|
1889
|
+
"source": "consolidate", "module": "consolidate",
|
|
1890
|
+
"source_ids": list(source_ids)[:20], "scope": "default",
|
|
1891
|
+
"previous_versions": []})
|
|
1892
|
+
except Exception:
|
|
1893
|
+
return None
|
|
1894
|
+
return summary
|
|
1895
|
+
|
|
1896
|
+
def _consolidate_writeback(self, result, stamp):
|
|
1897
|
+
try:
|
|
1898
|
+
from .consolidate import writeback as writeback_mod
|
|
1899
|
+
except Exception:
|
|
1900
|
+
return 0
|
|
1901
|
+
previous = writeback_mod.WRITEBACK_ENABLED
|
|
1902
|
+
writeback_mod.WRITEBACK_ENABLED = True
|
|
1903
|
+
written = 0
|
|
1904
|
+
try:
|
|
1905
|
+
if result.get("reflections"):
|
|
1906
|
+
outcome = writeback_mod.apply_reflections(self.store,
|
|
1907
|
+
result["reflections"])
|
|
1908
|
+
written += len(outcome.get("written") or [])
|
|
1909
|
+
result["reflections_written"] = outcome
|
|
1910
|
+
# Surface source_ids at the top level for CLI/read-only consumers.
|
|
1911
|
+
for entry_id in outcome.get("written") or []:
|
|
1912
|
+
entry = self.store.find(entry_id)
|
|
1913
|
+
if not isinstance(entry, dict):
|
|
1914
|
+
continue
|
|
1915
|
+
meta = entry.get("metadata")
|
|
1916
|
+
ids = meta.get("source_ids") if isinstance(meta, dict) else None
|
|
1917
|
+
if isinstance(ids, list) and ids and not entry.get("source_ids"):
|
|
1918
|
+
entry["source_ids"] = [str(x) for x in ids][:20]
|
|
1919
|
+
if result.get("neighbors"):
|
|
1920
|
+
outcome = writeback_mod.apply_neighbors(self.store,
|
|
1921
|
+
result["neighbors"])
|
|
1922
|
+
written += int(outcome.get("linked") or 0)
|
|
1923
|
+
result["neighbors_written"] = outcome
|
|
1924
|
+
tree_data = result.get("tree") or {}
|
|
1925
|
+
if tree_data.get("generated"):
|
|
1926
|
+
outcome = writeback_mod.write_tree(self.store, tree_data)
|
|
1927
|
+
written += int(outcome.get("written") or 0)
|
|
1928
|
+
result["tree_written"] = outcome
|
|
1929
|
+
except Exception as exc:
|
|
1930
|
+
result["writeback_error"] = str(exc)[:100]
|
|
1931
|
+
finally:
|
|
1932
|
+
writeback_mod.WRITEBACK_ENABLED = previous
|
|
1933
|
+
return written
|
|
1934
|
+
|
|
1935
|
+
# -- identity write-back ----------------------------------------------
|
|
1936
|
+
def identity_write(self, trigger=None, evidence=None, delta=0.0,
|
|
1937
|
+
entry_id=None, direction=1, step=None, major_event=False):
|
|
1938
|
+
"""Delegate to identity gate: evidence>=3 or major event, <=0.02/step."""
|
|
1939
|
+
result = _identity_write(
|
|
1940
|
+
self.store, self.identity_writeback_enabled, self._identity_state,
|
|
1941
|
+
trigger=trigger, evidence=evidence, delta=delta, entry_id=entry_id,
|
|
1942
|
+
direction=direction, step=step, major_event=major_event,
|
|
1943
|
+
now=now_ts(self.now()))
|
|
1944
|
+
if result.get("allowed"):
|
|
1945
|
+
self._bump("identity_write_count")
|
|
1946
|
+
self._audit_write("identity_write", entry_id or "", "identity")
|
|
1947
|
+
return result
|
|
1948
|
+
|
|
1949
|
+
# -- indexes / snapshots ---------------------------------------------
|
|
1950
|
+
def build_index(self):
|
|
1951
|
+
"""Build the cue index (entities/place/time/event/id) for CLI checkup."""
|
|
1952
|
+
try:
|
|
1953
|
+
from .index.cue_index import build_index as _build
|
|
1954
|
+
return _build(self.store.entries)
|
|
1955
|
+
except Exception:
|
|
1956
|
+
return {key: {} for key in _EMPTY_INDEX}
|
|
1957
|
+
|
|
1958
|
+
def graph_links(self):
|
|
1959
|
+
"""Return {id: [related_ids]} when the graph flag is on."""
|
|
1960
|
+
if not self._flag("graph"):
|
|
1961
|
+
return {}
|
|
1962
|
+
try:
|
|
1963
|
+
from .index.graph import build_links
|
|
1964
|
+
return build_links(self.store.entries) or {}
|
|
1965
|
+
except Exception:
|
|
1966
|
+
return {}
|
|
1967
|
+
|
|
1968
|
+
def timeline(self):
|
|
1969
|
+
"""Read-only dual-temporal timeline: valid_at/invalid_at per entry.
|
|
1970
|
+
|
|
1971
|
+
Old entries written before dual-temporal support have no valid_at and
|
|
1972
|
+
are returned with null fields (never back-filled here).
|
|
1973
|
+
"""
|
|
1974
|
+
rows = []
|
|
1975
|
+
for entry in self.store.entries:
|
|
1976
|
+
if not isinstance(entry, dict):
|
|
1977
|
+
continue
|
|
1978
|
+
valid_at = entry.get("valid_at")
|
|
1979
|
+
invalid_at = entry.get("invalid_at")
|
|
1980
|
+
try:
|
|
1981
|
+
valid_at = float(valid_at) if valid_at is not None else None
|
|
1982
|
+
except Exception:
|
|
1983
|
+
valid_at = None
|
|
1984
|
+
try:
|
|
1985
|
+
invalid_at = float(invalid_at) if invalid_at is not None else None
|
|
1986
|
+
except Exception:
|
|
1987
|
+
invalid_at = None
|
|
1988
|
+
rows.append({"id": str(entry.get("id") or ""),
|
|
1989
|
+
"valid_at": valid_at,
|
|
1990
|
+
"invalid_at": invalid_at,
|
|
1991
|
+
"text": str(entry.get("text") or "")})
|
|
1992
|
+
rows.sort(key=lambda row: (row["valid_at"] is None,
|
|
1993
|
+
row["valid_at"] if row["valid_at"] is not None else 0.0,
|
|
1994
|
+
row["id"]))
|
|
1995
|
+
return rows
|
|
1996
|
+
|
|
1997
|
+
def slots_snapshot(self):
|
|
1998
|
+
"""Read-only attention-slot budget snapshot (empty when slots are off)."""
|
|
1999
|
+
total_budget = 360
|
|
2000
|
+
try:
|
|
2001
|
+
from .slots.registry import TOTAL_BUDGET_CHARS
|
|
2002
|
+
total_budget = int(TOTAL_BUDGET_CHARS)
|
|
2003
|
+
except Exception:
|
|
2004
|
+
pass
|
|
2005
|
+
registry = self.slot_registry if self._flag("slots") else None
|
|
2006
|
+
if registry is None:
|
|
2007
|
+
return {"enabled": False, "slots": [], "total_chars": 0,
|
|
2008
|
+
"total_budget": total_budget}
|
|
2009
|
+
slots = []
|
|
2010
|
+
for slot in getattr(registry, "slots", []) or []:
|
|
2011
|
+
items = []
|
|
2012
|
+
for item in getattr(slot, "items", []) or []:
|
|
2013
|
+
if isinstance(item, dict):
|
|
2014
|
+
items.append({"id": str(item.get("id") or ""),
|
|
2015
|
+
"text": str(item.get("text")
|
|
2016
|
+
or item.get("content") or "")[:120]})
|
|
2017
|
+
else:
|
|
2018
|
+
items.append({"id": "", "text": str(item)[:120]})
|
|
2019
|
+
chars = 0
|
|
2020
|
+
try:
|
|
2021
|
+
chars = int(slot.chars())
|
|
2022
|
+
except Exception:
|
|
2023
|
+
chars = sum(len(item["text"]) for item in items)
|
|
2024
|
+
slots.append({"id": str(getattr(slot, "slot_id", "")),
|
|
2025
|
+
"kind": str(getattr(slot, "kind", "")),
|
|
2026
|
+
"items": items,
|
|
2027
|
+
"chars": chars,
|
|
2028
|
+
"budget": int(getattr(slot, "budget_chars", 0) or 0)})
|
|
2029
|
+
try:
|
|
2030
|
+
total_chars = int(registry.total_chars() or 0)
|
|
2031
|
+
except Exception:
|
|
2032
|
+
total_chars = sum(slot["chars"] for slot in slots)
|
|
2033
|
+
return {"enabled": True, "slots": slots, "total_chars": total_chars,
|
|
2034
|
+
"total_budget": total_budget}
|
|
2035
|
+
|
|
2036
|
+
def dedupe_actions(self):
|
|
2037
|
+
"""Read-only write/dedupe action counters (from persisted store stats)."""
|
|
2038
|
+
counts = self.store.stats()
|
|
2039
|
+
def _count(key):
|
|
2040
|
+
try:
|
|
2041
|
+
return int(counts.get(key, 0) or 0)
|
|
2042
|
+
except Exception:
|
|
2043
|
+
return 0
|
|
2044
|
+
return {"add": _count("dedupe_add_count"),
|
|
2045
|
+
"merge": (_count("dedupe_merge_count")
|
|
2046
|
+
+ _count("dedupe_exact_merge_count")),
|
|
2047
|
+
"update": _count("dedupe_update_count"),
|
|
2048
|
+
"flag": _count("dedupe_flag_count"),
|
|
2049
|
+
"noop": _count("dedupe_noop_count")}
|
|
2050
|
+
|
|
2051
|
+
def graph_snapshot(self, limit=12):
|
|
2052
|
+
"""Read-only weighted link graph (edges + node count) for the UI."""
|
|
2053
|
+
try:
|
|
2054
|
+
limit = max(1, min(200, int(limit or 12)))
|
|
2055
|
+
except Exception:
|
|
2056
|
+
limit = 12
|
|
2057
|
+
entries = [entry for entry in self.store.entries
|
|
2058
|
+
if isinstance(entry, dict) and str(entry.get("id") or "")]
|
|
2059
|
+
by_id = {str(entry.get("id") or ""): entry for entry in entries}
|
|
2060
|
+
if not by_id:
|
|
2061
|
+
return {"edges": [], "nodes": 0, "entries": 0, "total_edges": 0}
|
|
2062
|
+
try:
|
|
2063
|
+
from .index import graph as graph_mod
|
|
2064
|
+
links = graph_mod.build_links(entries) or {}
|
|
2065
|
+
except Exception:
|
|
2066
|
+
links = {}
|
|
2067
|
+
seen = set()
|
|
2068
|
+
edges = []
|
|
2069
|
+
for source, targets in links.items():
|
|
2070
|
+
if source not in by_id:
|
|
2071
|
+
continue
|
|
2072
|
+
for target in targets or []:
|
|
2073
|
+
target = str(target or "")
|
|
2074
|
+
if target not in by_id or target == source:
|
|
2075
|
+
continue
|
|
2076
|
+
key = (source, target) if source <= target else (target, source)
|
|
2077
|
+
if key in seen:
|
|
2078
|
+
continue
|
|
2079
|
+
seen.add(key)
|
|
2080
|
+
weight = 0.0
|
|
2081
|
+
try:
|
|
2082
|
+
weight = float(graph_mod.pair_score(by_id[source],
|
|
2083
|
+
by_id[target]) or 0.0)
|
|
2084
|
+
except Exception:
|
|
2085
|
+
weight = 0.0
|
|
2086
|
+
edges.append({"from": source, "to": target,
|
|
2087
|
+
"weight": round(weight, 4)})
|
|
2088
|
+
edges.sort(key=lambda edge: (-edge["weight"], edge["from"], edge["to"]))
|
|
2089
|
+
nodes = set()
|
|
2090
|
+
for edge in edges:
|
|
2091
|
+
nodes.add(edge["from"])
|
|
2092
|
+
nodes.add(edge["to"])
|
|
2093
|
+
return {"edges": edges[:limit], "nodes": len(nodes),
|
|
2094
|
+
"entries": len(by_id), "total_edges": len(edges)}
|
|
2095
|
+
|
|
2096
|
+
def _local_embedding_status(self):
|
|
2097
|
+
"""Health string for the optional CPU embedding fallback."""
|
|
2098
|
+
try:
|
|
2099
|
+
from .recall.embedding import local_embedding_status
|
|
2100
|
+
return str(local_embedding_status(self.embedder) or "missing")
|
|
2101
|
+
except Exception:
|
|
2102
|
+
return "missing"
|
|
2103
|
+
|
|
2104
|
+
def health(self):
|
|
2105
|
+
"""Compact health snapshot; additive (main.py may expose it as /health).
|
|
2106
|
+
|
|
2107
|
+
local_embedding is one of ok | missing | failed:
|
|
2108
|
+
ok local int8 model loaded (fallback or local-only)
|
|
2109
|
+
missing no local model configured (pure remote by design)
|
|
2110
|
+
failed configured but failed to load (no retry until restart)
|
|
2111
|
+
"""
|
|
2112
|
+
try:
|
|
2113
|
+
from .recall.embedding import embedding_health
|
|
2114
|
+
embedding = embedding_health(self.embedder)
|
|
2115
|
+
except Exception:
|
|
2116
|
+
embedding = {"local_embedding": "missing"}
|
|
2117
|
+
return {
|
|
2118
|
+
"ok": True,
|
|
2119
|
+
"local_embedding": embedding.get("local_embedding", "missing"),
|
|
2120
|
+
"embedding": embedding,
|
|
2121
|
+
"entries": len(self.store.entries),
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
def _stats_snapshot(self, candidates=None):
|
|
2125
|
+
counts = self.store.stats()
|
|
2126
|
+
order = candidates if candidates is not None else self._eviction_rows()
|
|
2127
|
+
active = [t for t in self._threads.values()
|
|
2128
|
+
if isinstance(t, dict) and t.get("status") == "active"]
|
|
2129
|
+
return {
|
|
2130
|
+
"entries": len(self.store.entries),
|
|
2131
|
+
"local_embedding": self._local_embedding_status(),
|
|
2132
|
+
"pinned": sum(1 for x in self.store.entries
|
|
2133
|
+
if isinstance(x, dict) and x.get("pin")),
|
|
2134
|
+
"tombstones": len(self.store.tombstones),
|
|
2135
|
+
"by_module": self._by_module(),
|
|
2136
|
+
"recalls": counts.get("recall_count", 0),
|
|
2137
|
+
"dedupe_merges": counts.get("dedupe_merge_count", 0),
|
|
2138
|
+
"evict_candidates": len(order),
|
|
2139
|
+
"identity_writes": counts.get("identity_write_count", 0),
|
|
2140
|
+
"relay_cards": counts.get("relay_count", 0),
|
|
2141
|
+
"threads_active": len(active),
|
|
2142
|
+
"source_filtered": counts.get("source_filtered", 0),
|
|
2143
|
+
"tier_high": counts.get("tier_high", 0),
|
|
2144
|
+
"tier_mid": counts.get("tier_mid", 0),
|
|
2145
|
+
"tier_low": counts.get("tier_low", 0),
|
|
2146
|
+
"slots_admitted": counts.get("slots_admitted", 0),
|
|
2147
|
+
"slots_evicted": counts.get("slots_evicted", 0),
|
|
2148
|
+
"delete_propagation": counts.get("delete_propagation", 0),
|
|
2149
|
+
"security_rejected": counts.get("security_rejected", 0),
|
|
2150
|
+
"audit_writes": counts.get("audit_writes", 0),
|
|
2151
|
+
"rate_limited": counts.get("rate_limited", 0),
|
|
2152
|
+
"permissions_denied": counts.get("permissions_denied", 0),
|
|
2153
|
+
"compress_runs": counts.get("compress_runs", 0),
|
|
2154
|
+
"media_attached": counts.get("media_attached", 0),
|
|
2155
|
+
"notes_written": counts.get("notes_written", 0),
|
|
2156
|
+
"notes_linked": counts.get("notes_linked", 0),
|
|
2157
|
+
"entity_boost_hits": counts.get("entity_boost_hits", 0),
|
|
2158
|
+
"graph_edges": counts.get("graph_edges", 0),
|
|
2159
|
+
"ppr_expansions": counts.get("ppr_expansions", 0),
|
|
2160
|
+
"tiering_archived": counts.get("tiering_archived", 0),
|
|
2161
|
+
"retrieval_practice": counts.get("retrieval_practice", 0),
|
|
2162
|
+
"inhibition_applied": counts.get("inhibition_applied", 0),
|
|
2163
|
+
"boundary_switches": counts.get("boundary_switches", 0),
|
|
2164
|
+
"ledger_writes": counts.get("ledger_writes", 0),
|
|
2165
|
+
"consolidate_runs": counts.get("consolidate_runs", 0),
|
|
2166
|
+
"consolidate_writes": counts.get("consolidate_writes", 0),
|
|
2167
|
+
"evictions_applied": counts.get("evictions_applied", 0),
|
|
2168
|
+
"evictions_restored": counts.get("evictions_restored", 0),
|
|
2169
|
+
"evict_tombstones": counts.get("evict_tombstones", 0),
|
|
2170
|
+
"deletion_audits": counts.get("deletion_audits", 0),
|
|
2171
|
+
"consistency_conflicts": counts.get("consistency_conflicts", 0),
|
|
2172
|
+
"adaptive_scored": counts.get("adaptive_scored", 0),
|
|
2173
|
+
"source_consistency_ranked": counts.get("source_consistency_ranked", 0),
|
|
2174
|
+
"source_consistency_conflicts": counts.get("source_consistency_conflicts", 0),
|
|
2175
|
+
"narrative_writes": counts.get("narrative_writes", 0),
|
|
2176
|
+
"by_scope": self._by_scope(),
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2179
|
+
def _by_scope(self):
|
|
2180
|
+
out = {}
|
|
2181
|
+
for entry in self.store.entries:
|
|
2182
|
+
if not isinstance(entry, dict):
|
|
2183
|
+
continue
|
|
2184
|
+
key = str(entry.get("scope") or "default")
|
|
2185
|
+
out[key] = out.get(key, 0) + 1
|
|
2186
|
+
return out
|
|
2187
|
+
|
|
2188
|
+
def _by_module(self):
|
|
2189
|
+
out = {}
|
|
2190
|
+
for entry in self.store.entries:
|
|
2191
|
+
if not isinstance(entry, dict):
|
|
2192
|
+
continue
|
|
2193
|
+
key = str(entry.get("module") or entry.get("memory_source") or "unknown")
|
|
2194
|
+
out[key] = out.get(key, 0) + 1
|
|
2195
|
+
return out
|
|
2196
|
+
|
|
2197
|
+
def _eviction_rows(self):
|
|
2198
|
+
total = max(1, len(self.store.entries))
|
|
2199
|
+
rows = []
|
|
2200
|
+
for position, entry in enumerate(self.store.entries):
|
|
2201
|
+
if not isinstance(entry, dict):
|
|
2202
|
+
continue
|
|
2203
|
+
redundancy = 1.0 if any(
|
|
2204
|
+
other is not entry
|
|
2205
|
+
and str(other.get("text") or "") == str(entry.get("text") or "")
|
|
2206
|
+
for other in self.store.entries) else 0.0
|
|
2207
|
+
rows.append({"entry": entry,
|
|
2208
|
+
"score": eviction_score(entry, now=now_ts(self.now()),
|
|
2209
|
+
position=position / float(total),
|
|
2210
|
+
redundancy=redundancy),
|
|
2211
|
+
"id": str(entry.get("id") or "")})
|
|
2212
|
+
return eviction_order(rows)
|
|
2213
|
+
|
|
2214
|
+
def monitor(self):
|
|
2215
|
+
"""Read-only module health snapshot (no defaults/behaviour change)."""
|
|
2216
|
+
return self._stats_snapshot()
|
|
2217
|
+
|
|
2218
|
+
def eviction_snapshot(self, now=None):
|
|
2219
|
+
if not self.eviction_readonly:
|
|
2220
|
+
return {"enabled": False, "candidates": [], "deleted": 0}
|
|
2221
|
+
report = self.tick(now=now)
|
|
2222
|
+
report.update({"enabled": True, "readonly": True})
|
|
2223
|
+
return report
|
|
2224
|
+
|
|
2225
|
+
def memory_points_snapshot(self):
|
|
2226
|
+
"""Read-only memory-point projection (stored flag, else computed live).
|
|
2227
|
+
|
|
2228
|
+
Legacy entries written before the memory_point field existed are gated
|
|
2229
|
+
on the fly with is_memory_point(value or importance, importance,
|
|
2230
|
+
self_relevance) and marked projected=true. Nothing is written back to
|
|
2231
|
+
the store; this is projection only.
|
|
2232
|
+
"""
|
|
2233
|
+
rows = []
|
|
2234
|
+
for entry in self.store.entries:
|
|
2235
|
+
if not isinstance(entry, dict):
|
|
2236
|
+
continue
|
|
2237
|
+
flag = entry.get("memory_point")
|
|
2238
|
+
projected = False
|
|
2239
|
+
if isinstance(flag, bool):
|
|
2240
|
+
is_point = flag
|
|
2241
|
+
else:
|
|
2242
|
+
try:
|
|
2243
|
+
raw_value = float(entry.get("value") or 0.0)
|
|
2244
|
+
except Exception:
|
|
2245
|
+
raw_value = 0.0
|
|
2246
|
+
try:
|
|
2247
|
+
importance = float(entry.get("importance") or 0.0)
|
|
2248
|
+
except Exception:
|
|
2249
|
+
importance = 0.0
|
|
2250
|
+
try:
|
|
2251
|
+
self_relevance = float(entry.get("self_relevance") or 0.0)
|
|
2252
|
+
except Exception:
|
|
2253
|
+
self_relevance = 0.0
|
|
2254
|
+
is_point = bool(is_memory_point(raw_value or importance,
|
|
2255
|
+
importance, self_relevance))
|
|
2256
|
+
projected = True
|
|
2257
|
+
if not is_point:
|
|
2258
|
+
continue
|
|
2259
|
+
try:
|
|
2260
|
+
value = round(float(entry.get("value") or 0.0), 6)
|
|
2261
|
+
except Exception:
|
|
2262
|
+
value = 0.0
|
|
2263
|
+
if projected and value == 0.0:
|
|
2264
|
+
try:
|
|
2265
|
+
value = round(float(entry.get("importance") or 0.0), 6)
|
|
2266
|
+
except Exception:
|
|
2267
|
+
value = 0.0
|
|
2268
|
+
row = {"id": str(entry.get("id") or ""),
|
|
2269
|
+
"text": str(entry.get("text") or ""),
|
|
2270
|
+
"value": value,
|
|
2271
|
+
"importance": round(float(entry.get("importance") or 0.0), 6),
|
|
2272
|
+
"self_relevance": round(
|
|
2273
|
+
float(entry.get("self_relevance") or 0.0), 6),
|
|
2274
|
+
"ts": float(entry.get("ts") or 0.0),
|
|
2275
|
+
"scope": str(entry.get("scope") or "default")}
|
|
2276
|
+
if projected:
|
|
2277
|
+
row["projected"] = True
|
|
2278
|
+
rows.append(row)
|
|
2279
|
+
rows.sort(key=lambda row: (-row["value"], -row["ts"], row["id"]))
|
|
2280
|
+
return rows
|
|
2281
|
+
|
|
2282
|
+
def index_snapshot(self):
|
|
2283
|
+
"""Real cue-index projection: entities/place/time_coarse/event_name counts."""
|
|
2284
|
+
try:
|
|
2285
|
+
index = self.build_index()
|
|
2286
|
+
except Exception:
|
|
2287
|
+
index = {}
|
|
2288
|
+
projection = {}
|
|
2289
|
+
for key in ("entities", "place", "time_coarse", "event_name"):
|
|
2290
|
+
bucket = index.get(key) if isinstance(index, dict) else None
|
|
2291
|
+
try:
|
|
2292
|
+
projection[key] = len(bucket) if isinstance(bucket, dict) else 0
|
|
2293
|
+
except Exception:
|
|
2294
|
+
projection[key] = 0
|
|
2295
|
+
return projection
|
|
2296
|
+
|
|
2297
|
+
def loops_snapshot(self):
|
|
2298
|
+
"""Read-only projection of the seven memory big loops A-G for the UI.
|
|
2299
|
+
|
|
2300
|
+
Each loop is {id, name, brain, modules: [flags], active, items: counts,
|
|
2301
|
+
small_loops: [{from, to, name, active}]}. active is True when any small
|
|
2302
|
+
loop is active; built-in capability nodes (store/index/recall/scores)
|
|
2303
|
+
are always active, optional mechanisms follow their engine flag.
|
|
2304
|
+
items are counters/totals projected from the existing store, flags,
|
|
2305
|
+
slots, graph and narrative state. Pure read: no algorithm, no writes.
|
|
2306
|
+
"""
|
|
2307
|
+
try:
|
|
2308
|
+
flags = dict(self.flags or {})
|
|
2309
|
+
entries = [e for e in self.store.entries if isinstance(e, dict)]
|
|
2310
|
+
stats = self.store.stats() or {}
|
|
2311
|
+
tombstones = len(self.store.tombstones)
|
|
2312
|
+
slot_info = self.slots_snapshot()
|
|
2313
|
+
slots = [s for s in (slot_info.get("slots") or [])
|
|
2314
|
+
if isinstance(s, dict)]
|
|
2315
|
+
index = self.build_index()
|
|
2316
|
+
identity_rows = [e for e in entries if is_identity(e)]
|
|
2317
|
+
narrative_version = 0
|
|
2318
|
+
notes = 0
|
|
2319
|
+
summaries = 0
|
|
2320
|
+
for row in entries:
|
|
2321
|
+
try:
|
|
2322
|
+
narrative_version = max(
|
|
2323
|
+
narrative_version,
|
|
2324
|
+
int(row.get("identity_narrative_version") or 0))
|
|
2325
|
+
except Exception:
|
|
2326
|
+
pass
|
|
2327
|
+
meta = row.get("metadata")
|
|
2328
|
+
if isinstance(meta, dict) and meta.get("note"):
|
|
2329
|
+
notes += 1
|
|
2330
|
+
if (str(row.get("role") or "") == "system"
|
|
2331
|
+
or str(row.get("source") or "") in _SUMMARY_SOURCES
|
|
2332
|
+
or row.get("consolidated")):
|
|
2333
|
+
summaries += 1
|
|
2334
|
+
threads_active = sum(
|
|
2335
|
+
1 for card in self._threads.values()
|
|
2336
|
+
if isinstance(card, dict) and card.get("status") == "active")
|
|
2337
|
+
slot_items = sum(len(s.get("items") or []) for s in slots)
|
|
2338
|
+
|
|
2339
|
+
def count(key):
|
|
2340
|
+
try:
|
|
2341
|
+
return int(stats.get(key, 0) or 0)
|
|
2342
|
+
except Exception:
|
|
2343
|
+
return 0
|
|
2344
|
+
|
|
2345
|
+
def small(spec):
|
|
2346
|
+
out = []
|
|
2347
|
+
for name, source, target, base in spec:
|
|
2348
|
+
out.append({"from": source, "to": target, "name": name,
|
|
2349
|
+
"active": bool(flags.get(name, False))
|
|
2350
|
+
if base is None else bool(base)})
|
|
2351
|
+
return out
|
|
2352
|
+
|
|
2353
|
+
specs = (
|
|
2354
|
+
("A", "编码-存储", "海马 ↔ 新皮层", (
|
|
2355
|
+
("store", "remember", "entries", True),
|
|
2356
|
+
("index", "text", "cue_index", True),
|
|
2357
|
+
("dedupe", "remember", "store", None),
|
|
2358
|
+
("provenance", "remember", "source_tag", None),
|
|
2359
|
+
("notes", "remember", "metadata.note", None),
|
|
2360
|
+
("notes_link", "remember", "links", None),
|
|
2361
|
+
("media", "remember", "metadata.media", None),
|
|
2362
|
+
("consistency", "memory", "conflict_flag", None),
|
|
2363
|
+
), {"entries": len(entries),
|
|
2364
|
+
"pinned": sum(1 for e in entries if e.get("pin")),
|
|
2365
|
+
"notes": notes,
|
|
2366
|
+
"dedupe_merges": count("dedupe_merge_count"),
|
|
2367
|
+
"indexed": len(index.get("id") or {})}),
|
|
2368
|
+
("B", "提取-强化", "前额叶 ↔ 海马", (
|
|
2369
|
+
("recall", "query", "ranked", True),
|
|
2370
|
+
("embedding", "text", "semantic", None),
|
|
2371
|
+
("three_channel", "candidates", "fusion", None),
|
|
2372
|
+
("bm25", "query", "lexical", None),
|
|
2373
|
+
("bm25_multi", "query", "lexical_multi", None),
|
|
2374
|
+
("ppr", "seeds", "graph_expand", None),
|
|
2375
|
+
("graph", "entries", "edges", None),
|
|
2376
|
+
("mmr", "ranked", "diverse", None),
|
|
2377
|
+
("complexity", "query", "budget", None),
|
|
2378
|
+
("constraints", "query", "hint", None),
|
|
2379
|
+
("tiering", "ranked", "tier", None),
|
|
2380
|
+
("entity_boost", "entities", "score", None),
|
|
2381
|
+
("inhibition", "losers", "suppressed", None),
|
|
2382
|
+
("retrieval_practice", "hits", "stability", None),
|
|
2383
|
+
("source_consistency", "candidates", "source_rank", None),
|
|
2384
|
+
), {"recalls": count("recall_count"),
|
|
2385
|
+
"tier_high": count("tier_high"),
|
|
2386
|
+
"tier_mid": count("tier_mid"),
|
|
2387
|
+
"tier_low": count("tier_low"),
|
|
2388
|
+
"graph_edges": count("graph_edges"),
|
|
2389
|
+
"ppr_expansions": count("ppr_expansions"),
|
|
2390
|
+
"retrieval_practice": count("retrieval_practice"),
|
|
2391
|
+
"inhibition_applied": count("inhibition_applied")}),
|
|
2392
|
+
("C", "巩固-重构", "海马-皮层、睡眠", (
|
|
2393
|
+
("consolidate_gated", "consolidate", "gate", None),
|
|
2394
|
+
("consolidate_tree", "events", "summary_tree", None),
|
|
2395
|
+
("consolidate_reflection", "events", "reflection", None),
|
|
2396
|
+
("consolidate_neighbors", "entry", "links", None),
|
|
2397
|
+
("consolidate_writeback", "summary", "store", None),
|
|
2398
|
+
("compress", "output", "summary", None),
|
|
2399
|
+
("narrative", "C", "F", None),
|
|
2400
|
+
), {"consolidate_runs": count("consolidate_runs"),
|
|
2401
|
+
"consolidate_writes": count("consolidate_writes"),
|
|
2402
|
+
"summaries": summaries,
|
|
2403
|
+
"narrative_writes": count("narrative_writes"),
|
|
2404
|
+
"narrative_version": narrative_version}),
|
|
2405
|
+
("D", "遗忘-抑制", "前扣带 ↔ 基底节", (
|
|
2406
|
+
("eviction_score", "tick", "scores", True),
|
|
2407
|
+
("eviction_readonly", "tick", "report", None),
|
|
2408
|
+
("heat", "entry", "heat", None),
|
|
2409
|
+
("consequence", "entry", "consequence", None),
|
|
2410
|
+
("adaptive_evict", "tick", "score", None),
|
|
2411
|
+
("tiering_exec", "tick", "archived", None),
|
|
2412
|
+
("apply_evictions", "tick", "soft_delete", None),
|
|
2413
|
+
("deletion_audit", "forget", "audit", None),
|
|
2414
|
+
), {"entries": len(entries),
|
|
2415
|
+
"tombstones": tombstones,
|
|
2416
|
+
"evictions_applied": count("evictions_applied"),
|
|
2417
|
+
"evictions_restored": count("evictions_restored"),
|
|
2418
|
+
"evict_tombstones": count("evict_tombstones"),
|
|
2419
|
+
"adaptive_scored": count("adaptive_scored")}),
|
|
2420
|
+
("E", "注意-选择", "丘脑 ↔ 顶叶", (
|
|
2421
|
+
("slots", "context", "registry", None),
|
|
2422
|
+
("boundary", "messages", "switch", None),
|
|
2423
|
+
("due_hints", "loops", "hint", None),
|
|
2424
|
+
), {"slots": len(slots),
|
|
2425
|
+
"slot_items": slot_items,
|
|
2426
|
+
"slot_chars": int(slot_info.get("total_chars") or 0),
|
|
2427
|
+
"slot_budget": int(slot_info.get("total_budget") or 0),
|
|
2428
|
+
"boundary_switches": count("boundary_switches")}),
|
|
2429
|
+
("F", "社会-自我", "默认网络 / 内侧前额叶", (
|
|
2430
|
+
("identity_read", "store", "identity", True),
|
|
2431
|
+
("identity", "entries", "snapshot", None),
|
|
2432
|
+
("identity_writeback", "evidence", "identity", None),
|
|
2433
|
+
("relay", "identity", "card", None),
|
|
2434
|
+
("thread", "turns", "thread_card", None),
|
|
2435
|
+
("ledger", "relation", "ledger", None),
|
|
2436
|
+
("narrative", "consolidate", "identity_narrative", None),
|
|
2437
|
+
), {"identity_entries": len(identity_rows),
|
|
2438
|
+
"relay_cards": count("relay_count"),
|
|
2439
|
+
"threads_active": threads_active,
|
|
2440
|
+
"ledger_writes": count("ledger_writes"),
|
|
2441
|
+
"narrative_writes": count("narrative_writes"),
|
|
2442
|
+
"narrative_version": narrative_version}),
|
|
2443
|
+
("G", "安全-监控", "岛叶 / 前扣带", (
|
|
2444
|
+
("tombstone", "delete", "tombstone", True),
|
|
2445
|
+
("security", "write", "reject", None),
|
|
2446
|
+
("source_filter", "recall", "filter", None),
|
|
2447
|
+
("temporal_filter", "recall", "validity", None),
|
|
2448
|
+
("audit", "write_delete", "audit_jsonl", None),
|
|
2449
|
+
("rate_limit", "scope", "window", None),
|
|
2450
|
+
("permissions", "scope", "gate", None),
|
|
2451
|
+
("deletion_audit", "forget", "audit", None),
|
|
2452
|
+
), {"security_rejected": count("security_rejected"),
|
|
2453
|
+
"source_filtered": count("source_filtered"),
|
|
2454
|
+
"audit_writes": count("audit_writes"),
|
|
2455
|
+
"rate_limited": count("rate_limited"),
|
|
2456
|
+
"permissions_denied": count("permissions_denied"),
|
|
2457
|
+
"deletion_audits": count("deletion_audits"),
|
|
2458
|
+
"tombstones": tombstones}),
|
|
2459
|
+
)
|
|
2460
|
+
loops = []
|
|
2461
|
+
for loop_id, name, brain, spec, items in specs:
|
|
2462
|
+
small_loops = small(spec)
|
|
2463
|
+
modules = []
|
|
2464
|
+
for row in spec:
|
|
2465
|
+
flag = row[0]
|
|
2466
|
+
if flag in FLAGS and flag not in modules:
|
|
2467
|
+
modules.append(flag)
|
|
2468
|
+
loops.append({
|
|
2469
|
+
"id": loop_id, "name": name, "brain": brain,
|
|
2470
|
+
"modules": modules,
|
|
2471
|
+
"active": any(item["active"] for item in small_loops),
|
|
2472
|
+
"items": dict(items),
|
|
2473
|
+
"small_loops": small_loops})
|
|
2474
|
+
return loops
|
|
2475
|
+
except Exception:
|
|
2476
|
+
return []
|