dsh-biocore 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/py/bridge.py +15 -7
- package/lib/py-bridge.js +4 -3
- package/package.json +3 -2
- package/python/biocore_memory/__init__.py +21 -0
- package/python/biocore_memory/adapters/__init__.py +2 -0
- package/python/biocore_memory/adapters/astrbot.py +24 -0
- package/python/biocore_memory/adapters/dsh_cli.py +335 -0
- package/python/biocore_memory/co_recall.py +324 -0
- package/python/biocore_memory/consolidate/__init__.py +11 -0
- package/python/biocore_memory/consolidate/gated.py +104 -0
- package/python/biocore_memory/consolidate/neighbors.py +68 -0
- package/python/biocore_memory/consolidate/reflection.py +45 -0
- package/python/biocore_memory/consolidate/tree.py +95 -0
- package/python/biocore_memory/consolidate/writeback.py +181 -0
- package/python/biocore_memory/core/__init__.py +4 -0
- package/python/biocore_memory/core/config.py +127 -0
- package/python/biocore_memory/core/permissions.py +69 -0
- package/python/biocore_memory/core/types.py +116 -0
- package/python/biocore_memory/core/util.py +26 -0
- package/python/biocore_memory/embedding_local.py +447 -0
- package/python/biocore_memory/embedding_mode.py +156 -0
- package/python/biocore_memory/engine.py +2476 -0
- package/python/biocore_memory/engine_helpers.py +266 -0
- package/python/biocore_memory/event/__init__.py +3 -0
- package/python/biocore_memory/event/_types.py +16 -0
- package/python/biocore_memory/event/consistency.py +117 -0
- package/python/biocore_memory/event/extract.py +48 -0
- package/python/biocore_memory/event/media.py +111 -0
- package/python/biocore_memory/event/notes.py +85 -0
- package/python/biocore_memory/event/notes_link.py +87 -0
- package/python/biocore_memory/event/vector.py +69 -0
- package/python/biocore_memory/evict/__init__.py +3 -0
- package/python/biocore_memory/evict/adaptive.py +73 -0
- package/python/biocore_memory/evict/consequence.py +58 -0
- package/python/biocore_memory/evict/heat.py +54 -0
- package/python/biocore_memory/evict/inhibition.py +105 -0
- package/python/biocore_memory/evict/policy.py +188 -0
- package/python/biocore_memory/evict/score.py +23 -0
- package/python/biocore_memory/identity/__init__.py +6 -0
- package/python/biocore_memory/identity/api.py +99 -0
- package/python/biocore_memory/identity/history.py +9 -0
- package/python/biocore_memory/identity/ledger.py +191 -0
- package/python/biocore_memory/identity/narrative.py +358 -0
- package/python/biocore_memory/identity/snapshot.py +13 -0
- package/python/biocore_memory/identity/writeback.py +61 -0
- package/python/biocore_memory/index/__init__.py +3 -0
- package/python/biocore_memory/index/cue_index.py +80 -0
- package/python/biocore_memory/index/entities.py +85 -0
- package/python/biocore_memory/index/entity_boost.py +104 -0
- package/python/biocore_memory/index/graph.py +172 -0
- package/python/biocore_memory/recall/__init__.py +5 -0
- package/python/biocore_memory/recall/api.py +65 -0
- package/python/biocore_memory/recall/bm25.py +118 -0
- package/python/biocore_memory/recall/channels.py +98 -0
- package/python/biocore_memory/recall/complexity.py +62 -0
- package/python/biocore_memory/recall/constraints.py +75 -0
- package/python/biocore_memory/recall/embedding.py +745 -0
- package/python/biocore_memory/recall/expand.py +26 -0
- package/python/biocore_memory/recall/filters.py +40 -0
- package/python/biocore_memory/recall/fusion.py +18 -0
- package/python/biocore_memory/recall/mmr.py +70 -0
- package/python/biocore_memory/recall/ppr.py +85 -0
- package/python/biocore_memory/recall/retrieval_practice.py +102 -0
- package/python/biocore_memory/relay/__init__.py +2 -0
- package/python/biocore_memory/relay/card.py +51 -0
- package/python/biocore_memory/slots/__init__.py +18 -0
- package/python/biocore_memory/slots/admission.py +64 -0
- package/python/biocore_memory/slots/boundary.py +128 -0
- package/python/biocore_memory/slots/evict_bridge.py +106 -0
- package/python/biocore_memory/slots/registry.py +135 -0
- package/python/biocore_memory/slots/switch.py +42 -0
- package/python/biocore_memory/store/__init__.py +4 -0
- package/python/biocore_memory/store/audit.py +74 -0
- package/python/biocore_memory/store/compress.py +81 -0
- package/python/biocore_memory/store/dedupe.py +178 -0
- package/python/biocore_memory/store/json_store.py +154 -0
- package/python/biocore_memory/store/media.py +145 -0
- package/python/biocore_memory/store/provenance.py +73 -0
- package/python/biocore_memory/store/rate_limit.py +93 -0
- package/python/biocore_memory/store/security.py +107 -0
- package/python/biocore_memory/store/source_consistency.py +123 -0
- package/python/biocore_memory/store/tiering.py +61 -0
- package/python/biocore_memory/store/tiering_exec.py +50 -0
- package/python/biocore_memory/store/tombstone.py +23 -0
- package/python/biocore_memory/store/versions.py +69 -0
- package/python/biocore_memory/thread/__init__.py +2 -0
- package/python/biocore_memory/thread/card.py +58 -0
- package/python/biocore_model/__init__.py +37 -0
- package/python/biocore_model/affective_labor.py +162 -0
- package/python/biocore_model/appraisal.py +257 -0
- package/python/biocore_model/appraisal_action.py +78 -0
- package/python/biocore_model/archive_queue.py +60 -0
- package/python/biocore_model/attachment_behavior.py +130 -0
- package/python/biocore_model/attachment_regulation.py +75 -0
- package/python/biocore_model/attachment_state.py +41 -0
- package/python/biocore_model/attribution.py +73 -0
- package/python/biocore_model/attributor.py +162 -0
- package/python/biocore_model/autonomic.py +91 -0
- package/python/biocore_model/autonomy_affect.py +51 -0
- package/python/biocore_model/autonomy_check.py +67 -0
- package/python/biocore_model/autonomy_loop.py +122 -0
- package/python/biocore_model/autonomy_mode.py +114 -0
- package/python/biocore_model/autonomy_model.py +733 -0
- package/python/biocore_model/autonomy_params.py +131 -0
- package/python/biocore_model/autonomy_reason.py +73 -0
- package/python/biocore_model/autonomy_tick.py +227 -0
- package/python/biocore_model/bio_appraisal.py +49 -0
- package/python/biocore_model/bio_autonomy_state.py +171 -0
- package/python/biocore_model/bio_config.py +221 -0
- package/python/biocore_model/bio_const.py +128 -0
- package/python/biocore_model/bio_daily.py +267 -0
- package/python/biocore_model/bio_emotion.py +538 -0
- package/python/biocore_model/bio_emotion2.py +214 -0
- package/python/biocore_model/bio_goal.py +220 -0
- package/python/biocore_model/bio_identity.py +82 -0
- package/python/biocore_model/bio_noise.py +119 -0
- package/python/biocore_model/bio_physiology2.py +114 -0
- package/python/biocore_model/bio_snapshot.py +324 -0
- package/python/biocore_model/bio_state.py +272 -0
- package/python/biocore_model/bio_tick.py +310 -0
- package/python/biocore_model/bio_util.py +82 -0
- package/python/biocore_model/blocks.py +264 -0
- package/python/biocore_model/boundary.py +251 -0
- package/python/biocore_model/boundary_gate.py +188 -0
- package/python/biocore_model/commitment_extractor.py +517 -0
- package/python/biocore_model/compassion_fatigue.py +84 -0
- package/python/biocore_model/connection_bids.py +120 -0
- package/python/biocore_model/core_bridge.py +177 -0
- package/python/biocore_model/core_client.py +127 -0
- package/python/biocore_model/core_commands.py +112 -0
- package/python/biocore_model/core_mode.py +104 -0
- package/python/biocore_model/core_mode_ui.mjs +104 -0
- package/python/biocore_model/criticality.py +81 -0
- package/python/biocore_model/defense_mechanisms.py +272 -0
- package/python/biocore_model/delta_budget.py +99 -0
- package/python/biocore_model/diag_trace.py +274 -0
- package/python/biocore_model/dissonance.py +134 -0
- package/python/biocore_model/emotion_geometry.py +108 -0
- package/python/biocore_model/emotion_probe.py +209 -0
- package/python/biocore_model/entity_graph.py +931 -0
- package/python/biocore_model/entity_resolve.py +260 -0
- package/python/biocore_model/envy.py +193 -0
- package/python/biocore_model/exit_or_silence.py +230 -0
- package/python/biocore_model/expr_guard.py +24 -0
- package/python/biocore_model/expr_intent.py +77 -0
- package/python/biocore_model/expr_state.py +204 -0
- package/python/biocore_model/expr_template.py +105 -0
- package/python/biocore_model/expr_tone.py +81 -0
- package/python/biocore_model/extreme_scenarios.py +205 -0
- package/python/biocore_model/feedback_flow.py +227 -0
- package/python/biocore_model/feedback_guard.py +129 -0
- package/python/biocore_model/feedback_llm.py +163 -0
- package/python/biocore_model/feedback_memory.py +96 -0
- package/python/biocore_model/feedback_user.py +93 -0
- package/python/biocore_model/frustration_curve.py +110 -0
- package/python/biocore_model/granularity.py +283 -0
- package/python/biocore_model/graph_extract.py +248 -0
- package/python/biocore_model/graph_recall.py +221 -0
- package/python/biocore_model/intent_render.py +91 -0
- package/python/biocore_model/intero_core.py +235 -0
- package/python/biocore_model/intero_style.py +116 -0
- package/python/biocore_model/layered_memory.py +297 -0
- package/python/biocore_model/learning_probe.py +120 -0
- package/python/biocore_model/memory_dual.py +224 -0
- package/python/biocore_model/memory_forgetting.py +125 -0
- package/python/biocore_model/memory_store.py +1399 -0
- package/python/biocore_model/metacog.py +153 -0
- package/python/biocore_model/migrate.py +733 -0
- package/python/biocore_model/migrate_store.py +337 -0
- package/python/biocore_model/mind_wander.py +111 -0
- package/python/biocore_model/mood.py +48 -0
- package/python/biocore_model/narrative.py +172 -0
- package/python/biocore_model/needs_esteem.py +116 -0
- package/python/biocore_model/needs_events.py +137 -0
- package/python/biocore_model/needs_sleep.py +72 -0
- package/python/biocore_model/noise_bank.py +95 -0
- package/python/biocore_model/persona_flow.mjs +25 -0
- package/python/biocore_model/pipeline.py +240 -0
- package/python/biocore_model/pipeline_api.py +200 -0
- package/python/biocore_model/plugin_config.py +83 -0
- package/python/biocore_model/plugin_routes.py +1435 -0
- package/python/biocore_model/policy_learning.py +132 -0
- package/python/biocore_model/possible_selves.py +110 -0
- package/python/biocore_model/predictor.py +178 -0
- package/python/biocore_model/preset.py +205 -0
- package/python/biocore_model/proactive_context.py +153 -0
- package/python/biocore_model/prompt_rules.py +90 -0
- package/python/biocore_model/prospective.py +183 -0
- package/python/biocore_model/provider_fallback.py +97 -0
- package/python/biocore_model/reconstruction.py +132 -0
- package/python/biocore_model/regulation_loop.py +179 -0
- package/python/biocore_model/regulation_strategies.py +294 -0
- package/python/biocore_model/relation_stage.py +130 -0
- package/python/biocore_model/relationship_stage.py +95 -0
- package/python/biocore_model/render.py +901 -0
- package/python/biocore_model/render_guard.mjs +34 -0
- package/python/biocore_model/repair_after_boundary.py +350 -0
- package/python/biocore_model/reply_limit.py +183 -0
- package/python/biocore_model/reply_split.py +203 -0
- package/python/biocore_model/rollback.py +143 -0
- package/python/biocore_model/runtime.py +479 -0
- package/python/biocore_model/sanitize.py +200 -0
- package/python/biocore_model/scheduler.py +123 -0
- package/python/biocore_model/schema.py +120 -0
- package/python/biocore_model/secrecy.py +145 -0
- package/python/biocore_model/self_esteem.py +50 -0
- package/python/biocore_model/self_intent.py +437 -0
- package/python/biocore_model/self_interest.py +263 -0
- package/python/biocore_model/self_verification.py +178 -0
- package/python/biocore_model/services.py +78 -0
- package/python/biocore_model/situation.py +254 -0
- package/python/biocore_model/sleep_circadian.py +116 -0
- package/python/biocore_model/social_pain.py +69 -0
- package/python/biocore_model/social_repair.py +111 -0
- package/python/biocore_model/social_theory.py +131 -0
- package/python/biocore_model/source_monitoring.py +292 -0
- package/python/biocore_model/stages.py +487 -0
- package/python/biocore_model/state_tag.py +231 -0
- package/python/biocore_model/switch_groups.mjs +201 -0
- package/python/biocore_model/tabs_state.mjs +37 -0
- package/python/biocore_model/template_expr.py +252 -0
- package/python/biocore_model/tools/caps_check.mjs +23 -0
- package/python/biocore_model/tools/init_symbols_check.mjs +11 -0
- package/python/biocore_model/tools/missing_ids.mjs +16 -0
- package/python/biocore_model/tools/scope_check.mjs +19 -0
- package/python/biocore_model/topic_control.py +268 -0
- package/python/biocore_model/turn_memory.py +152 -0
- package/python/biocore_model/ui_compat.mjs +48 -0
- package/python/biocore_model/ui_settings.py +266 -0
- package/python/biocore_model/ui_settings.py.bak_authority +108 -0
- package/python/biocore_model/value_order.py +186 -0
- package/python/biocore_model/view_mode.mjs +8 -0
- package/python/biocore_model/voice_bank.py +177 -0
- package/python/biocore_model/waiting_sense.py +229 -0
- package/python/biocore_model/x_directives.py +142 -0
- package/python/biocore_runtime/__init__.py +7 -0
- package/python/biocore_runtime/pipeline.py +1915 -0
- package/python/dsh_adapter/README.md +96 -0
- package/python/dsh_adapter/adapter.py +1077 -0
- package/python/dsh_adapter/cli.py +107 -0
- package/python/dsh_adapter/data/demo_state.json +164 -0
- package/python/dsh_adapter/data/verify_state.json +164 -0
- package/python/dsh_adapter/tests/__init__.py +1 -0
- package/python/dsh_adapter/tests/test_adapter.py +494 -0
- package/ui/panel-layout.js +3 -3
- package/ui/panel.css +53 -6
- package/ui/panel.html +33 -32
- package/ui/panel.js +9 -1
- package/lib/py/__pycache__/bridge.cpython-310.pyc +0 -0
|
@@ -0,0 +1,733 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""1.3.4 -> 1.4.0 data migration (settings / accumulator / old memory).
|
|
3
|
+
|
|
4
|
+
Design: one pure-ish entry point migrate(plugin) -> report dict. It only reads
|
|
5
|
+
and writes JSON files; it never imports main. Idempotent: ui_settings gets a
|
|
6
|
+
top-level schema_version; field backfills are also safe to repeat (missing-only).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import ui_settings
|
|
16
|
+
|
|
17
|
+
try: # AstrBot requires the host logger
|
|
18
|
+
from astrbot.api import logger
|
|
19
|
+
except Exception: # pragma: no cover - offline tests
|
|
20
|
+
class _NullLogger: # offline fallback: never imports stdlib logging
|
|
21
|
+
def _noop(self, *args, **kwargs):
|
|
22
|
+
return None
|
|
23
|
+
debug = info = warning = error = exception = critical = _noop
|
|
24
|
+
logger = _NullLogger()
|
|
25
|
+
|
|
26
|
+
SCHEMA_VERSION = 4
|
|
27
|
+
OLD_SCHEMA_VERSION = 1
|
|
28
|
+
MEMORY_STORAGE_S = 0.45
|
|
29
|
+
MEMORY_AROUSAL = 0.4
|
|
30
|
+
ACC_DEFAULTS = {"theta": 0.55, "cooldown_seconds": 1800,
|
|
31
|
+
"daily_cap": 5, "last_umo": ""}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _read(path):
|
|
35
|
+
try:
|
|
36
|
+
if path and path.is_file():
|
|
37
|
+
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
38
|
+
if isinstance(data, dict):
|
|
39
|
+
return data
|
|
40
|
+
except Exception:
|
|
41
|
+
pass
|
|
42
|
+
return {}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _write(path, data):
|
|
46
|
+
path = Path(path)
|
|
47
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
if path.is_file():
|
|
49
|
+
try:
|
|
50
|
+
ui_settings._backup(path)
|
|
51
|
+
except Exception:
|
|
52
|
+
pass
|
|
53
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
54
|
+
try:
|
|
55
|
+
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2),
|
|
56
|
+
encoding="utf-8")
|
|
57
|
+
os.replace(tmp, path)
|
|
58
|
+
except Exception:
|
|
59
|
+
try:
|
|
60
|
+
tmp.unlink(missing_ok=True)
|
|
61
|
+
except Exception:
|
|
62
|
+
pass
|
|
63
|
+
raise
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _data_dir(plugin):
|
|
67
|
+
for attr in ("data_dir", "DATA_DIR"):
|
|
68
|
+
raw = getattr(plugin, attr, None) if plugin is not None else None
|
|
69
|
+
if raw:
|
|
70
|
+
return Path(str(raw))
|
|
71
|
+
return ui_settings.authority_dir(plugin)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _legacy_settings(plugin):
|
|
75
|
+
try:
|
|
76
|
+
for path in ui_settings._legacy_candidates(plugin):
|
|
77
|
+
if path and path.is_file():
|
|
78
|
+
return Path(path)
|
|
79
|
+
except Exception:
|
|
80
|
+
pass
|
|
81
|
+
data = _data_dir(plugin)
|
|
82
|
+
if ui_settings._is_plugins_data(data):
|
|
83
|
+
candidate = data / ui_settings.FILE_NAME
|
|
84
|
+
if candidate.is_file():
|
|
85
|
+
return candidate
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _authority_settings(plugin):
|
|
90
|
+
return ui_settings.authority_dir(plugin) / ui_settings.FILE_NAME
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def schema_version(plugin=None):
|
|
94
|
+
try:
|
|
95
|
+
return int(_read(_authority_settings(plugin)).get("schema_version", 0) or 0)
|
|
96
|
+
except Exception:
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _atomic_copy(src, dst):
|
|
101
|
+
dst = Path(dst)
|
|
102
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
103
|
+
tmp = dst.with_name(dst.name + ".migrate-tmp")
|
|
104
|
+
try:
|
|
105
|
+
tmp.write_bytes(Path(src).read_bytes())
|
|
106
|
+
os.replace(tmp, dst)
|
|
107
|
+
except Exception:
|
|
108
|
+
try:
|
|
109
|
+
tmp.unlink(missing_ok=True)
|
|
110
|
+
except Exception:
|
|
111
|
+
pass
|
|
112
|
+
raise
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _create_backup(plugin):
|
|
116
|
+
"""Copy settings / memory docs / states before migration (auto, internal)."""
|
|
117
|
+
base = ui_settings.authority_dir(plugin)
|
|
118
|
+
stamp = time.strftime("%Y%m%d-%H%M%S", time.localtime())
|
|
119
|
+
root = base / "backups" / ("pre-migration-" + stamp)
|
|
120
|
+
copied = {"settings": 0, "memory": 0, "states": 0, "graph": 0}
|
|
121
|
+
for name in ("settings", "memory", "states", "graph"):
|
|
122
|
+
(root / name).mkdir(parents=True, exist_ok=True)
|
|
123
|
+
settings_candidates = []
|
|
124
|
+
try:
|
|
125
|
+
settings_candidates.extend(ui_settings._legacy_candidates(plugin))
|
|
126
|
+
except Exception:
|
|
127
|
+
pass
|
|
128
|
+
authority = _authority_settings(plugin)
|
|
129
|
+
if authority.is_file():
|
|
130
|
+
settings_candidates.append(authority)
|
|
131
|
+
seen = set()
|
|
132
|
+
for src in settings_candidates:
|
|
133
|
+
src = Path(src)
|
|
134
|
+
if not src.is_file() or str(src) in seen:
|
|
135
|
+
continue
|
|
136
|
+
seen.add(str(src))
|
|
137
|
+
target = root / "settings" / (src.name if len(seen) == 1
|
|
138
|
+
else "legacy_" + src.name)
|
|
139
|
+
try:
|
|
140
|
+
_atomic_copy(src, target)
|
|
141
|
+
copied["settings"] += 1
|
|
142
|
+
except Exception:
|
|
143
|
+
continue
|
|
144
|
+
memory = _memory_dir(plugin)
|
|
145
|
+
if memory is not None:
|
|
146
|
+
# task-146 rule 3: back up the whole memory/ directory (JSON docs and
|
|
147
|
+
# the SQLite db incl. WAL) before any migration write.
|
|
148
|
+
try:
|
|
149
|
+
_db = memory / "memory.sqlite3"
|
|
150
|
+
if _db.is_file():
|
|
151
|
+
try:
|
|
152
|
+
import sqlite3 as _sqlite3
|
|
153
|
+
_conn = _sqlite3.connect(str(_db), timeout=5.0)
|
|
154
|
+
try:
|
|
155
|
+
_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
156
|
+
finally:
|
|
157
|
+
_conn.close()
|
|
158
|
+
except Exception:
|
|
159
|
+
pass
|
|
160
|
+
except Exception:
|
|
161
|
+
pass
|
|
162
|
+
for src in sorted(memory.iterdir()):
|
|
163
|
+
if not src.is_file():
|
|
164
|
+
continue
|
|
165
|
+
if src.name.endswith(".corrupt.bak"):
|
|
166
|
+
continue
|
|
167
|
+
try:
|
|
168
|
+
_atomic_copy(src, root / "memory" / src.name)
|
|
169
|
+
copied["memory"] += 1
|
|
170
|
+
except Exception:
|
|
171
|
+
continue
|
|
172
|
+
states = _data_dir(plugin) / "states"
|
|
173
|
+
if not states.is_dir():
|
|
174
|
+
states = base / "states"
|
|
175
|
+
if states.is_dir():
|
|
176
|
+
for src in states.glob("*.json"):
|
|
177
|
+
try:
|
|
178
|
+
_atomic_copy(src, root / "states" / src.name)
|
|
179
|
+
copied["states"] += 1
|
|
180
|
+
except Exception:
|
|
181
|
+
continue
|
|
182
|
+
graph = _graph_dir(plugin)
|
|
183
|
+
if graph.is_dir():
|
|
184
|
+
for src in graph.glob("*.json"):
|
|
185
|
+
try:
|
|
186
|
+
_atomic_copy(src, root / "graph" / src.name)
|
|
187
|
+
copied["graph"] += 1
|
|
188
|
+
except Exception:
|
|
189
|
+
continue
|
|
190
|
+
try:
|
|
191
|
+
from . import entity_graph as _eg_backup
|
|
192
|
+
except Exception: # pragma: no cover
|
|
193
|
+
import entity_graph as _eg_backup # type: ignore
|
|
194
|
+
db = graph / _eg_backup.SQLITE_FILE
|
|
195
|
+
if db.is_file():
|
|
196
|
+
try:
|
|
197
|
+
_eg_backup.checkpoint(db)
|
|
198
|
+
_atomic_copy(db, root / "graph" / db.name)
|
|
199
|
+
copied["graph"] += 1
|
|
200
|
+
except Exception:
|
|
201
|
+
pass
|
|
202
|
+
if not any(copied.values()):
|
|
203
|
+
try:
|
|
204
|
+
root.rmdir()
|
|
205
|
+
except Exception:
|
|
206
|
+
pass
|
|
207
|
+
return "", copied
|
|
208
|
+
return str(root), copied
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def restore(backup_dir, plugin=None):
|
|
212
|
+
"""Copy a pre-migration backup back (one-command rollback)."""
|
|
213
|
+
root = Path(backup_dir)
|
|
214
|
+
report = {"ok": False,
|
|
215
|
+
"restored": {"settings": 0, "memory": 0, "states": 0,
|
|
216
|
+
"graph": 0},
|
|
217
|
+
"warnings": []}
|
|
218
|
+
if not root.is_dir():
|
|
219
|
+
report["warnings"].append("backup dir missing")
|
|
220
|
+
return report
|
|
221
|
+
if plugin is None:
|
|
222
|
+
plugin = type("RestoreHost", (), {
|
|
223
|
+
"data_dir": str(root.parent.parent)})()
|
|
224
|
+
base = _data_dir(plugin)
|
|
225
|
+
authority = _authority_settings(plugin)
|
|
226
|
+
for src in (root / "settings").glob("*.json"):
|
|
227
|
+
try:
|
|
228
|
+
target = authority if src.name == ui_settings.FILE_NAME else (
|
|
229
|
+
base / ("legacy_" + src.name))
|
|
230
|
+
_atomic_copy(src, target)
|
|
231
|
+
report["restored"]["settings"] += 1
|
|
232
|
+
except Exception as exc:
|
|
233
|
+
report["warnings"].append("settings restore failed: %s" % exc)
|
|
234
|
+
for name in ("memory", "states"):
|
|
235
|
+
target_dir = base / name
|
|
236
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
237
|
+
for src in (root / name).glob("*.json"):
|
|
238
|
+
try:
|
|
239
|
+
_atomic_copy(src, target_dir / src.name)
|
|
240
|
+
report["restored"][name] += 1
|
|
241
|
+
except Exception as exc:
|
|
242
|
+
report["warnings"].append("%s restore failed: %s" % (name, exc))
|
|
243
|
+
# task-146 rule 6: restore the SQLite store from the JSON backup (and the
|
|
244
|
+
# backed-up DB when present). JSON stays the fallback if this fails.
|
|
245
|
+
try:
|
|
246
|
+
try:
|
|
247
|
+
from . import memory_store as _ms_restore
|
|
248
|
+
except Exception: # pragma: no cover
|
|
249
|
+
import memory_store as _ms_restore # type: ignore
|
|
250
|
+
memory_target = _data_dir(plugin) / "memory"
|
|
251
|
+
memory_target.mkdir(parents=True, exist_ok=True)
|
|
252
|
+
backup_memory = root / "memory"
|
|
253
|
+
db_backup = backup_memory / _ms_restore.SQLITE_FILE
|
|
254
|
+
if db_backup.is_file():
|
|
255
|
+
target_db = memory_target / _ms_restore.SQLITE_FILE
|
|
256
|
+
try:
|
|
257
|
+
import sqlite3 as _sqlite3_restore
|
|
258
|
+
if target_db.is_file():
|
|
259
|
+
_conn = _sqlite3_restore.connect(str(target_db), timeout=5.0)
|
|
260
|
+
try:
|
|
261
|
+
_conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
262
|
+
finally:
|
|
263
|
+
_conn.close()
|
|
264
|
+
except Exception:
|
|
265
|
+
pass
|
|
266
|
+
_atomic_copy(db_backup, target_db)
|
|
267
|
+
for suffix in ("-wal", "-shm"):
|
|
268
|
+
try:
|
|
269
|
+
Path(str(target_db) + suffix).unlink(missing_ok=True)
|
|
270
|
+
except Exception:
|
|
271
|
+
pass
|
|
272
|
+
report["restored"]["memory_db"] = 1
|
|
273
|
+
elif backup_memory.is_dir():
|
|
274
|
+
result = _ms_restore.restore(backup_memory, memory_target)
|
|
275
|
+
if result.get("ok"):
|
|
276
|
+
report["restored"]["memory_db"] = int(
|
|
277
|
+
result.get("entries", 0) or 0)
|
|
278
|
+
else:
|
|
279
|
+
report["warnings"].append(
|
|
280
|
+
"memory db restore failed: " + str(result.get("error") or ""))
|
|
281
|
+
except Exception as exc:
|
|
282
|
+
report["warnings"].append("memory db restore failed: %s" % exc)
|
|
283
|
+
|
|
284
|
+
graph_target = _graph_dir(plugin)
|
|
285
|
+
graph_target.mkdir(parents=True, exist_ok=True)
|
|
286
|
+
try:
|
|
287
|
+
from . import entity_graph as _eg_restore
|
|
288
|
+
except Exception: # pragma: no cover
|
|
289
|
+
import entity_graph as _eg_restore # type: ignore
|
|
290
|
+
current_db = graph_target / _eg_restore.SQLITE_FILE
|
|
291
|
+
for src in (root / "graph").glob("*.json"):
|
|
292
|
+
try:
|
|
293
|
+
_atomic_copy(src, graph_target / src.name)
|
|
294
|
+
report["restored"]["graph"] += 1
|
|
295
|
+
except Exception:
|
|
296
|
+
report["warnings"].append("graph restore failed: " + src.name)
|
|
297
|
+
backup_db = root / "graph" / _eg_restore.SQLITE_FILE
|
|
298
|
+
if backup_db.is_file():
|
|
299
|
+
old_db = current_db.with_name(current_db.name + ".restore-old")
|
|
300
|
+
try:
|
|
301
|
+
if current_db.is_file():
|
|
302
|
+
_eg_restore.checkpoint(current_db)
|
|
303
|
+
_atomic_copy(current_db, old_db)
|
|
304
|
+
_atomic_copy(backup_db, current_db)
|
|
305
|
+
for suffix in ("-wal", "-shm"):
|
|
306
|
+
Path(str(current_db) + suffix).unlink(missing_ok=True)
|
|
307
|
+
report["restored"]["graph"] += 1
|
|
308
|
+
try:
|
|
309
|
+
old_db.unlink(missing_ok=True)
|
|
310
|
+
except Exception:
|
|
311
|
+
pass
|
|
312
|
+
except Exception as exc:
|
|
313
|
+
if old_db.is_file():
|
|
314
|
+
try:
|
|
315
|
+
_atomic_copy(old_db, current_db)
|
|
316
|
+
except Exception:
|
|
317
|
+
pass
|
|
318
|
+
report["warnings"].append(
|
|
319
|
+
"graph restore failed: %s" % type(exc).__name__)
|
|
320
|
+
report["ok"] = not report["warnings"]
|
|
321
|
+
logger.info("[biocore] migrate restore done: %s warnings=%d",
|
|
322
|
+
report["restored"], len(report["warnings"]))
|
|
323
|
+
return report
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _flatten(data, prefix=""):
|
|
327
|
+
out = {}
|
|
328
|
+
for key, value in (data or {}).items():
|
|
329
|
+
name = prefix + str(key)
|
|
330
|
+
if isinstance(value, dict) and value:
|
|
331
|
+
out.update(_flatten(value, name + "."))
|
|
332
|
+
else:
|
|
333
|
+
out[name] = value
|
|
334
|
+
return out
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _merge_settings(plugin, stats, warnings):
|
|
338
|
+
"""a) Merge old A -> new B when A exists and B is absent/stale."""
|
|
339
|
+
legacy = _legacy_settings(plugin)
|
|
340
|
+
authority = _authority_settings(plugin)
|
|
341
|
+
if legacy is None:
|
|
342
|
+
stats["settings"] = "absent"
|
|
343
|
+
return _read(authority)
|
|
344
|
+
old = _read(legacy)
|
|
345
|
+
if not old:
|
|
346
|
+
stats["settings"] = "empty_legacy"
|
|
347
|
+
return _read(authority)
|
|
348
|
+
new = _read(authority)
|
|
349
|
+
try:
|
|
350
|
+
stale = (not authority.is_file()
|
|
351
|
+
or legacy.stat().st_mtime > authority.stat().st_mtime)
|
|
352
|
+
except Exception:
|
|
353
|
+
stale = not authority.is_file()
|
|
354
|
+
if not stale:
|
|
355
|
+
stats["settings"] = "authority_newer"
|
|
356
|
+
return new
|
|
357
|
+
overridden = []
|
|
358
|
+
merged = dict(new)
|
|
359
|
+
for section, values in old.items():
|
|
360
|
+
if section == "schema":
|
|
361
|
+
continue
|
|
362
|
+
if isinstance(values, dict) and isinstance(merged.get(section), dict):
|
|
363
|
+
for key, value in values.items():
|
|
364
|
+
if key in merged[section] and merged[section][key] != value:
|
|
365
|
+
overridden.append(section + "." + str(key))
|
|
366
|
+
merged[section] = {**merged[section], **values}
|
|
367
|
+
else:
|
|
368
|
+
if section in merged and merged[section] != values:
|
|
369
|
+
overridden.append(str(section))
|
|
370
|
+
merged[section] = values
|
|
371
|
+
try:
|
|
372
|
+
_write(authority, merged)
|
|
373
|
+
except Exception as exc:
|
|
374
|
+
warnings.append("settings write failed: %s" % type(exc).__name__)
|
|
375
|
+
stats["settings"] = "write_failed"
|
|
376
|
+
return _read(authority)
|
|
377
|
+
stats["settings"] = "merged"
|
|
378
|
+
stats["overridden"] = len(overridden)
|
|
379
|
+
stats["overridden_keys"] = overridden[:12]
|
|
380
|
+
return merged
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _autonomy_params(plugin):
|
|
384
|
+
try:
|
|
385
|
+
getter = getattr(plugin, "_autonomy_params", None)
|
|
386
|
+
if callable(getter):
|
|
387
|
+
params = getter()
|
|
388
|
+
if isinstance(params, dict):
|
|
389
|
+
return params
|
|
390
|
+
except Exception:
|
|
391
|
+
pass
|
|
392
|
+
return {}
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _infer_last_umo(plugin):
|
|
396
|
+
data = _data_dir(plugin)
|
|
397
|
+
state_dir = data / "states"
|
|
398
|
+
if not state_dir.is_dir():
|
|
399
|
+
state_dir = ui_settings.authority_dir(plugin) / "states"
|
|
400
|
+
try:
|
|
401
|
+
files = sorted(state_dir.glob("*.json"), key=lambda p: p.stat().st_mtime,
|
|
402
|
+
reverse=True)
|
|
403
|
+
except Exception:
|
|
404
|
+
files = []
|
|
405
|
+
for path in files[:5]:
|
|
406
|
+
row = _read(path)
|
|
407
|
+
for key in ("umo", "last_umo", "session", "unified_msg_origin"):
|
|
408
|
+
value = str(row.get(key) or "").strip()
|
|
409
|
+
if value:
|
|
410
|
+
return value
|
|
411
|
+
if path.stem and path.stem not in ("state", "bio_state"):
|
|
412
|
+
return path.stem
|
|
413
|
+
return ""
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _fix_accumulator(plugin, data, stats):
|
|
417
|
+
"""b) Accumulator keys + dirty V/theta repair."""
|
|
418
|
+
auto = data.get("autonomy")
|
|
419
|
+
if not isinstance(auto, dict) or "accumulator" not in auto:
|
|
420
|
+
stats["accumulator"] = "absent"
|
|
421
|
+
return data
|
|
422
|
+
acc = auto.get("accumulator") or {}
|
|
423
|
+
if not isinstance(acc, dict):
|
|
424
|
+
stats["accumulator"] = "absent"
|
|
425
|
+
return data
|
|
426
|
+
params = _autonomy_params(plugin)
|
|
427
|
+
try:
|
|
428
|
+
theta = float(params.get("theta", 0.55) or 0.55)
|
|
429
|
+
except Exception:
|
|
430
|
+
theta = 0.55
|
|
431
|
+
fixed = []
|
|
432
|
+
try:
|
|
433
|
+
if float(acc.get("V", 0.0) or 0.0) < 0:
|
|
434
|
+
acc["V"] = 0.0
|
|
435
|
+
fixed.append("V")
|
|
436
|
+
except Exception:
|
|
437
|
+
acc["V"] = 0.0
|
|
438
|
+
fixed.append("V")
|
|
439
|
+
if "theta" not in acc or acc.get("theta") in (None, "", 0, "0"):
|
|
440
|
+
acc["theta"] = theta
|
|
441
|
+
fixed.append("theta")
|
|
442
|
+
else:
|
|
443
|
+
try:
|
|
444
|
+
if abs(float(acc.get("theta")) - 0.55) < 1e-9 and abs(theta - 0.55) > 1e-9:
|
|
445
|
+
acc["theta"] = theta
|
|
446
|
+
fixed.append("theta")
|
|
447
|
+
except Exception:
|
|
448
|
+
acc["theta"] = theta
|
|
449
|
+
fixed.append("theta")
|
|
450
|
+
for key in ("cooldown_seconds", "daily_cap"):
|
|
451
|
+
if key not in acc:
|
|
452
|
+
acc[key] = int(params.get("cooldown_s" if key == "cooldown_seconds"
|
|
453
|
+
else "daily_cap",
|
|
454
|
+
ACC_DEFAULTS[key]) or ACC_DEFAULTS[key])
|
|
455
|
+
fixed.append(key)
|
|
456
|
+
if not str(acc.get("last_umo") or "").strip():
|
|
457
|
+
acc["last_umo"] = _infer_last_umo(plugin)
|
|
458
|
+
if acc["last_umo"]:
|
|
459
|
+
fixed.append("last_umo")
|
|
460
|
+
data["autonomy"] = data.get("autonomy") or {}
|
|
461
|
+
data["autonomy"]["accumulator"] = acc
|
|
462
|
+
stats["accumulator"] = "fixed" if fixed else "ok"
|
|
463
|
+
stats["accumulator_fixed"] = fixed
|
|
464
|
+
return data
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _graph_dir(plugin):
|
|
468
|
+
"""Where the entity graph lives (authority first, legacy data dir second)."""
|
|
469
|
+
for candidate in (ui_settings.authority_dir(plugin) / "graph",
|
|
470
|
+
_data_dir(plugin) / "graph"):
|
|
471
|
+
if candidate.is_dir():
|
|
472
|
+
return candidate
|
|
473
|
+
return ui_settings.authority_dir(plugin) / "graph"
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _memory_dir(plugin):
|
|
477
|
+
data = _data_dir(plugin)
|
|
478
|
+
for candidate in (data / "memory",
|
|
479
|
+
ui_settings.authority_dir(plugin) / "memory"):
|
|
480
|
+
if candidate.is_dir():
|
|
481
|
+
return candidate
|
|
482
|
+
return None
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _retrieval_from_mtime(path, now=None):
|
|
486
|
+
now = float(now if now is not None else time.time())
|
|
487
|
+
try:
|
|
488
|
+
age_days = max(0.0, (now - path.stat().st_mtime) / 86400.0)
|
|
489
|
+
except Exception:
|
|
490
|
+
age_days = 0.0
|
|
491
|
+
return round(max(0.05, min(0.95, 0.9 / ((1.0 + age_days) ** 0.5))), 3)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _backfill_memory(plugin, stats):
|
|
495
|
+
"""c) Old entries missing fields -> storage_s/retrieval_s/access_count..."""
|
|
496
|
+
directory = _memory_dir(plugin)
|
|
497
|
+
if directory is None:
|
|
498
|
+
stats["memory"] = "absent"
|
|
499
|
+
return 0
|
|
500
|
+
touched = 0
|
|
501
|
+
entries_total = 0
|
|
502
|
+
now = time.time()
|
|
503
|
+
try:
|
|
504
|
+
files = sorted(directory.glob("*.json"))
|
|
505
|
+
except Exception:
|
|
506
|
+
files = []
|
|
507
|
+
for path in files:
|
|
508
|
+
doc = _read(path)
|
|
509
|
+
entries = doc.get("entries") if isinstance(doc.get("entries"), list) else None
|
|
510
|
+
if entries is None:
|
|
511
|
+
continue
|
|
512
|
+
changed = False
|
|
513
|
+
fallback_retrieval = _retrieval_from_mtime(path, now)
|
|
514
|
+
for entry in entries:
|
|
515
|
+
if not isinstance(entry, dict):
|
|
516
|
+
continue
|
|
517
|
+
entries_total += 1
|
|
518
|
+
if "storage_s" not in entry:
|
|
519
|
+
entry["storage_s"] = MEMORY_STORAGE_S
|
|
520
|
+
changed = True
|
|
521
|
+
if "retrieval_s" not in entry:
|
|
522
|
+
entry["retrieval_s"] = fallback_retrieval
|
|
523
|
+
changed = True
|
|
524
|
+
if "access_count" not in entry:
|
|
525
|
+
entry["access_count"] = 1
|
|
526
|
+
changed = True
|
|
527
|
+
if "arousal_at_encoding" not in entry:
|
|
528
|
+
entry["arousal_at_encoding"] = MEMORY_AROUSAL
|
|
529
|
+
changed = True
|
|
530
|
+
if "hidden" not in entry:
|
|
531
|
+
entry["hidden"] = False
|
|
532
|
+
changed = True
|
|
533
|
+
if changed:
|
|
534
|
+
doc["entries"] = entries
|
|
535
|
+
try:
|
|
536
|
+
_write(path, doc)
|
|
537
|
+
except Exception:
|
|
538
|
+
continue
|
|
539
|
+
touched += 1
|
|
540
|
+
stats["memory"] = ("backfilled" if touched else
|
|
541
|
+
("ok" if entries_total else "empty"))
|
|
542
|
+
stats["memory_files"] = touched
|
|
543
|
+
stats["memory_entries"] = entries_total
|
|
544
|
+
return entries_total
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _ensure_graph(plugin, stats, warnings):
|
|
548
|
+
"""v3: SQLite graph (create / import legacy graph.json / recover corrupt)."""
|
|
549
|
+
try:
|
|
550
|
+
try:
|
|
551
|
+
from . import entity_graph as _eg
|
|
552
|
+
except Exception: # pragma: no cover
|
|
553
|
+
import entity_graph as _eg # type: ignore
|
|
554
|
+
graph_dir = _graph_dir(plugin)
|
|
555
|
+
graph_dir.mkdir(parents=True, exist_ok=True)
|
|
556
|
+
db_path = graph_dir / _eg.SQLITE_FILE
|
|
557
|
+
json_path = graph_dir / "graph.json"
|
|
558
|
+
db_existed = db_path.is_file()
|
|
559
|
+
db_corrupt = False
|
|
560
|
+
if db_existed:
|
|
561
|
+
try:
|
|
562
|
+
with db_path.open("rb") as _fh:
|
|
563
|
+
db_corrupt = _fh.read(16) != b"SQLite format 3\x00"
|
|
564
|
+
except Exception:
|
|
565
|
+
db_corrupt = True
|
|
566
|
+
json_corrupt = False
|
|
567
|
+
json_data = None
|
|
568
|
+
if json_path.is_file():
|
|
569
|
+
try:
|
|
570
|
+
raw = json.loads(json_path.read_text(encoding="utf-8-sig"))
|
|
571
|
+
json_data = _eg.normalize_graph(raw)
|
|
572
|
+
except Exception:
|
|
573
|
+
json_corrupt = True
|
|
574
|
+
try:
|
|
575
|
+
import shutil as _shutil
|
|
576
|
+
_shutil.copyfile(
|
|
577
|
+
json_path,
|
|
578
|
+
json_path.with_name(json_path.name + ".corrupt.bak"))
|
|
579
|
+
except Exception:
|
|
580
|
+
pass
|
|
581
|
+
store = _eg.open_store(db_path)
|
|
582
|
+
try:
|
|
583
|
+
db_empty = _eg.stats(store)["entities"] == 0
|
|
584
|
+
imported = {"entities": 0, "edges": 0}
|
|
585
|
+
source = "none"
|
|
586
|
+
if db_empty and json_data is not None and json_data["entities"]:
|
|
587
|
+
imported = store.import_json(json_path)
|
|
588
|
+
source = "json"
|
|
589
|
+
store.export_json(json_path)
|
|
590
|
+
state = _eg.stats(store)
|
|
591
|
+
if json_corrupt or db_corrupt:
|
|
592
|
+
stats["graph"] = "recovered"
|
|
593
|
+
elif source == "json":
|
|
594
|
+
stats["graph"] = "imported"
|
|
595
|
+
elif not db_existed:
|
|
596
|
+
stats["graph"] = "created"
|
|
597
|
+
else:
|
|
598
|
+
stats["graph"] = ("ok" if state["entities"] else "empty")
|
|
599
|
+
stats["graph_imported"] = imported
|
|
600
|
+
stats["graph_entities"] = state["entities"]
|
|
601
|
+
stats["graph_edges"] = state["edges"]
|
|
602
|
+
logger.info(
|
|
603
|
+
"[biocore] graph sqlite ready: db=%s entities=%d edges=%d "
|
|
604
|
+
"source=%s recovered=%s", db_path.name, state["entities"],
|
|
605
|
+
state["edges"], source, bool(json_corrupt or db_corrupt))
|
|
606
|
+
finally:
|
|
607
|
+
store.close()
|
|
608
|
+
return db_path
|
|
609
|
+
except Exception as exc:
|
|
610
|
+
warnings.append("graph init failed: %s" % type(exc).__name__)
|
|
611
|
+
stats["graph"] = "failed"
|
|
612
|
+
return None
|
|
613
|
+
|
|
614
|
+
def _ensure_memory_sqlite(plugin, stats, warnings):
|
|
615
|
+
"""v4: memory/*.json entries -> memory.sqlite3 (auto backup, idempotent).
|
|
616
|
+
|
|
617
|
+
Rules (user requirement: 老用户升级不能丢数据):
|
|
618
|
+
- backup the whole memory/ dir under backups/pre-migration-<stamp>/ first;
|
|
619
|
+
- import only when the DB is empty (both sides present -> config wins, no
|
|
620
|
+
implicit merge);
|
|
621
|
+
- on any failure the JSON files stay untouched and the plugin keeps JSON
|
|
622
|
+
mode (this function never raises).
|
|
623
|
+
"""
|
|
624
|
+
try:
|
|
625
|
+
try:
|
|
626
|
+
from . import memory_store as _ms
|
|
627
|
+
except Exception: # pragma: no cover
|
|
628
|
+
import memory_store as _ms # type: ignore
|
|
629
|
+
memory_dir = _memory_dir(plugin)
|
|
630
|
+
if memory_dir is None:
|
|
631
|
+
memory_dir = ui_settings.authority_dir(plugin) / "memory"
|
|
632
|
+
memory_dir.mkdir(parents=True, exist_ok=True)
|
|
633
|
+
backup_root = ui_settings.authority_dir(plugin) / "backups"
|
|
634
|
+
backup = _ms._backup_memory_dir(memory_dir, backup_root,
|
|
635
|
+
label="pre-migration")
|
|
636
|
+
report = _ms.ensure_store(memory_dir, backup_root=None,
|
|
637
|
+
label="pre-migration")
|
|
638
|
+
stats["memory_backup"] = backup
|
|
639
|
+
stats["memory_db"] = report.get("state", "failed")
|
|
640
|
+
stats["memory_db_entries"] = int(report.get("entries", 0) or 0)
|
|
641
|
+
stats["memory_db_sessions"] = int(report.get("sessions", 0) or 0)
|
|
642
|
+
_migration = report.get("migration") or {}
|
|
643
|
+
stats["memory_dedupe_merged"] = int(
|
|
644
|
+
_migration.get("merged_pairs", 0) or 0)
|
|
645
|
+
stats["memory_dedupe_deleted"] = int(
|
|
646
|
+
_migration.get("deleted_rows", 0) or 0)
|
|
647
|
+
stats["memory_dedupe_backup"] = str(
|
|
648
|
+
report.get("dedupe_backup") or "")
|
|
649
|
+
if _migration.get("ran"):
|
|
650
|
+
logger.info(
|
|
651
|
+
"[biocore] memory dedupe: merged_pairs=%d deleted_rows=%d "
|
|
652
|
+
"backfilled=%d backup=%s",
|
|
653
|
+
int(stats["memory_dedupe_merged"]),
|
|
654
|
+
int(stats["memory_dedupe_deleted"]),
|
|
655
|
+
int(_migration.get("backfilled", 0) or 0),
|
|
656
|
+
bool(stats["memory_dedupe_backup"]))
|
|
657
|
+
if not report.get("ok"):
|
|
658
|
+
warnings.append("memory sqlite ensure failed: %s"
|
|
659
|
+
% str(report.get("error") or "")[:120])
|
|
660
|
+
logger.info(
|
|
661
|
+
"[biocore] memory sqlite ready: backend=%s sessions=%d entries=%d "
|
|
662
|
+
"source=%s backup=%s",
|
|
663
|
+
"sqlite", int(stats["memory_db_sessions"]),
|
|
664
|
+
int(stats["memory_db_entries"]), report.get("state", "?"),
|
|
665
|
+
bool(backup.get("dir")))
|
|
666
|
+
return report
|
|
667
|
+
except Exception as exc:
|
|
668
|
+
warnings.append("memory sqlite failed: %s: %s"
|
|
669
|
+
% (type(exc).__name__, exc))
|
|
670
|
+
stats["memory_db"] = "failed"
|
|
671
|
+
return {}
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def migrate(plugin) -> dict:
|
|
675
|
+
"""Run the 1.3.4 -> 1.4.0 migration once; safe to call again."""
|
|
676
|
+
report = {"applied": [], "warnings": [], "stats": {},
|
|
677
|
+
"schema_version": SCHEMA_VERSION, "ts": time.time(),
|
|
678
|
+
"backup_dir": ""}
|
|
679
|
+
stats = report["stats"]
|
|
680
|
+
warnings = report["warnings"]
|
|
681
|
+
authority = _authority_settings(plugin)
|
|
682
|
+
current = _read(authority)
|
|
683
|
+
try:
|
|
684
|
+
version = int(current.get("schema_version", 0) or 0)
|
|
685
|
+
except Exception:
|
|
686
|
+
version = SCHEMA_VERSION
|
|
687
|
+
if version >= SCHEMA_VERSION:
|
|
688
|
+
stats["already"] = True
|
|
689
|
+
report["schema_version"] = version
|
|
690
|
+
logger.info("[biocore] migrate skipped: schema_version=%s (up-to-date)", version)
|
|
691
|
+
return report
|
|
692
|
+
# Auto backup before any write; failure is a warning, migration continues.
|
|
693
|
+
try:
|
|
694
|
+
backup_dir, copied = _create_backup(plugin)
|
|
695
|
+
report["backup_dir"] = backup_dir
|
|
696
|
+
stats["backup"] = copied
|
|
697
|
+
if backup_dir:
|
|
698
|
+
logger.info("[biocore] migrate backup: %s", backup_dir)
|
|
699
|
+
except Exception as exc:
|
|
700
|
+
warnings.append("backup failed: %s" % type(exc).__name__)
|
|
701
|
+
data = _merge_settings(plugin, stats, warnings)
|
|
702
|
+
if stats.get("settings") == "merged":
|
|
703
|
+
report["applied"].append("settings")
|
|
704
|
+
before_acc = json.dumps((data.get("autonomy") or {}).get("accumulator") or {},
|
|
705
|
+
ensure_ascii=False, sort_keys=True)
|
|
706
|
+
data = _fix_accumulator(plugin, data, stats)
|
|
707
|
+
after_acc = json.dumps((data.get("autonomy") or {}).get("accumulator") or {},
|
|
708
|
+
ensure_ascii=False, sort_keys=True)
|
|
709
|
+
if stats.get("accumulator") == "fixed":
|
|
710
|
+
report["applied"].append("accumulator")
|
|
711
|
+
stats["accumulator_before"] = before_acc
|
|
712
|
+
stats["accumulator_after"] = after_acc
|
|
713
|
+
_backfill_memory(plugin, stats)
|
|
714
|
+
if stats.get("memory") == "backfilled":
|
|
715
|
+
report["applied"].append("memory")
|
|
716
|
+
_ensure_graph(plugin, stats, warnings)
|
|
717
|
+
_ensure_memory_sqlite(plugin, stats, warnings)
|
|
718
|
+
data["schema_version"] = SCHEMA_VERSION
|
|
719
|
+
try:
|
|
720
|
+
_write(authority, data)
|
|
721
|
+
report["schema_version"] = SCHEMA_VERSION
|
|
722
|
+
except Exception as exc:
|
|
723
|
+
warnings.append("authority write failed: %s" % type(exc).__name__)
|
|
724
|
+
report["schema_version"] = version
|
|
725
|
+
fixed = stats.get("accumulator_fixed") or []
|
|
726
|
+
logger.info(
|
|
727
|
+
"[biocore] migrate done: settings=%s(%d keys overridden), "
|
|
728
|
+
"accumulator=%s(%s), memory=%d entries backfilled, warnings=%d",
|
|
729
|
+
stats.get("settings", "?"), int(stats.get("overridden", 0) or 0),
|
|
730
|
+
stats.get("accumulator", "?"), ",".join(fixed) or "none",
|
|
731
|
+
int(stats.get("memory_entries", 0) or 0
|
|
732
|
+
if stats.get("memory") == "backfilled" else 0), len(warnings))
|
|
733
|
+
return report
|