dsh-biocore 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/README.md +2 -2
  2. package/lib/py/bridge.py +15 -7
  3. package/lib/py-bridge.js +4 -3
  4. package/package.json +3 -2
  5. package/python/biocore_memory/__init__.py +21 -0
  6. package/python/biocore_memory/adapters/__init__.py +2 -0
  7. package/python/biocore_memory/adapters/astrbot.py +24 -0
  8. package/python/biocore_memory/adapters/dsh_cli.py +335 -0
  9. package/python/biocore_memory/co_recall.py +324 -0
  10. package/python/biocore_memory/consolidate/__init__.py +11 -0
  11. package/python/biocore_memory/consolidate/gated.py +104 -0
  12. package/python/biocore_memory/consolidate/neighbors.py +68 -0
  13. package/python/biocore_memory/consolidate/reflection.py +45 -0
  14. package/python/biocore_memory/consolidate/tree.py +95 -0
  15. package/python/biocore_memory/consolidate/writeback.py +181 -0
  16. package/python/biocore_memory/core/__init__.py +4 -0
  17. package/python/biocore_memory/core/config.py +127 -0
  18. package/python/biocore_memory/core/permissions.py +69 -0
  19. package/python/biocore_memory/core/types.py +116 -0
  20. package/python/biocore_memory/core/util.py +26 -0
  21. package/python/biocore_memory/embedding_local.py +447 -0
  22. package/python/biocore_memory/embedding_mode.py +156 -0
  23. package/python/biocore_memory/engine.py +2476 -0
  24. package/python/biocore_memory/engine_helpers.py +266 -0
  25. package/python/biocore_memory/event/__init__.py +3 -0
  26. package/python/biocore_memory/event/_types.py +16 -0
  27. package/python/biocore_memory/event/consistency.py +117 -0
  28. package/python/biocore_memory/event/extract.py +48 -0
  29. package/python/biocore_memory/event/media.py +111 -0
  30. package/python/biocore_memory/event/notes.py +85 -0
  31. package/python/biocore_memory/event/notes_link.py +87 -0
  32. package/python/biocore_memory/event/vector.py +69 -0
  33. package/python/biocore_memory/evict/__init__.py +3 -0
  34. package/python/biocore_memory/evict/adaptive.py +73 -0
  35. package/python/biocore_memory/evict/consequence.py +58 -0
  36. package/python/biocore_memory/evict/heat.py +54 -0
  37. package/python/biocore_memory/evict/inhibition.py +105 -0
  38. package/python/biocore_memory/evict/policy.py +188 -0
  39. package/python/biocore_memory/evict/score.py +23 -0
  40. package/python/biocore_memory/identity/__init__.py +6 -0
  41. package/python/biocore_memory/identity/api.py +99 -0
  42. package/python/biocore_memory/identity/history.py +9 -0
  43. package/python/biocore_memory/identity/ledger.py +191 -0
  44. package/python/biocore_memory/identity/narrative.py +358 -0
  45. package/python/biocore_memory/identity/snapshot.py +13 -0
  46. package/python/biocore_memory/identity/writeback.py +61 -0
  47. package/python/biocore_memory/index/__init__.py +3 -0
  48. package/python/biocore_memory/index/cue_index.py +80 -0
  49. package/python/biocore_memory/index/entities.py +85 -0
  50. package/python/biocore_memory/index/entity_boost.py +104 -0
  51. package/python/biocore_memory/index/graph.py +172 -0
  52. package/python/biocore_memory/recall/__init__.py +5 -0
  53. package/python/biocore_memory/recall/api.py +65 -0
  54. package/python/biocore_memory/recall/bm25.py +118 -0
  55. package/python/biocore_memory/recall/channels.py +98 -0
  56. package/python/biocore_memory/recall/complexity.py +62 -0
  57. package/python/biocore_memory/recall/constraints.py +75 -0
  58. package/python/biocore_memory/recall/embedding.py +745 -0
  59. package/python/biocore_memory/recall/expand.py +26 -0
  60. package/python/biocore_memory/recall/filters.py +40 -0
  61. package/python/biocore_memory/recall/fusion.py +18 -0
  62. package/python/biocore_memory/recall/mmr.py +70 -0
  63. package/python/biocore_memory/recall/ppr.py +85 -0
  64. package/python/biocore_memory/recall/retrieval_practice.py +102 -0
  65. package/python/biocore_memory/relay/__init__.py +2 -0
  66. package/python/biocore_memory/relay/card.py +51 -0
  67. package/python/biocore_memory/slots/__init__.py +18 -0
  68. package/python/biocore_memory/slots/admission.py +64 -0
  69. package/python/biocore_memory/slots/boundary.py +128 -0
  70. package/python/biocore_memory/slots/evict_bridge.py +106 -0
  71. package/python/biocore_memory/slots/registry.py +135 -0
  72. package/python/biocore_memory/slots/switch.py +42 -0
  73. package/python/biocore_memory/store/__init__.py +4 -0
  74. package/python/biocore_memory/store/audit.py +74 -0
  75. package/python/biocore_memory/store/compress.py +81 -0
  76. package/python/biocore_memory/store/dedupe.py +178 -0
  77. package/python/biocore_memory/store/json_store.py +154 -0
  78. package/python/biocore_memory/store/media.py +145 -0
  79. package/python/biocore_memory/store/provenance.py +73 -0
  80. package/python/biocore_memory/store/rate_limit.py +93 -0
  81. package/python/biocore_memory/store/security.py +107 -0
  82. package/python/biocore_memory/store/source_consistency.py +123 -0
  83. package/python/biocore_memory/store/tiering.py +61 -0
  84. package/python/biocore_memory/store/tiering_exec.py +50 -0
  85. package/python/biocore_memory/store/tombstone.py +23 -0
  86. package/python/biocore_memory/store/versions.py +69 -0
  87. package/python/biocore_memory/thread/__init__.py +2 -0
  88. package/python/biocore_memory/thread/card.py +58 -0
  89. package/python/biocore_model/__init__.py +37 -0
  90. package/python/biocore_model/affective_labor.py +162 -0
  91. package/python/biocore_model/appraisal.py +257 -0
  92. package/python/biocore_model/appraisal_action.py +78 -0
  93. package/python/biocore_model/archive_queue.py +60 -0
  94. package/python/biocore_model/attachment_behavior.py +130 -0
  95. package/python/biocore_model/attachment_regulation.py +75 -0
  96. package/python/biocore_model/attachment_state.py +41 -0
  97. package/python/biocore_model/attribution.py +73 -0
  98. package/python/biocore_model/attributor.py +162 -0
  99. package/python/biocore_model/autonomic.py +91 -0
  100. package/python/biocore_model/autonomy_affect.py +51 -0
  101. package/python/biocore_model/autonomy_check.py +67 -0
  102. package/python/biocore_model/autonomy_loop.py +122 -0
  103. package/python/biocore_model/autonomy_mode.py +114 -0
  104. package/python/biocore_model/autonomy_model.py +733 -0
  105. package/python/biocore_model/autonomy_params.py +131 -0
  106. package/python/biocore_model/autonomy_reason.py +73 -0
  107. package/python/biocore_model/autonomy_tick.py +227 -0
  108. package/python/biocore_model/bio_appraisal.py +49 -0
  109. package/python/biocore_model/bio_autonomy_state.py +171 -0
  110. package/python/biocore_model/bio_config.py +221 -0
  111. package/python/biocore_model/bio_const.py +128 -0
  112. package/python/biocore_model/bio_daily.py +267 -0
  113. package/python/biocore_model/bio_emotion.py +538 -0
  114. package/python/biocore_model/bio_emotion2.py +214 -0
  115. package/python/biocore_model/bio_goal.py +220 -0
  116. package/python/biocore_model/bio_identity.py +82 -0
  117. package/python/biocore_model/bio_noise.py +119 -0
  118. package/python/biocore_model/bio_physiology2.py +114 -0
  119. package/python/biocore_model/bio_snapshot.py +324 -0
  120. package/python/biocore_model/bio_state.py +272 -0
  121. package/python/biocore_model/bio_tick.py +310 -0
  122. package/python/biocore_model/bio_util.py +82 -0
  123. package/python/biocore_model/blocks.py +264 -0
  124. package/python/biocore_model/boundary.py +251 -0
  125. package/python/biocore_model/boundary_gate.py +188 -0
  126. package/python/biocore_model/commitment_extractor.py +517 -0
  127. package/python/biocore_model/compassion_fatigue.py +84 -0
  128. package/python/biocore_model/connection_bids.py +120 -0
  129. package/python/biocore_model/core_bridge.py +177 -0
  130. package/python/biocore_model/core_client.py +127 -0
  131. package/python/biocore_model/core_commands.py +112 -0
  132. package/python/biocore_model/core_mode.py +104 -0
  133. package/python/biocore_model/core_mode_ui.mjs +104 -0
  134. package/python/biocore_model/criticality.py +81 -0
  135. package/python/biocore_model/defense_mechanisms.py +272 -0
  136. package/python/biocore_model/delta_budget.py +99 -0
  137. package/python/biocore_model/diag_trace.py +274 -0
  138. package/python/biocore_model/dissonance.py +134 -0
  139. package/python/biocore_model/emotion_geometry.py +108 -0
  140. package/python/biocore_model/emotion_probe.py +209 -0
  141. package/python/biocore_model/entity_graph.py +931 -0
  142. package/python/biocore_model/entity_resolve.py +260 -0
  143. package/python/biocore_model/envy.py +193 -0
  144. package/python/biocore_model/exit_or_silence.py +230 -0
  145. package/python/biocore_model/expr_guard.py +24 -0
  146. package/python/biocore_model/expr_intent.py +77 -0
  147. package/python/biocore_model/expr_state.py +204 -0
  148. package/python/biocore_model/expr_template.py +105 -0
  149. package/python/biocore_model/expr_tone.py +81 -0
  150. package/python/biocore_model/extreme_scenarios.py +205 -0
  151. package/python/biocore_model/feedback_flow.py +227 -0
  152. package/python/biocore_model/feedback_guard.py +129 -0
  153. package/python/biocore_model/feedback_llm.py +163 -0
  154. package/python/biocore_model/feedback_memory.py +96 -0
  155. package/python/biocore_model/feedback_user.py +93 -0
  156. package/python/biocore_model/frustration_curve.py +110 -0
  157. package/python/biocore_model/granularity.py +283 -0
  158. package/python/biocore_model/graph_extract.py +248 -0
  159. package/python/biocore_model/graph_recall.py +221 -0
  160. package/python/biocore_model/intent_render.py +91 -0
  161. package/python/biocore_model/intero_core.py +235 -0
  162. package/python/biocore_model/intero_style.py +116 -0
  163. package/python/biocore_model/layered_memory.py +297 -0
  164. package/python/biocore_model/learning_probe.py +120 -0
  165. package/python/biocore_model/memory_dual.py +224 -0
  166. package/python/biocore_model/memory_forgetting.py +125 -0
  167. package/python/biocore_model/memory_store.py +1399 -0
  168. package/python/biocore_model/metacog.py +153 -0
  169. package/python/biocore_model/migrate.py +733 -0
  170. package/python/biocore_model/migrate_store.py +337 -0
  171. package/python/biocore_model/mind_wander.py +111 -0
  172. package/python/biocore_model/mood.py +48 -0
  173. package/python/biocore_model/narrative.py +172 -0
  174. package/python/biocore_model/needs_esteem.py +116 -0
  175. package/python/biocore_model/needs_events.py +137 -0
  176. package/python/biocore_model/needs_sleep.py +72 -0
  177. package/python/biocore_model/noise_bank.py +95 -0
  178. package/python/biocore_model/persona_flow.mjs +25 -0
  179. package/python/biocore_model/pipeline.py +240 -0
  180. package/python/biocore_model/pipeline_api.py +200 -0
  181. package/python/biocore_model/plugin_config.py +83 -0
  182. package/python/biocore_model/plugin_routes.py +1435 -0
  183. package/python/biocore_model/policy_learning.py +132 -0
  184. package/python/biocore_model/possible_selves.py +110 -0
  185. package/python/biocore_model/predictor.py +178 -0
  186. package/python/biocore_model/preset.py +205 -0
  187. package/python/biocore_model/proactive_context.py +153 -0
  188. package/python/biocore_model/prompt_rules.py +90 -0
  189. package/python/biocore_model/prospective.py +183 -0
  190. package/python/biocore_model/provider_fallback.py +97 -0
  191. package/python/biocore_model/reconstruction.py +132 -0
  192. package/python/biocore_model/regulation_loop.py +179 -0
  193. package/python/biocore_model/regulation_strategies.py +294 -0
  194. package/python/biocore_model/relation_stage.py +130 -0
  195. package/python/biocore_model/relationship_stage.py +95 -0
  196. package/python/biocore_model/render.py +901 -0
  197. package/python/biocore_model/render_guard.mjs +34 -0
  198. package/python/biocore_model/repair_after_boundary.py +350 -0
  199. package/python/biocore_model/reply_limit.py +183 -0
  200. package/python/biocore_model/reply_split.py +203 -0
  201. package/python/biocore_model/rollback.py +143 -0
  202. package/python/biocore_model/runtime.py +479 -0
  203. package/python/biocore_model/sanitize.py +200 -0
  204. package/python/biocore_model/scheduler.py +123 -0
  205. package/python/biocore_model/schema.py +120 -0
  206. package/python/biocore_model/secrecy.py +145 -0
  207. package/python/biocore_model/self_esteem.py +50 -0
  208. package/python/biocore_model/self_intent.py +437 -0
  209. package/python/biocore_model/self_interest.py +263 -0
  210. package/python/biocore_model/self_verification.py +178 -0
  211. package/python/biocore_model/services.py +78 -0
  212. package/python/biocore_model/situation.py +254 -0
  213. package/python/biocore_model/sleep_circadian.py +116 -0
  214. package/python/biocore_model/social_pain.py +69 -0
  215. package/python/biocore_model/social_repair.py +111 -0
  216. package/python/biocore_model/social_theory.py +131 -0
  217. package/python/biocore_model/source_monitoring.py +292 -0
  218. package/python/biocore_model/stages.py +487 -0
  219. package/python/biocore_model/state_tag.py +231 -0
  220. package/python/biocore_model/switch_groups.mjs +201 -0
  221. package/python/biocore_model/tabs_state.mjs +37 -0
  222. package/python/biocore_model/template_expr.py +252 -0
  223. package/python/biocore_model/tools/caps_check.mjs +23 -0
  224. package/python/biocore_model/tools/init_symbols_check.mjs +11 -0
  225. package/python/biocore_model/tools/missing_ids.mjs +16 -0
  226. package/python/biocore_model/tools/scope_check.mjs +19 -0
  227. package/python/biocore_model/topic_control.py +268 -0
  228. package/python/biocore_model/turn_memory.py +152 -0
  229. package/python/biocore_model/ui_compat.mjs +48 -0
  230. package/python/biocore_model/ui_settings.py +266 -0
  231. package/python/biocore_model/ui_settings.py.bak_authority +108 -0
  232. package/python/biocore_model/value_order.py +186 -0
  233. package/python/biocore_model/view_mode.mjs +8 -0
  234. package/python/biocore_model/voice_bank.py +177 -0
  235. package/python/biocore_model/waiting_sense.py +229 -0
  236. package/python/biocore_model/x_directives.py +142 -0
  237. package/python/biocore_runtime/__init__.py +7 -0
  238. package/python/biocore_runtime/pipeline.py +1915 -0
  239. package/python/dsh_adapter/README.md +96 -0
  240. package/python/dsh_adapter/adapter.py +1077 -0
  241. package/python/dsh_adapter/cli.py +107 -0
  242. package/python/dsh_adapter/data/demo_state.json +164 -0
  243. package/python/dsh_adapter/data/verify_state.json +164 -0
  244. package/python/dsh_adapter/tests/__init__.py +1 -0
  245. package/python/dsh_adapter/tests/test_adapter.py +494 -0
  246. package/ui/panel-layout.js +3 -3
  247. package/ui/panel.css +53 -6
  248. package/ui/panel.html +33 -32
  249. package/ui/panel.js +9 -1
  250. package/lib/py/__pycache__/bridge.cpython-310.pyc +0 -0
@@ -0,0 +1,931 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Full entity graph memory: nodes + typed, timestamped edges (task-134).
3
+
4
+ Storage contract: one JSON document {"schema": 3, "entities": [...],
5
+ "edges": [...]}. Core operations are pure graph transforms; only
6
+ load_graph/save_graph touch the filesystem (atomic replace, corrupt recovery).
7
+ No tag approximations: entities are matched by normalized name or an explicit
8
+ alias, and edges carry rel/ts/confidence/source_entry for time reasoning and
9
+ two-hop retrieval.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import copy
14
+ import hashlib
15
+ import json
16
+ import os
17
+ import shutil
18
+ import sqlite3
19
+ from pathlib import Path
20
+
21
+ from . import entity_resolve as _er
22
+
23
+ SCHEMA_VERSION = 3
24
+ ENTITY_TYPES = ("person", "place", "event", "topic")
25
+ DEFAULT_TYPE = "topic"
26
+ MAX_ALIASES = 12
27
+ MAX_SOURCES = 8
28
+
29
+
30
+ def _num(value, default=0.0):
31
+ try:
32
+ number = float(value)
33
+ except Exception:
34
+ return default
35
+ return number if number == number else default
36
+
37
+
38
+ def _norm(text):
39
+ return "".join(ch for ch in str(text or "").lower()
40
+ if ch.isalnum())
41
+
42
+
43
+ def _clean_text(text):
44
+ return " ".join(str(text or "").split()).strip()
45
+
46
+
47
+ def _id(prefix, *parts):
48
+ raw = "|".join(str(part) for part in parts)
49
+ return prefix + hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12]
50
+
51
+
52
+ def empty_graph():
53
+ return {"schema": SCHEMA_VERSION, "entities": [], "edges": []}
54
+
55
+
56
+ def normalize_graph(raw):
57
+ """Sanitize a stored document into a valid graph (never raises)."""
58
+ graph = empty_graph()
59
+ if not isinstance(raw, dict):
60
+ return graph
61
+ entities = raw.get("entities")
62
+ if isinstance(entities, list):
63
+ seen = set()
64
+ for item in entities:
65
+ if not isinstance(item, dict):
66
+ continue
67
+ name = _clean_text(item.get("name"))
68
+ norm = _norm(name)
69
+ if not norm or norm in seen:
70
+ continue
71
+ seen.add(norm)
72
+ kind = str(item.get("type") or DEFAULT_TYPE).strip().lower()
73
+ aliases = []
74
+ for alias in (item.get("aliases") or []):
75
+ text = _clean_text(alias)
76
+ if text and _norm(text) != norm and text not in aliases:
77
+ aliases.append(text)
78
+ graph["entities"].append({
79
+ "id": str(item.get("id") or _id("e_", norm)),
80
+ "name": name,
81
+ "type": kind if kind in ENTITY_TYPES else DEFAULT_TYPE,
82
+ "aliases": aliases[:MAX_ALIASES],
83
+ "first_seen": _num(item.get("first_seen"), 0.0),
84
+ "last_seen": _num(item.get("last_seen"), 0.0),
85
+ "sources": [str(x) for x in (item.get("sources") or [])
86
+ if str(x)][:MAX_SOURCES],
87
+ })
88
+ edges = raw.get("edges")
89
+ if isinstance(edges, list):
90
+ known = {row["id"] for row in graph["entities"]}
91
+ seen = set()
92
+ for item in edges:
93
+ if not isinstance(item, dict):
94
+ continue
95
+ src = str(item.get("src") or "").strip()
96
+ dst = str(item.get("dst") or "").strip()
97
+ rel = str(item.get("rel") or "").strip().lower()
98
+ if not src or not dst or not rel or src not in known or dst not in known:
99
+ continue
100
+ ts = _num(item.get("ts"), 0.0)
101
+ edge_id = str(item.get("id") or _id("r_", src, rel, dst, ts))
102
+ if edge_id in seen:
103
+ continue
104
+ seen.add(edge_id)
105
+ graph["edges"].append({
106
+ "id": edge_id,
107
+ "src": src,
108
+ "rel": rel,
109
+ "dst": dst,
110
+ "ts": ts,
111
+ "confidence": max(0.0, min(1.0, _num(item.get("confidence"), 1.0))),
112
+ "source_entry": str(item.get("source_entry") or ""),
113
+ })
114
+ graph["schema"] = SCHEMA_VERSION
115
+ return graph
116
+
117
+
118
+ def _matches(entity, key):
119
+ norm = _norm(key)
120
+ if not norm:
121
+ return False
122
+ return _norm(entity.get("name")) == norm or any(
123
+ _norm(alias) == norm for alias in (entity.get("aliases") or []))
124
+
125
+
126
+ def find_entity(graph, name):
127
+ """Return the entity matched by name or alias, or None."""
128
+ for entity in (graph or {}).get("entities") or []:
129
+ if isinstance(entity, dict) and _matches(entity, name):
130
+ return entity
131
+ return None
132
+
133
+
134
+ def resolve_name(graph, name):
135
+ entity = find_entity(graph, name)
136
+ if entity is not None:
137
+ return entity["id"]
138
+ key = _clean_text(name)
139
+ return _id("e_", _norm(key)) if key else ""
140
+
141
+
142
+ def upsert_entity(graph, name, type=None, aliases=None, now=None, entry_id=""):
143
+ """Add or merge one entity; returns (graph_or_store, entity_id)."""
144
+ if isinstance(graph, GraphStore):
145
+ return graph.upsert_entity(name, type=type, aliases=aliases, now=now,
146
+ entry_id=entry_id)
147
+ out = normalize_graph(copy.deepcopy(graph))
148
+ key = _clean_text(name)
149
+ norm = _norm(key)
150
+ if not norm:
151
+ return out, ""
152
+ stamp = _num(now, 0.0)
153
+ extra_aliases = [_clean_text(x) for x in (aliases or []) if _clean_text(x)]
154
+ entity = find_entity(out, key)
155
+ if entity is None:
156
+ # P0-1: fold an alias / near-identical spelling into the existing node.
157
+ hit = _er.match(out["entities"], key, extra_aliases)
158
+ if hit is not None:
159
+ entity = next((row for row in out["entities"]
160
+ if row.get("id") == hit["id"]), None)
161
+ if (entity is not None
162
+ and _norm(key) != _norm(entity.get("name"))):
163
+ extra_aliases.append(key)
164
+ if entity is None:
165
+ entity = {
166
+ "id": _id("e_", norm),
167
+ "name": key,
168
+ "type": (str(type).strip().lower() if type else DEFAULT_TYPE),
169
+ "aliases": [],
170
+ "first_seen": stamp,
171
+ "last_seen": stamp,
172
+ "sources": [],
173
+ }
174
+ if entity["type"] not in ENTITY_TYPES:
175
+ entity["type"] = DEFAULT_TYPE
176
+ out["entities"].append(entity)
177
+ else:
178
+ if type:
179
+ kind = str(type).strip().lower()
180
+ if kind in ENTITY_TYPES:
181
+ entity["type"] = kind
182
+ entity["last_seen"] = max(_num(entity.get("last_seen"), 0.0), stamp)
183
+ entity["first_seen"] = min(
184
+ _num(entity.get("first_seen"), stamp) or stamp, stamp or 0.0)
185
+ name_norm = _norm(entity.get("name"))
186
+ aliases_now = [text for text in (entity.get("aliases") or [])
187
+ if _norm(text) != name_norm]
188
+ for alias in extra_aliases:
189
+ text = _clean_text(alias)
190
+ if not text or _norm(text) == name_norm or text in aliases_now:
191
+ continue
192
+ aliases_now.append(text)
193
+ entity["aliases"] = aliases_now[:MAX_ALIASES]
194
+ if entry_id:
195
+ sources = list(entity.get("sources") or [])
196
+ if entry_id not in sources:
197
+ sources.append(str(entry_id))
198
+ entity["sources"] = sources[-MAX_SOURCES:]
199
+ return out, entity["id"]
200
+
201
+
202
+ def upsert_edge(graph, src, rel, dst, ts=None, confidence=1.0, source_entry="",
203
+ src_type=None, dst_type=None, now=None):
204
+ """Add one typed, timestamped edge (auto-creating entities); returns graph/id."""
205
+ if isinstance(graph, GraphStore):
206
+ return graph.upsert_edge(src, rel, dst, ts=ts, confidence=confidence,
207
+ source_entry=source_entry, src_type=src_type,
208
+ dst_type=dst_type, now=now)
209
+ out = normalize_graph(copy.deepcopy(graph))
210
+ source = _clean_text(src)
211
+ target = _clean_text(dst)
212
+ relation = str(rel or "").strip().lower()
213
+ if not source or not target or not relation:
214
+ return out, ""
215
+ out, src_id = upsert_entity(out, source, type=src_type, now=now,
216
+ entry_id=source_entry)
217
+ out, dst_id = upsert_entity(out, target, type=dst_type, now=now,
218
+ entry_id=source_entry)
219
+ if not src_id or not dst_id or src_id == dst_id:
220
+ # P0-1: both spellings resolved to one node -> a self-loop is noise.
221
+ return out, ""
222
+ stamp = _num(ts, _num(now, 0.0))
223
+ confidence = max(0.0, min(1.0, _num(confidence, 1.0)))
224
+ edge_id = _id("r_", src_id, relation, dst_id, stamp)
225
+ for edge in out["edges"]:
226
+ if edge.get("id") == edge_id:
227
+ edge["confidence"] = max(float(edge.get("confidence", 0.0) or 0.0),
228
+ confidence)
229
+ if source_entry:
230
+ edge["source_entry"] = str(source_entry)
231
+ return out, edge_id
232
+ out["edges"].append({
233
+ "id": edge_id,
234
+ "src": src_id,
235
+ "rel": relation,
236
+ "dst": dst_id,
237
+ "ts": stamp,
238
+ "confidence": confidence,
239
+ "source_entry": str(source_entry or ""),
240
+ })
241
+ return out, edge_id
242
+
243
+
244
+ def _merge_report(keep_id, drop_id):
245
+ return {"ok": False, "keep_id": str(keep_id or ""),
246
+ "drop_id": str(drop_id or ""), "aliases": 0, "sources": 0,
247
+ "edges_rewired": 0, "edges_dropped": 0, "error": ""}
248
+
249
+
250
+ def find_similar(graph, name, aliases=(), threshold=None):
251
+ """Best {"id","name","score","via"} for a name/alias, or None (P0-1)."""
252
+ if isinstance(graph, GraphStore):
253
+ return graph.find_similar(name, aliases=aliases, threshold=threshold)
254
+ clean = normalize_graph(copy.deepcopy(graph))
255
+ limit = _er.SIMILARITY_THRESHOLD if threshold is None else threshold
256
+ return _er.match(clean["entities"], name, aliases, limit)
257
+
258
+
259
+ def merge_entities(graph, keep_id, drop_id):
260
+ """Merge drop_id into keep_id; returns (graph_or_store, report).
261
+
262
+ Fields are unioned (aliases/sources/first/last/type) and every edge that
263
+ pointed at the dropped node is rewired to the kept one. Edges that become a
264
+ self-loop are dropped; edges that collapse onto an existing row are folded.
265
+ """
266
+ if isinstance(graph, GraphStore):
267
+ return graph, graph.merge_entities(keep_id, drop_id)
268
+ out = normalize_graph(copy.deepcopy(graph))
269
+ report = _merge_report(keep_id, drop_id)
270
+ if not report["keep_id"] or not report["drop_id"]:
271
+ report["error"] = "keep_id/drop_id required"
272
+ return out, report
273
+ if report["keep_id"] == report["drop_id"]:
274
+ report["error"] = "same entity"
275
+ return out, report
276
+ keep = next((row for row in out["entities"]
277
+ if row["id"] == report["keep_id"]), None)
278
+ drop = next((row for row in out["entities"]
279
+ if row["id"] == report["drop_id"]), None)
280
+ if keep is None or drop is None:
281
+ report["error"] = "entity not found"
282
+ return out, report
283
+ keep["aliases"] = _er.merged_aliases(
284
+ keep["name"], keep["aliases"], drop["name"], drop["aliases"])
285
+ keep["sources"] = _er.merged_sources(keep["sources"], drop["sources"])
286
+ keep["type"] = _er.merged_type(keep["type"], drop["type"])
287
+ keep["first_seen"], keep["last_seen"] = _er.merged_span(
288
+ keep.get("first_seen"), keep.get("last_seen"),
289
+ drop.get("first_seen"), drop.get("last_seen"))
290
+ out["entities"] = [row for row in out["entities"]
291
+ if row["id"] != report["drop_id"]]
292
+ out["edges"], stats = _er.rewrite_edges(
293
+ out["edges"], report["keep_id"], report["drop_id"],
294
+ make_id=lambda src, rel, dst, ts: _id("r_", src, rel, dst, ts))
295
+ report.update({"ok": True, "aliases": len(keep["aliases"]),
296
+ "sources": len(keep["sources"]),
297
+ "edges_rewired": stats["rewired"],
298
+ "edges_dropped": stats["dropped"]})
299
+ return out, report
300
+
301
+
302
+ def _window_start(window_days, now):
303
+ if window_days is None:
304
+ return None
305
+ days = _num(window_days, 0.0)
306
+ if days <= 0:
307
+ return None
308
+ base = _num(now, 0.0)
309
+ if base <= 0:
310
+ latest = 0.0
311
+ return None
312
+ return base - days * 86400.0
313
+
314
+
315
+ def neighbors(graph, entity, hops=2, window_days=None, now=None, limit=80):
316
+ """Neighbours around an entity/alias (dict = in-memory, GraphStore = CTE)."""
317
+ if isinstance(graph, GraphStore):
318
+ return graph.neighbors(entity, hops=hops, window_days=window_days,
319
+ now=now, limit=limit)
320
+ clean = normalize_graph(graph)
321
+ start = find_entity(clean, entity)
322
+ result = {"entities": [], "edges": []}
323
+ if start is None:
324
+ return result
325
+ try:
326
+ depth_limit = max(1, min(4, int(hops)))
327
+ except Exception:
328
+ depth_limit = 2
329
+ try:
330
+ row_limit = max(1, min(500, int(limit)))
331
+ except Exception:
332
+ row_limit = 80
333
+ start_ts = _window_start(window_days, now)
334
+ by_id = {row["id"]: row for row in clean["entities"]}
335
+ visited = {start["id"]}
336
+ frontier = [start["id"]]
337
+ used_edge_ids = set()
338
+ for _depth in range(depth_limit):
339
+ next_frontier = []
340
+ for node_id in frontier:
341
+ for edge in clean["edges"]:
342
+ if edge["src"] != node_id and edge["dst"] != node_id:
343
+ continue
344
+ if start_ts is not None and _num(edge.get("ts"), 0.0) < start_ts:
345
+ continue
346
+ other = edge["dst"] if edge["src"] == node_id else edge["src"]
347
+ if other not in visited:
348
+ visited.add(other)
349
+ next_frontier.append(other)
350
+ if edge["id"] not in used_edge_ids:
351
+ used_edge_ids.add(edge["id"])
352
+ result["edges"].append(dict(edge))
353
+ frontier = next_frontier
354
+ if not frontier:
355
+ break
356
+ result["entities"] = [dict(by_id[eid]) for eid in visited
357
+ if eid in by_id][:row_limit]
358
+ result["edges"].sort(key=lambda row: (-_num(row.get("ts"), 0.0),
359
+ -_num(row.get("confidence"), 0.0)))
360
+ result["edges"] = result["edges"][:row_limit]
361
+ return result
362
+
363
+
364
+ def search(graph, text, limit=8):
365
+ """Entities whose name or alias appears in *text* (longest match first)."""
366
+ if isinstance(graph, GraphStore):
367
+ return graph.search(text, limit=limit)
368
+ clean = normalize_graph(graph)
369
+ haystack = _norm(text)
370
+ hits = []
371
+ if not haystack:
372
+ return hits
373
+ for entity in clean["entities"]:
374
+ names = [entity.get("name")] + list(entity.get("aliases") or [])
375
+ best = 0
376
+ for name in names:
377
+ norm = _norm(name)
378
+ if norm and norm in haystack:
379
+ best = max(best, len(norm))
380
+ if best:
381
+ hits.append((best, entity))
382
+ hits.sort(key=lambda row: row[0], reverse=True)
383
+ return [dict(entity) for _score, entity in hits[:max(1, int(limit or 8))]]
384
+
385
+
386
+ def stats(graph):
387
+ """Bounded counts used by the panel and health checks."""
388
+ if isinstance(graph, GraphStore):
389
+ return graph.stats()
390
+ clean = normalize_graph(graph)
391
+ types = {}
392
+ aliases = 0
393
+ for entity in clean["entities"]:
394
+ kind = str(entity.get("type") or DEFAULT_TYPE)
395
+ types[kind] = int(types.get(kind, 0)) + 1
396
+ aliases += len(entity.get("aliases") or [])
397
+ latest = max([_num(edge.get("ts"), 0.0) for edge in clean["edges"]] or [0.0])
398
+ return {"schema": clean["schema"],
399
+ "entities": len(clean["entities"]),
400
+ "edges": len(clean["edges"]),
401
+ "types": types,
402
+ "aliases": aliases,
403
+ "latest_ts": latest}
404
+
405
+
406
+ def load_graph(path):
407
+ """Read a graph document; corrupt files are backed up and recovered empty."""
408
+ target = Path(path)
409
+ if not target.is_file():
410
+ return empty_graph()
411
+ try:
412
+ raw = json.loads(target.read_text(encoding="utf-8-sig"))
413
+ return normalize_graph(raw)
414
+ except Exception:
415
+ try:
416
+ shutil.copyfile(target, target.with_name(target.name + ".corrupt.bak"))
417
+ except Exception:
418
+ pass
419
+ return empty_graph()
420
+
421
+
422
+ def save_graph(path, graph):
423
+ """Atomic write of the graph document (tmp + os.replace)."""
424
+ target = Path(path)
425
+ target.parent.mkdir(parents=True, exist_ok=True)
426
+ payload = normalize_graph(graph)
427
+ tmp = target.with_name(target.name + ".tmp")
428
+ try:
429
+ tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
430
+ encoding="utf-8")
431
+ os.replace(tmp, target)
432
+ except Exception:
433
+ try:
434
+ tmp.unlink(missing_ok=True)
435
+ except Exception:
436
+ pass
437
+ raise
438
+ return target
439
+
440
+
441
+
442
+ # ---------------------------------------------------------------------------
443
+ # SQLite storage layer (task-134 addendum). The public functions above accept
444
+ # a GraphStore and delegate to it; the JSON helpers below stay the
445
+ # export/backup format only.
446
+ # ---------------------------------------------------------------------------
447
+
448
+ SQLITE_FILE = "graph.sqlite3"
449
+ _SCHEMA_SQL = """
450
+ CREATE TABLE IF NOT EXISTS entities(
451
+ id TEXT PRIMARY KEY, name TEXT, type TEXT, aliases TEXT,
452
+ first_seen REAL, last_seen REAL, sources TEXT);
453
+ CREATE TABLE IF NOT EXISTS edges(
454
+ id TEXT PRIMARY KEY, src TEXT, rel TEXT, dst TEXT, ts REAL,
455
+ confidence REAL, source_entry TEXT);
456
+ CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
457
+ CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src);
458
+ CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst);
459
+ CREATE INDEX IF NOT EXISTS idx_edges_ts ON edges(ts);
460
+ CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(rel);
461
+ """
462
+
463
+
464
+ def _row_entity(row):
465
+ def _json_list(raw):
466
+ try:
467
+ value = json.loads(raw or "[]")
468
+ except Exception:
469
+ return []
470
+ return [str(x) for x in value if str(x)] if isinstance(value, list) else []
471
+
472
+ return {"id": row[0], "name": row[1], "type": row[2] or DEFAULT_TYPE,
473
+ "aliases": _json_list(row[3]),
474
+ "first_seen": _num(row[4], 0.0), "last_seen": _num(row[5], 0.0),
475
+ "sources": _json_list(row[6])}
476
+
477
+
478
+ def _row_edge(row):
479
+ return {"id": row[0], "src": row[1], "rel": row[2], "dst": row[3],
480
+ "ts": _num(row[4], 0.0),
481
+ "confidence": _num(row[5], 1.0),
482
+ "source_entry": row[6] or ""}
483
+
484
+
485
+ _ENTITY_COLS = "id, name, type, aliases, first_seen, last_seen, sources"
486
+ _EDGE_COLS = "id, src, rel, dst, ts, confidence, source_entry"
487
+
488
+
489
+ class GraphStore:
490
+ """Thin SQLite wrapper. One connection, WAL, explicit transactions."""
491
+
492
+ def __init__(self, path, recover=True, similarity_threshold=None):
493
+ self.path = Path(path)
494
+ self._conn = None
495
+ # P0-1: <=0 disables the similarity fallback (exact name/alias only).
496
+ self.similarity_threshold = (
497
+ _er.SIMILARITY_THRESHOLD if similarity_threshold is None
498
+ else float(similarity_threshold))
499
+ self._open(recover=recover)
500
+
501
+ # -- connection / schema ------------------------------------------------
502
+ def _open(self, recover=True):
503
+ self.path.parent.mkdir(parents=True, exist_ok=True)
504
+ try:
505
+ self._conn = sqlite3.connect(str(self.path), timeout=5.0)
506
+ self._configure()
507
+ self._ensure_schema()
508
+ except sqlite3.DatabaseError:
509
+ if not recover:
510
+ raise
511
+ try:
512
+ self._conn.close()
513
+ except Exception:
514
+ pass
515
+ self._conn = None
516
+ self._recover_corrupt()
517
+ self._conn = sqlite3.connect(str(self.path), timeout=5.0)
518
+ self._configure()
519
+ self._ensure_schema()
520
+
521
+ def _configure(self):
522
+ cur = self._conn.cursor()
523
+ cur.execute("PRAGMA journal_mode=WAL")
524
+ cur.execute("PRAGMA synchronous=NORMAL")
525
+ cur.execute("PRAGMA busy_timeout=5000")
526
+
527
+ def _ensure_schema(self):
528
+ with self._conn:
529
+ self._conn.executescript(_SCHEMA_SQL)
530
+
531
+ def _recover_corrupt(self):
532
+ try:
533
+ if self.path.is_file():
534
+ shutil.copyfile(self.path,
535
+ self.path.with_name(self.path.name + ".corrupt.bak"))
536
+ except Exception:
537
+ pass
538
+ for suffix in ("", "-wal", "-shm"):
539
+ try:
540
+ Path(str(self.path) + suffix).unlink(missing_ok=True)
541
+ except Exception:
542
+ pass
543
+
544
+ def close(self):
545
+ if self._conn is not None:
546
+ try:
547
+ self._conn.close()
548
+ except Exception:
549
+ pass
550
+ self._conn = None
551
+
552
+ def __enter__(self):
553
+ return self
554
+
555
+ def __exit__(self, *args):
556
+ self.close()
557
+
558
+ def checkpoint(self):
559
+ """Fold the WAL back into the main file before copying it."""
560
+ try:
561
+ self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
562
+ return True
563
+ except Exception:
564
+ return False
565
+
566
+ # -- entity / edge writes ----------------------------------------------
567
+ def _find_entity(self, name):
568
+ norm = _norm(name)
569
+ if not norm:
570
+ return None
571
+ cur = self._conn.execute(
572
+ "SELECT " + _ENTITY_COLS + " FROM entities "
573
+ "WHERE lower(replace(name,' ',''))=? OR aliases LIKE ? LIMIT 1",
574
+ (norm, "%" + str(name) + "%"))
575
+ row = cur.fetchone()
576
+ return _row_entity(row) if row else None
577
+
578
+ def _row_by_id(self, entity_id):
579
+ row = self._conn.execute(
580
+ "SELECT " + _ENTITY_COLS + " FROM entities WHERE id=?",
581
+ (str(entity_id or ""),)).fetchone()
582
+ return _row_entity(row) if row else None
583
+
584
+ def find_similar(self, name, aliases=(), threshold=None):
585
+ """Best {"id","name","score","via"} match, bounded scan (P0-1)."""
586
+ limit = (self.similarity_threshold if threshold is None
587
+ else float(threshold))
588
+ if not limit or limit <= 0:
589
+ return None
590
+ rows = [_row_entity(row) for row in self._conn.execute(
591
+ "SELECT " + _ENTITY_COLS + " FROM entities "
592
+ "ORDER BY last_seen DESC LIMIT ?", (_er.SCAN_LIMIT,)).fetchall()]
593
+ return _er.match(rows, name, aliases, limit)
594
+
595
+ def upsert_entity(self, name, type=None, aliases=None, now=None, entry_id=""):
596
+ key = _clean_text(name)
597
+ norm = _norm(key)
598
+ if not norm:
599
+ return self, ""
600
+ stamp = _num(now, 0.0)
601
+ with self._conn:
602
+ entity = self._find_entity(key)
603
+ if entity is None:
604
+ # P0-1: fold a near-identical spelling into the existing node.
605
+ hit = self.find_similar(key, aliases or ())
606
+ if hit is not None:
607
+ entity = self._row_by_id(hit["id"])
608
+ entity_name = _clean_text(entity.get("name")) if entity else key
609
+ name_norm = _norm(entity_name)
610
+ if entity is not None and _norm(key) != name_norm:
611
+ # Keep the spelling this message used as an alias.
612
+ aliases = list(aliases or []) + [key]
613
+ aliases_now = [text for text in
614
+ (_clean_text(alias) for alias in
615
+ (entity.get("aliases") if entity else []))
616
+ if text and _norm(text) != name_norm]
617
+ for alias in (aliases or []):
618
+ text = _clean_text(alias)
619
+ if text and _norm(text) != name_norm and text not in aliases_now:
620
+ aliases_now.append(text)
621
+ aliases_now = aliases_now[:MAX_ALIASES]
622
+ sources = list(entity.get("sources") or []) if entity else []
623
+ if entry_id and str(entry_id) not in sources:
624
+ sources.append(str(entry_id))
625
+ if entity is None:
626
+ eid = _id("e_", norm)
627
+ kind = str(type).strip().lower() if type else DEFAULT_TYPE
628
+ if kind not in ENTITY_TYPES:
629
+ kind = DEFAULT_TYPE
630
+ self._conn.execute(
631
+ "INSERT OR REPLACE INTO entities(" + _ENTITY_COLS + ") "
632
+ "VALUES(?,?,?,?,?,?,?)",
633
+ (eid, key, kind, json.dumps(aliases_now, ensure_ascii=False),
634
+ stamp, stamp, json.dumps(sources, ensure_ascii=False)))
635
+ return self, eid
636
+ eid = str(entity["id"])
637
+ kind = str(entity.get("type") or DEFAULT_TYPE)
638
+ if type:
639
+ candidate = str(type).strip().lower()
640
+ if candidate in ENTITY_TYPES:
641
+ kind = candidate
642
+ first_seen = _num(entity.get("first_seen"), stamp)
643
+ if not first_seen:
644
+ first_seen = stamp
645
+ self._conn.execute(
646
+ "UPDATE entities SET name=?, type=?, aliases=?, first_seen=?, "
647
+ "last_seen=?, sources=? WHERE id=?",
648
+ (entity.get("name") or key, kind,
649
+ json.dumps(aliases_now, ensure_ascii=False),
650
+ min(first_seen, stamp or first_seen),
651
+ max(_num(entity.get("last_seen"), 0.0), stamp),
652
+ json.dumps(sources, ensure_ascii=False), eid))
653
+ return self, eid
654
+
655
+ def upsert_edge(self, src, rel, dst, ts=None, confidence=1.0,
656
+ source_entry="", src_type=None, dst_type=None, now=None):
657
+ source = _clean_text(src)
658
+ target = _clean_text(dst)
659
+ relation = str(rel or "").strip().lower()
660
+ if not source or not target or not relation:
661
+ return self, ""
662
+ _, src_id = self.upsert_entity(source, type=src_type, now=now,
663
+ entry_id=source_entry)
664
+ _, dst_id = self.upsert_entity(target, type=dst_type, now=now,
665
+ entry_id=source_entry)
666
+ if not src_id or not dst_id or src_id == dst_id:
667
+ # P0-1: both spellings resolved to one node -> a self-loop is noise.
668
+ return self, ""
669
+ stamp = _num(ts, _num(now, 0.0))
670
+ score = max(0.0, min(1.0, _num(confidence, 1.0)))
671
+ edge_id = _id("r_", src_id, relation, dst_id, stamp)
672
+ with self._conn:
673
+ self._conn.execute(
674
+ "INSERT OR IGNORE INTO edges(" + _EDGE_COLS + ") "
675
+ "VALUES(?,?,?,?,?,?,?)",
676
+ (edge_id, src_id, relation, dst_id, stamp, score,
677
+ str(source_entry or "")))
678
+ self._conn.execute(
679
+ "UPDATE edges SET confidence=MAX(confidence,?), "
680
+ "source_entry=CASE WHEN ?<>'' THEN ? ELSE source_entry END "
681
+ "WHERE id=?",
682
+ (score, str(source_entry or ""), str(source_entry or ""), edge_id))
683
+ return self, edge_id
684
+
685
+ def merge_entities(self, keep_id, drop_id):
686
+ """Merge one entity into another (aliases/sources/time + edge rewire)."""
687
+ report = _merge_report(keep_id, drop_id)
688
+ if not report["keep_id"] or not report["drop_id"]:
689
+ report["error"] = "keep_id/drop_id required"
690
+ return report
691
+ if report["keep_id"] == report["drop_id"]:
692
+ report["error"] = "same entity"
693
+ return report
694
+ keep = self._row_by_id(report["keep_id"])
695
+ drop = self._row_by_id(report["drop_id"])
696
+ if keep is None or drop is None:
697
+ report["error"] = "entity not found"
698
+ return report
699
+ aliases = _er.merged_aliases(keep["name"], keep["aliases"],
700
+ drop["name"], drop["aliases"])
701
+ sources = _er.merged_sources(keep["sources"], drop["sources"])
702
+ kind = _er.merged_type(keep["type"], drop["type"])
703
+ first_seen, last_seen = _er.merged_span(
704
+ keep.get("first_seen"), keep.get("last_seen"),
705
+ drop.get("first_seen"), drop.get("last_seen"))
706
+ with self._conn:
707
+ rewired, dropped = self._rewire_edges(keep["id"], drop["id"])
708
+ self._conn.execute(
709
+ "UPDATE entities SET aliases=?, type=?, first_seen=?, "
710
+ "last_seen=?, sources=? WHERE id=?",
711
+ (json.dumps(aliases, ensure_ascii=False), kind, first_seen,
712
+ last_seen, json.dumps(sources, ensure_ascii=False),
713
+ keep["id"]))
714
+ self._conn.execute("DELETE FROM entities WHERE id=?", (drop["id"],))
715
+ report.update({"ok": True, "aliases": len(aliases),
716
+ "sources": len(sources), "edges_rewired": rewired,
717
+ "edges_dropped": dropped})
718
+ return report
719
+
720
+ def _rewire_edges(self, keep_id, drop_id):
721
+ """Move drop_id edge endpoints onto keep_id; returns (rewired, dropped)."""
722
+ rows = self._conn.execute(
723
+ "SELECT " + _EDGE_COLS + " FROM edges WHERE src=? OR dst=?",
724
+ (drop_id, drop_id)).fetchall()
725
+ rewired = 0
726
+ dropped = 0
727
+ for row in rows:
728
+ edge = _row_edge(row)
729
+ src = keep_id if edge["src"] == drop_id else edge["src"]
730
+ dst = keep_id if edge["dst"] == drop_id else edge["dst"]
731
+ changed = src != edge["src"] or dst != edge["dst"]
732
+ if changed and src == dst:
733
+ self._conn.execute("DELETE FROM edges WHERE id=?",
734
+ (edge["id"],))
735
+ dropped += 1
736
+ continue
737
+ if not changed:
738
+ continue
739
+ rewired += 1
740
+ new_id = _id("r_", src, edge["rel"], dst, edge["ts"])
741
+ existing = self._conn.execute(
742
+ "SELECT id FROM edges WHERE id=?", (new_id,)).fetchone()
743
+ if existing is None:
744
+ self._conn.execute(
745
+ "INSERT INTO edges(" + _EDGE_COLS + ") "
746
+ "VALUES(?,?,?,?,?,?,?)",
747
+ (new_id, src, edge["rel"], dst, edge["ts"],
748
+ edge["confidence"], edge["source_entry"]))
749
+ else:
750
+ self._conn.execute(
751
+ "UPDATE edges SET confidence=MAX(confidence,?), "
752
+ "source_entry=CASE WHEN ?<>'' THEN ? ELSE source_entry END "
753
+ "WHERE id=?",
754
+ (edge["confidence"], edge["source_entry"],
755
+ edge["source_entry"], new_id))
756
+ dropped += 1
757
+ self._conn.execute("DELETE FROM edges WHERE id=?", (edge["id"],))
758
+ return rewired, dropped
759
+
760
+ # -- queries ------------------------------------------------------------
761
+ def neighbors(self, entity, hops=2, window_days=None, now=None, limit=80):
762
+ start = self._find_entity(entity)
763
+ if start is None:
764
+ return {"entities": [], "edges": []}
765
+ try:
766
+ depth_limit = max(1, min(4, int(hops)))
767
+ except Exception:
768
+ depth_limit = 2
769
+ try:
770
+ row_limit = max(1, min(500, int(limit)))
771
+ except Exception:
772
+ row_limit = 80
773
+ window = _window_start(window_days, now)
774
+ params = [str(start["id"]), depth_limit]
775
+ window_sql = "? IS NULL OR e.ts >= ?"
776
+ params_sql = (None, window) if window is None else (1, window)
777
+ sql = ("WITH RECURSIVE reach(id, depth) AS ("
778
+ "SELECT ?, 0 "
779
+ "UNION "
780
+ "SELECT CASE WHEN e.src = reach.id THEN e.dst ELSE e.src END,"
781
+ " reach.depth + 1 "
782
+ "FROM edges e JOIN reach "
783
+ "ON (e.src = reach.id OR e.dst = reach.id) "
784
+ "WHERE reach.depth < ? AND (" + window_sql + ")) "
785
+ "SELECT id FROM reach")
786
+ rows = self._conn.execute(sql, params + list(params_sql)).fetchall()
787
+ ids = [str(row[0]) for row in rows]
788
+ if not ids:
789
+ return {"entities": [], "edges": []}
790
+ qmarks = ",".join("?" for _ in ids)
791
+ entities = [_row_entity(row) for row in self._conn.execute(
792
+ "SELECT " + _ENTITY_COLS + " FROM entities WHERE id IN (" + qmarks + ") LIMIT ?",
793
+ ids + [row_limit]).fetchall()]
794
+ edge_rows = self._conn.execute(
795
+ "SELECT " + _EDGE_COLS + " FROM edges "
796
+ "WHERE src IN (" + qmarks + ") AND dst IN (" + qmarks + ") "
797
+ "AND (? IS NULL OR ts >= ?) "
798
+ "ORDER BY ts DESC, confidence DESC LIMIT ?",
799
+ ids + ids + [window, window, row_limit]).fetchall()
800
+ return {"entities": entities,
801
+ "edges": [_row_edge(row) for row in edge_rows]}
802
+
803
+ def search(self, text, limit=8):
804
+ haystack = _clean_text(text)
805
+ if not haystack:
806
+ return []
807
+ try:
808
+ cap = max(1, int(limit or 8))
809
+ except Exception:
810
+ cap = 8
811
+ try:
812
+ rows = self._conn.execute(
813
+ "SELECT " + _ENTITY_COLS + ", length(name) AS ln FROM entities "
814
+ "WHERE ? LIKE '%' || name || '%' OR EXISTS("
815
+ "SELECT 1 FROM json_each(entities.aliases) "
816
+ "WHERE ? LIKE '%' || json_each.value || '%') "
817
+ "ORDER BY ln DESC LIMIT ?", (haystack, haystack, cap)).fetchall()
818
+ except sqlite3.DatabaseError:
819
+ rows = []
820
+ if rows:
821
+ return [_row_entity(row[:7]) for row in rows]
822
+ # Fallback for SQLite builds without JSON1: bounded LIKE scan.
823
+ out = []
824
+ for row in self._conn.execute(
825
+ "SELECT " + _ENTITY_COLS + " FROM entities LIMIT 500").fetchall():
826
+ entity = _row_entity(row)
827
+ for name in [entity["name"]] + list(entity["aliases"]):
828
+ if _norm(name) and _norm(name) in _norm(haystack):
829
+ out.append((len(_norm(name)), entity))
830
+ break
831
+ out.sort(key=lambda item: item[0], reverse=True)
832
+ return [entity for _length, entity in out[:cap]]
833
+
834
+ def stats(self):
835
+ entities = int(self._conn.execute(
836
+ "SELECT COUNT(*) FROM entities").fetchone()[0] or 0)
837
+ edges = int(self._conn.execute(
838
+ "SELECT COUNT(*) FROM edges").fetchone()[0] or 0)
839
+ types = {str(kind): int(count) for kind, count in self._conn.execute(
840
+ "SELECT type, COUNT(*) FROM entities GROUP BY type").fetchall()}
841
+ try:
842
+ aliases = int(self._conn.execute(
843
+ "SELECT COALESCE(SUM(json_array_length(aliases)),0) FROM entities"
844
+ ).fetchone()[0] or 0)
845
+ except sqlite3.DatabaseError:
846
+ aliases = 0
847
+ latest = _num(self._conn.execute(
848
+ "SELECT COALESCE(MAX(ts),0) FROM edges").fetchone()[0], 0.0)
849
+ return {"schema": SCHEMA_VERSION, "entities": entities, "edges": edges,
850
+ "types": types, "aliases": aliases, "latest_ts": latest}
851
+
852
+ # -- JSON export / import ----------------------------------------------
853
+ def export_json(self, path=None):
854
+ target = Path(path) if path else self.path.with_name("graph.json")
855
+ graph = {"schema": SCHEMA_VERSION,
856
+ "entities": [_row_entity(row) for row in self._conn.execute(
857
+ "SELECT " + _ENTITY_COLS + " FROM entities ORDER BY name")],
858
+ "edges": [_row_edge(row) for row in self._conn.execute(
859
+ "SELECT " + _EDGE_COLS + " FROM edges ORDER BY ts")]}
860
+ save_graph(target, graph)
861
+ return target
862
+
863
+ def import_json(self, path):
864
+ data = load_graph(path)
865
+ imported = {"entities": 0, "edges": 0}
866
+ if not data["entities"] and not data["edges"]:
867
+ return imported
868
+ with self._conn:
869
+ for entity in data["entities"]:
870
+ self._conn.execute(
871
+ "INSERT OR IGNORE INTO entities(" + _ENTITY_COLS + ") "
872
+ "VALUES(?,?,?,?,?,?,?)",
873
+ (entity["id"], entity["name"], entity["type"],
874
+ json.dumps(entity["aliases"], ensure_ascii=False),
875
+ entity["first_seen"], entity["last_seen"],
876
+ json.dumps(entity["sources"], ensure_ascii=False)))
877
+ imported["entities"] += 1
878
+ for edge in data["edges"]:
879
+ self._conn.execute(
880
+ "INSERT OR IGNORE INTO edges(" + _EDGE_COLS + ") "
881
+ "VALUES(?,?,?,?,?,?,?)",
882
+ (edge["id"], edge["src"], edge["rel"], edge["dst"],
883
+ edge["ts"], edge["confidence"], edge["source_entry"]))
884
+ imported["edges"] += 1
885
+ return imported
886
+
887
+
888
+ def open_store(path, recover=True, similarity_threshold=None):
889
+ return GraphStore(path, recover=recover,
890
+ similarity_threshold=similarity_threshold)
891
+
892
+
893
+ def checkpoint(path):
894
+ """Fold WAL into the SQLite file; safe no-op when the file is absent/broken."""
895
+ target = Path(path)
896
+ if not target.is_file():
897
+ return False
898
+ try:
899
+ conn = sqlite3.connect(str(target), timeout=5.0)
900
+ try:
901
+ conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
902
+ finally:
903
+ conn.close()
904
+ return True
905
+ except Exception:
906
+ return False
907
+
908
+
909
+ def export_json(store, path=None):
910
+ if isinstance(store, GraphStore):
911
+ return store.export_json(path)
912
+ target = Path(path)
913
+ save_graph(target, store)
914
+ return target
915
+
916
+
917
+ def import_json(store, path):
918
+ if isinstance(store, GraphStore):
919
+ return store.import_json(path)
920
+ data = load_graph(path)
921
+ graph = normalize_graph(store)
922
+ entity_sources = {entity["id"]: entity for entity in graph["entities"]}
923
+ merged = {row["id"]: row for row in graph["entities"]}
924
+ for entity in data["entities"]:
925
+ merged.setdefault(entity["id"], entity)
926
+ edges = {row["id"]: row for row in graph["edges"]}
927
+ for edge in data["edges"]:
928
+ edges.setdefault(edge["id"], edge)
929
+ target = {"schema": SCHEMA_VERSION, "entities": list(merged.values()),
930
+ "edges": list(edges.values())}
931
+ return target