dsh-biocore 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/py/bridge.py +15 -7
- package/lib/py-bridge.js +4 -3
- package/package.json +3 -2
- package/python/biocore_memory/__init__.py +21 -0
- package/python/biocore_memory/adapters/__init__.py +2 -0
- package/python/biocore_memory/adapters/astrbot.py +24 -0
- package/python/biocore_memory/adapters/dsh_cli.py +335 -0
- package/python/biocore_memory/co_recall.py +324 -0
- package/python/biocore_memory/consolidate/__init__.py +11 -0
- package/python/biocore_memory/consolidate/gated.py +104 -0
- package/python/biocore_memory/consolidate/neighbors.py +68 -0
- package/python/biocore_memory/consolidate/reflection.py +45 -0
- package/python/biocore_memory/consolidate/tree.py +95 -0
- package/python/biocore_memory/consolidate/writeback.py +181 -0
- package/python/biocore_memory/core/__init__.py +4 -0
- package/python/biocore_memory/core/config.py +127 -0
- package/python/biocore_memory/core/permissions.py +69 -0
- package/python/biocore_memory/core/types.py +116 -0
- package/python/biocore_memory/core/util.py +26 -0
- package/python/biocore_memory/embedding_local.py +447 -0
- package/python/biocore_memory/embedding_mode.py +156 -0
- package/python/biocore_memory/engine.py +2476 -0
- package/python/biocore_memory/engine_helpers.py +266 -0
- package/python/biocore_memory/event/__init__.py +3 -0
- package/python/biocore_memory/event/_types.py +16 -0
- package/python/biocore_memory/event/consistency.py +117 -0
- package/python/biocore_memory/event/extract.py +48 -0
- package/python/biocore_memory/event/media.py +111 -0
- package/python/biocore_memory/event/notes.py +85 -0
- package/python/biocore_memory/event/notes_link.py +87 -0
- package/python/biocore_memory/event/vector.py +69 -0
- package/python/biocore_memory/evict/__init__.py +3 -0
- package/python/biocore_memory/evict/adaptive.py +73 -0
- package/python/biocore_memory/evict/consequence.py +58 -0
- package/python/biocore_memory/evict/heat.py +54 -0
- package/python/biocore_memory/evict/inhibition.py +105 -0
- package/python/biocore_memory/evict/policy.py +188 -0
- package/python/biocore_memory/evict/score.py +23 -0
- package/python/biocore_memory/identity/__init__.py +6 -0
- package/python/biocore_memory/identity/api.py +99 -0
- package/python/biocore_memory/identity/history.py +9 -0
- package/python/biocore_memory/identity/ledger.py +191 -0
- package/python/biocore_memory/identity/narrative.py +358 -0
- package/python/biocore_memory/identity/snapshot.py +13 -0
- package/python/biocore_memory/identity/writeback.py +61 -0
- package/python/biocore_memory/index/__init__.py +3 -0
- package/python/biocore_memory/index/cue_index.py +80 -0
- package/python/biocore_memory/index/entities.py +85 -0
- package/python/biocore_memory/index/entity_boost.py +104 -0
- package/python/biocore_memory/index/graph.py +172 -0
- package/python/biocore_memory/recall/__init__.py +5 -0
- package/python/biocore_memory/recall/api.py +65 -0
- package/python/biocore_memory/recall/bm25.py +118 -0
- package/python/biocore_memory/recall/channels.py +98 -0
- package/python/biocore_memory/recall/complexity.py +62 -0
- package/python/biocore_memory/recall/constraints.py +75 -0
- package/python/biocore_memory/recall/embedding.py +745 -0
- package/python/biocore_memory/recall/expand.py +26 -0
- package/python/biocore_memory/recall/filters.py +40 -0
- package/python/biocore_memory/recall/fusion.py +18 -0
- package/python/biocore_memory/recall/mmr.py +70 -0
- package/python/biocore_memory/recall/ppr.py +85 -0
- package/python/biocore_memory/recall/retrieval_practice.py +102 -0
- package/python/biocore_memory/relay/__init__.py +2 -0
- package/python/biocore_memory/relay/card.py +51 -0
- package/python/biocore_memory/slots/__init__.py +18 -0
- package/python/biocore_memory/slots/admission.py +64 -0
- package/python/biocore_memory/slots/boundary.py +128 -0
- package/python/biocore_memory/slots/evict_bridge.py +106 -0
- package/python/biocore_memory/slots/registry.py +135 -0
- package/python/biocore_memory/slots/switch.py +42 -0
- package/python/biocore_memory/store/__init__.py +4 -0
- package/python/biocore_memory/store/audit.py +74 -0
- package/python/biocore_memory/store/compress.py +81 -0
- package/python/biocore_memory/store/dedupe.py +178 -0
- package/python/biocore_memory/store/json_store.py +154 -0
- package/python/biocore_memory/store/media.py +145 -0
- package/python/biocore_memory/store/provenance.py +73 -0
- package/python/biocore_memory/store/rate_limit.py +93 -0
- package/python/biocore_memory/store/security.py +107 -0
- package/python/biocore_memory/store/source_consistency.py +123 -0
- package/python/biocore_memory/store/tiering.py +61 -0
- package/python/biocore_memory/store/tiering_exec.py +50 -0
- package/python/biocore_memory/store/tombstone.py +23 -0
- package/python/biocore_memory/store/versions.py +69 -0
- package/python/biocore_memory/thread/__init__.py +2 -0
- package/python/biocore_memory/thread/card.py +58 -0
- package/python/biocore_model/__init__.py +37 -0
- package/python/biocore_model/affective_labor.py +162 -0
- package/python/biocore_model/appraisal.py +257 -0
- package/python/biocore_model/appraisal_action.py +78 -0
- package/python/biocore_model/archive_queue.py +60 -0
- package/python/biocore_model/attachment_behavior.py +130 -0
- package/python/biocore_model/attachment_regulation.py +75 -0
- package/python/biocore_model/attachment_state.py +41 -0
- package/python/biocore_model/attribution.py +73 -0
- package/python/biocore_model/attributor.py +162 -0
- package/python/biocore_model/autonomic.py +91 -0
- package/python/biocore_model/autonomy_affect.py +51 -0
- package/python/biocore_model/autonomy_check.py +67 -0
- package/python/biocore_model/autonomy_loop.py +122 -0
- package/python/biocore_model/autonomy_mode.py +114 -0
- package/python/biocore_model/autonomy_model.py +733 -0
- package/python/biocore_model/autonomy_params.py +131 -0
- package/python/biocore_model/autonomy_reason.py +73 -0
- package/python/biocore_model/autonomy_tick.py +227 -0
- package/python/biocore_model/bio_appraisal.py +49 -0
- package/python/biocore_model/bio_autonomy_state.py +171 -0
- package/python/biocore_model/bio_config.py +221 -0
- package/python/biocore_model/bio_const.py +128 -0
- package/python/biocore_model/bio_daily.py +267 -0
- package/python/biocore_model/bio_emotion.py +538 -0
- package/python/biocore_model/bio_emotion2.py +214 -0
- package/python/biocore_model/bio_goal.py +220 -0
- package/python/biocore_model/bio_identity.py +82 -0
- package/python/biocore_model/bio_noise.py +119 -0
- package/python/biocore_model/bio_physiology2.py +114 -0
- package/python/biocore_model/bio_snapshot.py +324 -0
- package/python/biocore_model/bio_state.py +272 -0
- package/python/biocore_model/bio_tick.py +310 -0
- package/python/biocore_model/bio_util.py +82 -0
- package/python/biocore_model/blocks.py +264 -0
- package/python/biocore_model/boundary.py +251 -0
- package/python/biocore_model/boundary_gate.py +188 -0
- package/python/biocore_model/commitment_extractor.py +517 -0
- package/python/biocore_model/compassion_fatigue.py +84 -0
- package/python/biocore_model/connection_bids.py +120 -0
- package/python/biocore_model/core_bridge.py +177 -0
- package/python/biocore_model/core_client.py +127 -0
- package/python/biocore_model/core_commands.py +112 -0
- package/python/biocore_model/core_mode.py +104 -0
- package/python/biocore_model/core_mode_ui.mjs +104 -0
- package/python/biocore_model/criticality.py +81 -0
- package/python/biocore_model/defense_mechanisms.py +272 -0
- package/python/biocore_model/delta_budget.py +99 -0
- package/python/biocore_model/diag_trace.py +274 -0
- package/python/biocore_model/dissonance.py +134 -0
- package/python/biocore_model/emotion_geometry.py +108 -0
- package/python/biocore_model/emotion_probe.py +209 -0
- package/python/biocore_model/entity_graph.py +931 -0
- package/python/biocore_model/entity_resolve.py +260 -0
- package/python/biocore_model/envy.py +193 -0
- package/python/biocore_model/exit_or_silence.py +230 -0
- package/python/biocore_model/expr_guard.py +24 -0
- package/python/biocore_model/expr_intent.py +77 -0
- package/python/biocore_model/expr_state.py +204 -0
- package/python/biocore_model/expr_template.py +105 -0
- package/python/biocore_model/expr_tone.py +81 -0
- package/python/biocore_model/extreme_scenarios.py +205 -0
- package/python/biocore_model/feedback_flow.py +227 -0
- package/python/biocore_model/feedback_guard.py +129 -0
- package/python/biocore_model/feedback_llm.py +163 -0
- package/python/biocore_model/feedback_memory.py +96 -0
- package/python/biocore_model/feedback_user.py +93 -0
- package/python/biocore_model/frustration_curve.py +110 -0
- package/python/biocore_model/granularity.py +283 -0
- package/python/biocore_model/graph_extract.py +248 -0
- package/python/biocore_model/graph_recall.py +221 -0
- package/python/biocore_model/intent_render.py +91 -0
- package/python/biocore_model/intero_core.py +235 -0
- package/python/biocore_model/intero_style.py +116 -0
- package/python/biocore_model/layered_memory.py +297 -0
- package/python/biocore_model/learning_probe.py +120 -0
- package/python/biocore_model/memory_dual.py +224 -0
- package/python/biocore_model/memory_forgetting.py +125 -0
- package/python/biocore_model/memory_store.py +1399 -0
- package/python/biocore_model/metacog.py +153 -0
- package/python/biocore_model/migrate.py +733 -0
- package/python/biocore_model/migrate_store.py +337 -0
- package/python/biocore_model/mind_wander.py +111 -0
- package/python/biocore_model/mood.py +48 -0
- package/python/biocore_model/narrative.py +172 -0
- package/python/biocore_model/needs_esteem.py +116 -0
- package/python/biocore_model/needs_events.py +137 -0
- package/python/biocore_model/needs_sleep.py +72 -0
- package/python/biocore_model/noise_bank.py +95 -0
- package/python/biocore_model/persona_flow.mjs +25 -0
- package/python/biocore_model/pipeline.py +240 -0
- package/python/biocore_model/pipeline_api.py +200 -0
- package/python/biocore_model/plugin_config.py +83 -0
- package/python/biocore_model/plugin_routes.py +1435 -0
- package/python/biocore_model/policy_learning.py +132 -0
- package/python/biocore_model/possible_selves.py +110 -0
- package/python/biocore_model/predictor.py +178 -0
- package/python/biocore_model/preset.py +205 -0
- package/python/biocore_model/proactive_context.py +153 -0
- package/python/biocore_model/prompt_rules.py +90 -0
- package/python/biocore_model/prospective.py +183 -0
- package/python/biocore_model/provider_fallback.py +97 -0
- package/python/biocore_model/reconstruction.py +132 -0
- package/python/biocore_model/regulation_loop.py +179 -0
- package/python/biocore_model/regulation_strategies.py +294 -0
- package/python/biocore_model/relation_stage.py +130 -0
- package/python/biocore_model/relationship_stage.py +95 -0
- package/python/biocore_model/render.py +901 -0
- package/python/biocore_model/render_guard.mjs +34 -0
- package/python/biocore_model/repair_after_boundary.py +350 -0
- package/python/biocore_model/reply_limit.py +183 -0
- package/python/biocore_model/reply_split.py +203 -0
- package/python/biocore_model/rollback.py +143 -0
- package/python/biocore_model/runtime.py +479 -0
- package/python/biocore_model/sanitize.py +200 -0
- package/python/biocore_model/scheduler.py +123 -0
- package/python/biocore_model/schema.py +120 -0
- package/python/biocore_model/secrecy.py +145 -0
- package/python/biocore_model/self_esteem.py +50 -0
- package/python/biocore_model/self_intent.py +437 -0
- package/python/biocore_model/self_interest.py +263 -0
- package/python/biocore_model/self_verification.py +178 -0
- package/python/biocore_model/services.py +78 -0
- package/python/biocore_model/situation.py +254 -0
- package/python/biocore_model/sleep_circadian.py +116 -0
- package/python/biocore_model/social_pain.py +69 -0
- package/python/biocore_model/social_repair.py +111 -0
- package/python/biocore_model/social_theory.py +131 -0
- package/python/biocore_model/source_monitoring.py +292 -0
- package/python/biocore_model/stages.py +487 -0
- package/python/biocore_model/state_tag.py +231 -0
- package/python/biocore_model/switch_groups.mjs +201 -0
- package/python/biocore_model/tabs_state.mjs +37 -0
- package/python/biocore_model/template_expr.py +252 -0
- package/python/biocore_model/tools/caps_check.mjs +23 -0
- package/python/biocore_model/tools/init_symbols_check.mjs +11 -0
- package/python/biocore_model/tools/missing_ids.mjs +16 -0
- package/python/biocore_model/tools/scope_check.mjs +19 -0
- package/python/biocore_model/topic_control.py +268 -0
- package/python/biocore_model/turn_memory.py +152 -0
- package/python/biocore_model/ui_compat.mjs +48 -0
- package/python/biocore_model/ui_settings.py +266 -0
- package/python/biocore_model/ui_settings.py.bak_authority +108 -0
- package/python/biocore_model/value_order.py +186 -0
- package/python/biocore_model/view_mode.mjs +8 -0
- package/python/biocore_model/voice_bank.py +177 -0
- package/python/biocore_model/waiting_sense.py +229 -0
- package/python/biocore_model/x_directives.py +142 -0
- package/python/biocore_runtime/__init__.py +7 -0
- package/python/biocore_runtime/pipeline.py +1915 -0
- package/python/dsh_adapter/README.md +96 -0
- package/python/dsh_adapter/adapter.py +1077 -0
- package/python/dsh_adapter/cli.py +107 -0
- package/python/dsh_adapter/data/demo_state.json +164 -0
- package/python/dsh_adapter/data/verify_state.json +164 -0
- package/python/dsh_adapter/tests/__init__.py +1 -0
- package/python/dsh_adapter/tests/test_adapter.py +494 -0
- package/ui/panel-layout.js +3 -3
- package/ui/panel.css +53 -6
- package/ui/panel.html +33 -32
- package/ui/panel.js +9 -1
- package/lib/py/__pycache__/bridge.cpython-310.pyc +0 -0
|
@@ -0,0 +1,931 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Full entity graph memory: nodes + typed, timestamped edges (task-134).
|
|
3
|
+
|
|
4
|
+
Storage contract: one JSON document {"schema": 3, "entities": [...],
|
|
5
|
+
"edges": [...]}. Core operations are pure graph transforms; only
|
|
6
|
+
load_graph/save_graph touch the filesystem (atomic replace, corrupt recovery).
|
|
7
|
+
No tag approximations: entities are matched by normalized name or an explicit
|
|
8
|
+
alias, and edges carry rel/ts/confidence/source_entry for time reasoning and
|
|
9
|
+
two-hop retrieval.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import copy
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import sqlite3
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
from . import entity_resolve as _er
|
|
22
|
+
|
|
23
|
+
SCHEMA_VERSION = 3
|
|
24
|
+
ENTITY_TYPES = ("person", "place", "event", "topic")
|
|
25
|
+
DEFAULT_TYPE = "topic"
|
|
26
|
+
MAX_ALIASES = 12
|
|
27
|
+
MAX_SOURCES = 8
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _num(value, default=0.0):
|
|
31
|
+
try:
|
|
32
|
+
number = float(value)
|
|
33
|
+
except Exception:
|
|
34
|
+
return default
|
|
35
|
+
return number if number == number else default
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _norm(text):
|
|
39
|
+
return "".join(ch for ch in str(text or "").lower()
|
|
40
|
+
if ch.isalnum())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _clean_text(text):
|
|
44
|
+
return " ".join(str(text or "").split()).strip()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _id(prefix, *parts):
|
|
48
|
+
raw = "|".join(str(part) for part in parts)
|
|
49
|
+
return prefix + hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def empty_graph():
|
|
53
|
+
return {"schema": SCHEMA_VERSION, "entities": [], "edges": []}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def normalize_graph(raw):
|
|
57
|
+
"""Sanitize a stored document into a valid graph (never raises)."""
|
|
58
|
+
graph = empty_graph()
|
|
59
|
+
if not isinstance(raw, dict):
|
|
60
|
+
return graph
|
|
61
|
+
entities = raw.get("entities")
|
|
62
|
+
if isinstance(entities, list):
|
|
63
|
+
seen = set()
|
|
64
|
+
for item in entities:
|
|
65
|
+
if not isinstance(item, dict):
|
|
66
|
+
continue
|
|
67
|
+
name = _clean_text(item.get("name"))
|
|
68
|
+
norm = _norm(name)
|
|
69
|
+
if not norm or norm in seen:
|
|
70
|
+
continue
|
|
71
|
+
seen.add(norm)
|
|
72
|
+
kind = str(item.get("type") or DEFAULT_TYPE).strip().lower()
|
|
73
|
+
aliases = []
|
|
74
|
+
for alias in (item.get("aliases") or []):
|
|
75
|
+
text = _clean_text(alias)
|
|
76
|
+
if text and _norm(text) != norm and text not in aliases:
|
|
77
|
+
aliases.append(text)
|
|
78
|
+
graph["entities"].append({
|
|
79
|
+
"id": str(item.get("id") or _id("e_", norm)),
|
|
80
|
+
"name": name,
|
|
81
|
+
"type": kind if kind in ENTITY_TYPES else DEFAULT_TYPE,
|
|
82
|
+
"aliases": aliases[:MAX_ALIASES],
|
|
83
|
+
"first_seen": _num(item.get("first_seen"), 0.0),
|
|
84
|
+
"last_seen": _num(item.get("last_seen"), 0.0),
|
|
85
|
+
"sources": [str(x) for x in (item.get("sources") or [])
|
|
86
|
+
if str(x)][:MAX_SOURCES],
|
|
87
|
+
})
|
|
88
|
+
edges = raw.get("edges")
|
|
89
|
+
if isinstance(edges, list):
|
|
90
|
+
known = {row["id"] for row in graph["entities"]}
|
|
91
|
+
seen = set()
|
|
92
|
+
for item in edges:
|
|
93
|
+
if not isinstance(item, dict):
|
|
94
|
+
continue
|
|
95
|
+
src = str(item.get("src") or "").strip()
|
|
96
|
+
dst = str(item.get("dst") or "").strip()
|
|
97
|
+
rel = str(item.get("rel") or "").strip().lower()
|
|
98
|
+
if not src or not dst or not rel or src not in known or dst not in known:
|
|
99
|
+
continue
|
|
100
|
+
ts = _num(item.get("ts"), 0.0)
|
|
101
|
+
edge_id = str(item.get("id") or _id("r_", src, rel, dst, ts))
|
|
102
|
+
if edge_id in seen:
|
|
103
|
+
continue
|
|
104
|
+
seen.add(edge_id)
|
|
105
|
+
graph["edges"].append({
|
|
106
|
+
"id": edge_id,
|
|
107
|
+
"src": src,
|
|
108
|
+
"rel": rel,
|
|
109
|
+
"dst": dst,
|
|
110
|
+
"ts": ts,
|
|
111
|
+
"confidence": max(0.0, min(1.0, _num(item.get("confidence"), 1.0))),
|
|
112
|
+
"source_entry": str(item.get("source_entry") or ""),
|
|
113
|
+
})
|
|
114
|
+
graph["schema"] = SCHEMA_VERSION
|
|
115
|
+
return graph
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _matches(entity, key):
|
|
119
|
+
norm = _norm(key)
|
|
120
|
+
if not norm:
|
|
121
|
+
return False
|
|
122
|
+
return _norm(entity.get("name")) == norm or any(
|
|
123
|
+
_norm(alias) == norm for alias in (entity.get("aliases") or []))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def find_entity(graph, name):
|
|
127
|
+
"""Return the entity matched by name or alias, or None."""
|
|
128
|
+
for entity in (graph or {}).get("entities") or []:
|
|
129
|
+
if isinstance(entity, dict) and _matches(entity, name):
|
|
130
|
+
return entity
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def resolve_name(graph, name):
|
|
135
|
+
entity = find_entity(graph, name)
|
|
136
|
+
if entity is not None:
|
|
137
|
+
return entity["id"]
|
|
138
|
+
key = _clean_text(name)
|
|
139
|
+
return _id("e_", _norm(key)) if key else ""
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def upsert_entity(graph, name, type=None, aliases=None, now=None, entry_id=""):
|
|
143
|
+
"""Add or merge one entity; returns (graph_or_store, entity_id)."""
|
|
144
|
+
if isinstance(graph, GraphStore):
|
|
145
|
+
return graph.upsert_entity(name, type=type, aliases=aliases, now=now,
|
|
146
|
+
entry_id=entry_id)
|
|
147
|
+
out = normalize_graph(copy.deepcopy(graph))
|
|
148
|
+
key = _clean_text(name)
|
|
149
|
+
norm = _norm(key)
|
|
150
|
+
if not norm:
|
|
151
|
+
return out, ""
|
|
152
|
+
stamp = _num(now, 0.0)
|
|
153
|
+
extra_aliases = [_clean_text(x) for x in (aliases or []) if _clean_text(x)]
|
|
154
|
+
entity = find_entity(out, key)
|
|
155
|
+
if entity is None:
|
|
156
|
+
# P0-1: fold an alias / near-identical spelling into the existing node.
|
|
157
|
+
hit = _er.match(out["entities"], key, extra_aliases)
|
|
158
|
+
if hit is not None:
|
|
159
|
+
entity = next((row for row in out["entities"]
|
|
160
|
+
if row.get("id") == hit["id"]), None)
|
|
161
|
+
if (entity is not None
|
|
162
|
+
and _norm(key) != _norm(entity.get("name"))):
|
|
163
|
+
extra_aliases.append(key)
|
|
164
|
+
if entity is None:
|
|
165
|
+
entity = {
|
|
166
|
+
"id": _id("e_", norm),
|
|
167
|
+
"name": key,
|
|
168
|
+
"type": (str(type).strip().lower() if type else DEFAULT_TYPE),
|
|
169
|
+
"aliases": [],
|
|
170
|
+
"first_seen": stamp,
|
|
171
|
+
"last_seen": stamp,
|
|
172
|
+
"sources": [],
|
|
173
|
+
}
|
|
174
|
+
if entity["type"] not in ENTITY_TYPES:
|
|
175
|
+
entity["type"] = DEFAULT_TYPE
|
|
176
|
+
out["entities"].append(entity)
|
|
177
|
+
else:
|
|
178
|
+
if type:
|
|
179
|
+
kind = str(type).strip().lower()
|
|
180
|
+
if kind in ENTITY_TYPES:
|
|
181
|
+
entity["type"] = kind
|
|
182
|
+
entity["last_seen"] = max(_num(entity.get("last_seen"), 0.0), stamp)
|
|
183
|
+
entity["first_seen"] = min(
|
|
184
|
+
_num(entity.get("first_seen"), stamp) or stamp, stamp or 0.0)
|
|
185
|
+
name_norm = _norm(entity.get("name"))
|
|
186
|
+
aliases_now = [text for text in (entity.get("aliases") or [])
|
|
187
|
+
if _norm(text) != name_norm]
|
|
188
|
+
for alias in extra_aliases:
|
|
189
|
+
text = _clean_text(alias)
|
|
190
|
+
if not text or _norm(text) == name_norm or text in aliases_now:
|
|
191
|
+
continue
|
|
192
|
+
aliases_now.append(text)
|
|
193
|
+
entity["aliases"] = aliases_now[:MAX_ALIASES]
|
|
194
|
+
if entry_id:
|
|
195
|
+
sources = list(entity.get("sources") or [])
|
|
196
|
+
if entry_id not in sources:
|
|
197
|
+
sources.append(str(entry_id))
|
|
198
|
+
entity["sources"] = sources[-MAX_SOURCES:]
|
|
199
|
+
return out, entity["id"]
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def upsert_edge(graph, src, rel, dst, ts=None, confidence=1.0, source_entry="",
|
|
203
|
+
src_type=None, dst_type=None, now=None):
|
|
204
|
+
"""Add one typed, timestamped edge (auto-creating entities); returns graph/id."""
|
|
205
|
+
if isinstance(graph, GraphStore):
|
|
206
|
+
return graph.upsert_edge(src, rel, dst, ts=ts, confidence=confidence,
|
|
207
|
+
source_entry=source_entry, src_type=src_type,
|
|
208
|
+
dst_type=dst_type, now=now)
|
|
209
|
+
out = normalize_graph(copy.deepcopy(graph))
|
|
210
|
+
source = _clean_text(src)
|
|
211
|
+
target = _clean_text(dst)
|
|
212
|
+
relation = str(rel or "").strip().lower()
|
|
213
|
+
if not source or not target or not relation:
|
|
214
|
+
return out, ""
|
|
215
|
+
out, src_id = upsert_entity(out, source, type=src_type, now=now,
|
|
216
|
+
entry_id=source_entry)
|
|
217
|
+
out, dst_id = upsert_entity(out, target, type=dst_type, now=now,
|
|
218
|
+
entry_id=source_entry)
|
|
219
|
+
if not src_id or not dst_id or src_id == dst_id:
|
|
220
|
+
# P0-1: both spellings resolved to one node -> a self-loop is noise.
|
|
221
|
+
return out, ""
|
|
222
|
+
stamp = _num(ts, _num(now, 0.0))
|
|
223
|
+
confidence = max(0.0, min(1.0, _num(confidence, 1.0)))
|
|
224
|
+
edge_id = _id("r_", src_id, relation, dst_id, stamp)
|
|
225
|
+
for edge in out["edges"]:
|
|
226
|
+
if edge.get("id") == edge_id:
|
|
227
|
+
edge["confidence"] = max(float(edge.get("confidence", 0.0) or 0.0),
|
|
228
|
+
confidence)
|
|
229
|
+
if source_entry:
|
|
230
|
+
edge["source_entry"] = str(source_entry)
|
|
231
|
+
return out, edge_id
|
|
232
|
+
out["edges"].append({
|
|
233
|
+
"id": edge_id,
|
|
234
|
+
"src": src_id,
|
|
235
|
+
"rel": relation,
|
|
236
|
+
"dst": dst_id,
|
|
237
|
+
"ts": stamp,
|
|
238
|
+
"confidence": confidence,
|
|
239
|
+
"source_entry": str(source_entry or ""),
|
|
240
|
+
})
|
|
241
|
+
return out, edge_id
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _merge_report(keep_id, drop_id):
|
|
245
|
+
return {"ok": False, "keep_id": str(keep_id or ""),
|
|
246
|
+
"drop_id": str(drop_id or ""), "aliases": 0, "sources": 0,
|
|
247
|
+
"edges_rewired": 0, "edges_dropped": 0, "error": ""}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def find_similar(graph, name, aliases=(), threshold=None):
|
|
251
|
+
"""Best {"id","name","score","via"} for a name/alias, or None (P0-1)."""
|
|
252
|
+
if isinstance(graph, GraphStore):
|
|
253
|
+
return graph.find_similar(name, aliases=aliases, threshold=threshold)
|
|
254
|
+
clean = normalize_graph(copy.deepcopy(graph))
|
|
255
|
+
limit = _er.SIMILARITY_THRESHOLD if threshold is None else threshold
|
|
256
|
+
return _er.match(clean["entities"], name, aliases, limit)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def merge_entities(graph, keep_id, drop_id):
|
|
260
|
+
"""Merge drop_id into keep_id; returns (graph_or_store, report).
|
|
261
|
+
|
|
262
|
+
Fields are unioned (aliases/sources/first/last/type) and every edge that
|
|
263
|
+
pointed at the dropped node is rewired to the kept one. Edges that become a
|
|
264
|
+
self-loop are dropped; edges that collapse onto an existing row are folded.
|
|
265
|
+
"""
|
|
266
|
+
if isinstance(graph, GraphStore):
|
|
267
|
+
return graph, graph.merge_entities(keep_id, drop_id)
|
|
268
|
+
out = normalize_graph(copy.deepcopy(graph))
|
|
269
|
+
report = _merge_report(keep_id, drop_id)
|
|
270
|
+
if not report["keep_id"] or not report["drop_id"]:
|
|
271
|
+
report["error"] = "keep_id/drop_id required"
|
|
272
|
+
return out, report
|
|
273
|
+
if report["keep_id"] == report["drop_id"]:
|
|
274
|
+
report["error"] = "same entity"
|
|
275
|
+
return out, report
|
|
276
|
+
keep = next((row for row in out["entities"]
|
|
277
|
+
if row["id"] == report["keep_id"]), None)
|
|
278
|
+
drop = next((row for row in out["entities"]
|
|
279
|
+
if row["id"] == report["drop_id"]), None)
|
|
280
|
+
if keep is None or drop is None:
|
|
281
|
+
report["error"] = "entity not found"
|
|
282
|
+
return out, report
|
|
283
|
+
keep["aliases"] = _er.merged_aliases(
|
|
284
|
+
keep["name"], keep["aliases"], drop["name"], drop["aliases"])
|
|
285
|
+
keep["sources"] = _er.merged_sources(keep["sources"], drop["sources"])
|
|
286
|
+
keep["type"] = _er.merged_type(keep["type"], drop["type"])
|
|
287
|
+
keep["first_seen"], keep["last_seen"] = _er.merged_span(
|
|
288
|
+
keep.get("first_seen"), keep.get("last_seen"),
|
|
289
|
+
drop.get("first_seen"), drop.get("last_seen"))
|
|
290
|
+
out["entities"] = [row for row in out["entities"]
|
|
291
|
+
if row["id"] != report["drop_id"]]
|
|
292
|
+
out["edges"], stats = _er.rewrite_edges(
|
|
293
|
+
out["edges"], report["keep_id"], report["drop_id"],
|
|
294
|
+
make_id=lambda src, rel, dst, ts: _id("r_", src, rel, dst, ts))
|
|
295
|
+
report.update({"ok": True, "aliases": len(keep["aliases"]),
|
|
296
|
+
"sources": len(keep["sources"]),
|
|
297
|
+
"edges_rewired": stats["rewired"],
|
|
298
|
+
"edges_dropped": stats["dropped"]})
|
|
299
|
+
return out, report
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _window_start(window_days, now):
|
|
303
|
+
if window_days is None:
|
|
304
|
+
return None
|
|
305
|
+
days = _num(window_days, 0.0)
|
|
306
|
+
if days <= 0:
|
|
307
|
+
return None
|
|
308
|
+
base = _num(now, 0.0)
|
|
309
|
+
if base <= 0:
|
|
310
|
+
latest = 0.0
|
|
311
|
+
return None
|
|
312
|
+
return base - days * 86400.0
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def neighbors(graph, entity, hops=2, window_days=None, now=None, limit=80):
|
|
316
|
+
"""Neighbours around an entity/alias (dict = in-memory, GraphStore = CTE)."""
|
|
317
|
+
if isinstance(graph, GraphStore):
|
|
318
|
+
return graph.neighbors(entity, hops=hops, window_days=window_days,
|
|
319
|
+
now=now, limit=limit)
|
|
320
|
+
clean = normalize_graph(graph)
|
|
321
|
+
start = find_entity(clean, entity)
|
|
322
|
+
result = {"entities": [], "edges": []}
|
|
323
|
+
if start is None:
|
|
324
|
+
return result
|
|
325
|
+
try:
|
|
326
|
+
depth_limit = max(1, min(4, int(hops)))
|
|
327
|
+
except Exception:
|
|
328
|
+
depth_limit = 2
|
|
329
|
+
try:
|
|
330
|
+
row_limit = max(1, min(500, int(limit)))
|
|
331
|
+
except Exception:
|
|
332
|
+
row_limit = 80
|
|
333
|
+
start_ts = _window_start(window_days, now)
|
|
334
|
+
by_id = {row["id"]: row for row in clean["entities"]}
|
|
335
|
+
visited = {start["id"]}
|
|
336
|
+
frontier = [start["id"]]
|
|
337
|
+
used_edge_ids = set()
|
|
338
|
+
for _depth in range(depth_limit):
|
|
339
|
+
next_frontier = []
|
|
340
|
+
for node_id in frontier:
|
|
341
|
+
for edge in clean["edges"]:
|
|
342
|
+
if edge["src"] != node_id and edge["dst"] != node_id:
|
|
343
|
+
continue
|
|
344
|
+
if start_ts is not None and _num(edge.get("ts"), 0.0) < start_ts:
|
|
345
|
+
continue
|
|
346
|
+
other = edge["dst"] if edge["src"] == node_id else edge["src"]
|
|
347
|
+
if other not in visited:
|
|
348
|
+
visited.add(other)
|
|
349
|
+
next_frontier.append(other)
|
|
350
|
+
if edge["id"] not in used_edge_ids:
|
|
351
|
+
used_edge_ids.add(edge["id"])
|
|
352
|
+
result["edges"].append(dict(edge))
|
|
353
|
+
frontier = next_frontier
|
|
354
|
+
if not frontier:
|
|
355
|
+
break
|
|
356
|
+
result["entities"] = [dict(by_id[eid]) for eid in visited
|
|
357
|
+
if eid in by_id][:row_limit]
|
|
358
|
+
result["edges"].sort(key=lambda row: (-_num(row.get("ts"), 0.0),
|
|
359
|
+
-_num(row.get("confidence"), 0.0)))
|
|
360
|
+
result["edges"] = result["edges"][:row_limit]
|
|
361
|
+
return result
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def search(graph, text, limit=8):
|
|
365
|
+
"""Entities whose name or alias appears in *text* (longest match first)."""
|
|
366
|
+
if isinstance(graph, GraphStore):
|
|
367
|
+
return graph.search(text, limit=limit)
|
|
368
|
+
clean = normalize_graph(graph)
|
|
369
|
+
haystack = _norm(text)
|
|
370
|
+
hits = []
|
|
371
|
+
if not haystack:
|
|
372
|
+
return hits
|
|
373
|
+
for entity in clean["entities"]:
|
|
374
|
+
names = [entity.get("name")] + list(entity.get("aliases") or [])
|
|
375
|
+
best = 0
|
|
376
|
+
for name in names:
|
|
377
|
+
norm = _norm(name)
|
|
378
|
+
if norm and norm in haystack:
|
|
379
|
+
best = max(best, len(norm))
|
|
380
|
+
if best:
|
|
381
|
+
hits.append((best, entity))
|
|
382
|
+
hits.sort(key=lambda row: row[0], reverse=True)
|
|
383
|
+
return [dict(entity) for _score, entity in hits[:max(1, int(limit or 8))]]
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def stats(graph):
|
|
387
|
+
"""Bounded counts used by the panel and health checks."""
|
|
388
|
+
if isinstance(graph, GraphStore):
|
|
389
|
+
return graph.stats()
|
|
390
|
+
clean = normalize_graph(graph)
|
|
391
|
+
types = {}
|
|
392
|
+
aliases = 0
|
|
393
|
+
for entity in clean["entities"]:
|
|
394
|
+
kind = str(entity.get("type") or DEFAULT_TYPE)
|
|
395
|
+
types[kind] = int(types.get(kind, 0)) + 1
|
|
396
|
+
aliases += len(entity.get("aliases") or [])
|
|
397
|
+
latest = max([_num(edge.get("ts"), 0.0) for edge in clean["edges"]] or [0.0])
|
|
398
|
+
return {"schema": clean["schema"],
|
|
399
|
+
"entities": len(clean["entities"]),
|
|
400
|
+
"edges": len(clean["edges"]),
|
|
401
|
+
"types": types,
|
|
402
|
+
"aliases": aliases,
|
|
403
|
+
"latest_ts": latest}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def load_graph(path):
|
|
407
|
+
"""Read a graph document; corrupt files are backed up and recovered empty."""
|
|
408
|
+
target = Path(path)
|
|
409
|
+
if not target.is_file():
|
|
410
|
+
return empty_graph()
|
|
411
|
+
try:
|
|
412
|
+
raw = json.loads(target.read_text(encoding="utf-8-sig"))
|
|
413
|
+
return normalize_graph(raw)
|
|
414
|
+
except Exception:
|
|
415
|
+
try:
|
|
416
|
+
shutil.copyfile(target, target.with_name(target.name + ".corrupt.bak"))
|
|
417
|
+
except Exception:
|
|
418
|
+
pass
|
|
419
|
+
return empty_graph()
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def save_graph(path, graph):
|
|
423
|
+
"""Atomic write of the graph document (tmp + os.replace)."""
|
|
424
|
+
target = Path(path)
|
|
425
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
426
|
+
payload = normalize_graph(graph)
|
|
427
|
+
tmp = target.with_name(target.name + ".tmp")
|
|
428
|
+
try:
|
|
429
|
+
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2),
|
|
430
|
+
encoding="utf-8")
|
|
431
|
+
os.replace(tmp, target)
|
|
432
|
+
except Exception:
|
|
433
|
+
try:
|
|
434
|
+
tmp.unlink(missing_ok=True)
|
|
435
|
+
except Exception:
|
|
436
|
+
pass
|
|
437
|
+
raise
|
|
438
|
+
return target
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
# ---------------------------------------------------------------------------
|
|
443
|
+
# SQLite storage layer (task-134 addendum). The public functions above accept
|
|
444
|
+
# a GraphStore and delegate to it; the JSON helpers below stay the
|
|
445
|
+
# export/backup format only.
|
|
446
|
+
# ---------------------------------------------------------------------------
|
|
447
|
+
|
|
448
|
+
SQLITE_FILE = "graph.sqlite3"
|
|
449
|
+
_SCHEMA_SQL = """
|
|
450
|
+
CREATE TABLE IF NOT EXISTS entities(
|
|
451
|
+
id TEXT PRIMARY KEY, name TEXT, type TEXT, aliases TEXT,
|
|
452
|
+
first_seen REAL, last_seen REAL, sources TEXT);
|
|
453
|
+
CREATE TABLE IF NOT EXISTS edges(
|
|
454
|
+
id TEXT PRIMARY KEY, src TEXT, rel TEXT, dst TEXT, ts REAL,
|
|
455
|
+
confidence REAL, source_entry TEXT);
|
|
456
|
+
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
|
|
457
|
+
CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src);
|
|
458
|
+
CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst);
|
|
459
|
+
CREATE INDEX IF NOT EXISTS idx_edges_ts ON edges(ts);
|
|
460
|
+
CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(rel);
|
|
461
|
+
"""
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _row_entity(row):
|
|
465
|
+
def _json_list(raw):
|
|
466
|
+
try:
|
|
467
|
+
value = json.loads(raw or "[]")
|
|
468
|
+
except Exception:
|
|
469
|
+
return []
|
|
470
|
+
return [str(x) for x in value if str(x)] if isinstance(value, list) else []
|
|
471
|
+
|
|
472
|
+
return {"id": row[0], "name": row[1], "type": row[2] or DEFAULT_TYPE,
|
|
473
|
+
"aliases": _json_list(row[3]),
|
|
474
|
+
"first_seen": _num(row[4], 0.0), "last_seen": _num(row[5], 0.0),
|
|
475
|
+
"sources": _json_list(row[6])}
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _row_edge(row):
|
|
479
|
+
return {"id": row[0], "src": row[1], "rel": row[2], "dst": row[3],
|
|
480
|
+
"ts": _num(row[4], 0.0),
|
|
481
|
+
"confidence": _num(row[5], 1.0),
|
|
482
|
+
"source_entry": row[6] or ""}
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
_ENTITY_COLS = "id, name, type, aliases, first_seen, last_seen, sources"
|
|
486
|
+
_EDGE_COLS = "id, src, rel, dst, ts, confidence, source_entry"
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
class GraphStore:
|
|
490
|
+
"""Thin SQLite wrapper. One connection, WAL, explicit transactions."""
|
|
491
|
+
|
|
492
|
+
def __init__(self, path, recover=True, similarity_threshold=None):
|
|
493
|
+
self.path = Path(path)
|
|
494
|
+
self._conn = None
|
|
495
|
+
# P0-1: <=0 disables the similarity fallback (exact name/alias only).
|
|
496
|
+
self.similarity_threshold = (
|
|
497
|
+
_er.SIMILARITY_THRESHOLD if similarity_threshold is None
|
|
498
|
+
else float(similarity_threshold))
|
|
499
|
+
self._open(recover=recover)
|
|
500
|
+
|
|
501
|
+
# -- connection / schema ------------------------------------------------
|
|
502
|
+
def _open(self, recover=True):
|
|
503
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
504
|
+
try:
|
|
505
|
+
self._conn = sqlite3.connect(str(self.path), timeout=5.0)
|
|
506
|
+
self._configure()
|
|
507
|
+
self._ensure_schema()
|
|
508
|
+
except sqlite3.DatabaseError:
|
|
509
|
+
if not recover:
|
|
510
|
+
raise
|
|
511
|
+
try:
|
|
512
|
+
self._conn.close()
|
|
513
|
+
except Exception:
|
|
514
|
+
pass
|
|
515
|
+
self._conn = None
|
|
516
|
+
self._recover_corrupt()
|
|
517
|
+
self._conn = sqlite3.connect(str(self.path), timeout=5.0)
|
|
518
|
+
self._configure()
|
|
519
|
+
self._ensure_schema()
|
|
520
|
+
|
|
521
|
+
def _configure(self):
|
|
522
|
+
cur = self._conn.cursor()
|
|
523
|
+
cur.execute("PRAGMA journal_mode=WAL")
|
|
524
|
+
cur.execute("PRAGMA synchronous=NORMAL")
|
|
525
|
+
cur.execute("PRAGMA busy_timeout=5000")
|
|
526
|
+
|
|
527
|
+
def _ensure_schema(self):
|
|
528
|
+
with self._conn:
|
|
529
|
+
self._conn.executescript(_SCHEMA_SQL)
|
|
530
|
+
|
|
531
|
+
def _recover_corrupt(self):
|
|
532
|
+
try:
|
|
533
|
+
if self.path.is_file():
|
|
534
|
+
shutil.copyfile(self.path,
|
|
535
|
+
self.path.with_name(self.path.name + ".corrupt.bak"))
|
|
536
|
+
except Exception:
|
|
537
|
+
pass
|
|
538
|
+
for suffix in ("", "-wal", "-shm"):
|
|
539
|
+
try:
|
|
540
|
+
Path(str(self.path) + suffix).unlink(missing_ok=True)
|
|
541
|
+
except Exception:
|
|
542
|
+
pass
|
|
543
|
+
|
|
544
|
+
def close(self):
|
|
545
|
+
if self._conn is not None:
|
|
546
|
+
try:
|
|
547
|
+
self._conn.close()
|
|
548
|
+
except Exception:
|
|
549
|
+
pass
|
|
550
|
+
self._conn = None
|
|
551
|
+
|
|
552
|
+
def __enter__(self):
|
|
553
|
+
return self
|
|
554
|
+
|
|
555
|
+
def __exit__(self, *args):
|
|
556
|
+
self.close()
|
|
557
|
+
|
|
558
|
+
def checkpoint(self):
|
|
559
|
+
"""Fold the WAL back into the main file before copying it."""
|
|
560
|
+
try:
|
|
561
|
+
self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
562
|
+
return True
|
|
563
|
+
except Exception:
|
|
564
|
+
return False
|
|
565
|
+
|
|
566
|
+
# -- entity / edge writes ----------------------------------------------
|
|
567
|
+
def _find_entity(self, name):
|
|
568
|
+
norm = _norm(name)
|
|
569
|
+
if not norm:
|
|
570
|
+
return None
|
|
571
|
+
cur = self._conn.execute(
|
|
572
|
+
"SELECT " + _ENTITY_COLS + " FROM entities "
|
|
573
|
+
"WHERE lower(replace(name,' ',''))=? OR aliases LIKE ? LIMIT 1",
|
|
574
|
+
(norm, "%" + str(name) + "%"))
|
|
575
|
+
row = cur.fetchone()
|
|
576
|
+
return _row_entity(row) if row else None
|
|
577
|
+
|
|
578
|
+
def _row_by_id(self, entity_id):
|
|
579
|
+
row = self._conn.execute(
|
|
580
|
+
"SELECT " + _ENTITY_COLS + " FROM entities WHERE id=?",
|
|
581
|
+
(str(entity_id or ""),)).fetchone()
|
|
582
|
+
return _row_entity(row) if row else None
|
|
583
|
+
|
|
584
|
+
def find_similar(self, name, aliases=(), threshold=None):
|
|
585
|
+
"""Best {"id","name","score","via"} match, bounded scan (P0-1)."""
|
|
586
|
+
limit = (self.similarity_threshold if threshold is None
|
|
587
|
+
else float(threshold))
|
|
588
|
+
if not limit or limit <= 0:
|
|
589
|
+
return None
|
|
590
|
+
rows = [_row_entity(row) for row in self._conn.execute(
|
|
591
|
+
"SELECT " + _ENTITY_COLS + " FROM entities "
|
|
592
|
+
"ORDER BY last_seen DESC LIMIT ?", (_er.SCAN_LIMIT,)).fetchall()]
|
|
593
|
+
return _er.match(rows, name, aliases, limit)
|
|
594
|
+
|
|
595
|
+
def upsert_entity(self, name, type=None, aliases=None, now=None, entry_id=""):
|
|
596
|
+
key = _clean_text(name)
|
|
597
|
+
norm = _norm(key)
|
|
598
|
+
if not norm:
|
|
599
|
+
return self, ""
|
|
600
|
+
stamp = _num(now, 0.0)
|
|
601
|
+
with self._conn:
|
|
602
|
+
entity = self._find_entity(key)
|
|
603
|
+
if entity is None:
|
|
604
|
+
# P0-1: fold a near-identical spelling into the existing node.
|
|
605
|
+
hit = self.find_similar(key, aliases or ())
|
|
606
|
+
if hit is not None:
|
|
607
|
+
entity = self._row_by_id(hit["id"])
|
|
608
|
+
entity_name = _clean_text(entity.get("name")) if entity else key
|
|
609
|
+
name_norm = _norm(entity_name)
|
|
610
|
+
if entity is not None and _norm(key) != name_norm:
|
|
611
|
+
# Keep the spelling this message used as an alias.
|
|
612
|
+
aliases = list(aliases or []) + [key]
|
|
613
|
+
aliases_now = [text for text in
|
|
614
|
+
(_clean_text(alias) for alias in
|
|
615
|
+
(entity.get("aliases") if entity else []))
|
|
616
|
+
if text and _norm(text) != name_norm]
|
|
617
|
+
for alias in (aliases or []):
|
|
618
|
+
text = _clean_text(alias)
|
|
619
|
+
if text and _norm(text) != name_norm and text not in aliases_now:
|
|
620
|
+
aliases_now.append(text)
|
|
621
|
+
aliases_now = aliases_now[:MAX_ALIASES]
|
|
622
|
+
sources = list(entity.get("sources") or []) if entity else []
|
|
623
|
+
if entry_id and str(entry_id) not in sources:
|
|
624
|
+
sources.append(str(entry_id))
|
|
625
|
+
if entity is None:
|
|
626
|
+
eid = _id("e_", norm)
|
|
627
|
+
kind = str(type).strip().lower() if type else DEFAULT_TYPE
|
|
628
|
+
if kind not in ENTITY_TYPES:
|
|
629
|
+
kind = DEFAULT_TYPE
|
|
630
|
+
self._conn.execute(
|
|
631
|
+
"INSERT OR REPLACE INTO entities(" + _ENTITY_COLS + ") "
|
|
632
|
+
"VALUES(?,?,?,?,?,?,?)",
|
|
633
|
+
(eid, key, kind, json.dumps(aliases_now, ensure_ascii=False),
|
|
634
|
+
stamp, stamp, json.dumps(sources, ensure_ascii=False)))
|
|
635
|
+
return self, eid
|
|
636
|
+
eid = str(entity["id"])
|
|
637
|
+
kind = str(entity.get("type") or DEFAULT_TYPE)
|
|
638
|
+
if type:
|
|
639
|
+
candidate = str(type).strip().lower()
|
|
640
|
+
if candidate in ENTITY_TYPES:
|
|
641
|
+
kind = candidate
|
|
642
|
+
first_seen = _num(entity.get("first_seen"), stamp)
|
|
643
|
+
if not first_seen:
|
|
644
|
+
first_seen = stamp
|
|
645
|
+
self._conn.execute(
|
|
646
|
+
"UPDATE entities SET name=?, type=?, aliases=?, first_seen=?, "
|
|
647
|
+
"last_seen=?, sources=? WHERE id=?",
|
|
648
|
+
(entity.get("name") or key, kind,
|
|
649
|
+
json.dumps(aliases_now, ensure_ascii=False),
|
|
650
|
+
min(first_seen, stamp or first_seen),
|
|
651
|
+
max(_num(entity.get("last_seen"), 0.0), stamp),
|
|
652
|
+
json.dumps(sources, ensure_ascii=False), eid))
|
|
653
|
+
return self, eid
|
|
654
|
+
|
|
655
|
+
def upsert_edge(self, src, rel, dst, ts=None, confidence=1.0,
|
|
656
|
+
source_entry="", src_type=None, dst_type=None, now=None):
|
|
657
|
+
source = _clean_text(src)
|
|
658
|
+
target = _clean_text(dst)
|
|
659
|
+
relation = str(rel or "").strip().lower()
|
|
660
|
+
if not source or not target or not relation:
|
|
661
|
+
return self, ""
|
|
662
|
+
_, src_id = self.upsert_entity(source, type=src_type, now=now,
|
|
663
|
+
entry_id=source_entry)
|
|
664
|
+
_, dst_id = self.upsert_entity(target, type=dst_type, now=now,
|
|
665
|
+
entry_id=source_entry)
|
|
666
|
+
if not src_id or not dst_id or src_id == dst_id:
|
|
667
|
+
# P0-1: both spellings resolved to one node -> a self-loop is noise.
|
|
668
|
+
return self, ""
|
|
669
|
+
stamp = _num(ts, _num(now, 0.0))
|
|
670
|
+
score = max(0.0, min(1.0, _num(confidence, 1.0)))
|
|
671
|
+
edge_id = _id("r_", src_id, relation, dst_id, stamp)
|
|
672
|
+
with self._conn:
|
|
673
|
+
self._conn.execute(
|
|
674
|
+
"INSERT OR IGNORE INTO edges(" + _EDGE_COLS + ") "
|
|
675
|
+
"VALUES(?,?,?,?,?,?,?)",
|
|
676
|
+
(edge_id, src_id, relation, dst_id, stamp, score,
|
|
677
|
+
str(source_entry or "")))
|
|
678
|
+
self._conn.execute(
|
|
679
|
+
"UPDATE edges SET confidence=MAX(confidence,?), "
|
|
680
|
+
"source_entry=CASE WHEN ?<>'' THEN ? ELSE source_entry END "
|
|
681
|
+
"WHERE id=?",
|
|
682
|
+
(score, str(source_entry or ""), str(source_entry or ""), edge_id))
|
|
683
|
+
return self, edge_id
|
|
684
|
+
|
|
685
|
+
def merge_entities(self, keep_id, drop_id):
|
|
686
|
+
"""Merge one entity into another (aliases/sources/time + edge rewire)."""
|
|
687
|
+
report = _merge_report(keep_id, drop_id)
|
|
688
|
+
if not report["keep_id"] or not report["drop_id"]:
|
|
689
|
+
report["error"] = "keep_id/drop_id required"
|
|
690
|
+
return report
|
|
691
|
+
if report["keep_id"] == report["drop_id"]:
|
|
692
|
+
report["error"] = "same entity"
|
|
693
|
+
return report
|
|
694
|
+
keep = self._row_by_id(report["keep_id"])
|
|
695
|
+
drop = self._row_by_id(report["drop_id"])
|
|
696
|
+
if keep is None or drop is None:
|
|
697
|
+
report["error"] = "entity not found"
|
|
698
|
+
return report
|
|
699
|
+
aliases = _er.merged_aliases(keep["name"], keep["aliases"],
|
|
700
|
+
drop["name"], drop["aliases"])
|
|
701
|
+
sources = _er.merged_sources(keep["sources"], drop["sources"])
|
|
702
|
+
kind = _er.merged_type(keep["type"], drop["type"])
|
|
703
|
+
first_seen, last_seen = _er.merged_span(
|
|
704
|
+
keep.get("first_seen"), keep.get("last_seen"),
|
|
705
|
+
drop.get("first_seen"), drop.get("last_seen"))
|
|
706
|
+
with self._conn:
|
|
707
|
+
rewired, dropped = self._rewire_edges(keep["id"], drop["id"])
|
|
708
|
+
self._conn.execute(
|
|
709
|
+
"UPDATE entities SET aliases=?, type=?, first_seen=?, "
|
|
710
|
+
"last_seen=?, sources=? WHERE id=?",
|
|
711
|
+
(json.dumps(aliases, ensure_ascii=False), kind, first_seen,
|
|
712
|
+
last_seen, json.dumps(sources, ensure_ascii=False),
|
|
713
|
+
keep["id"]))
|
|
714
|
+
self._conn.execute("DELETE FROM entities WHERE id=?", (drop["id"],))
|
|
715
|
+
report.update({"ok": True, "aliases": len(aliases),
|
|
716
|
+
"sources": len(sources), "edges_rewired": rewired,
|
|
717
|
+
"edges_dropped": dropped})
|
|
718
|
+
return report
|
|
719
|
+
|
|
720
|
+
def _rewire_edges(self, keep_id, drop_id):
|
|
721
|
+
"""Move drop_id edge endpoints onto keep_id; returns (rewired, dropped)."""
|
|
722
|
+
rows = self._conn.execute(
|
|
723
|
+
"SELECT " + _EDGE_COLS + " FROM edges WHERE src=? OR dst=?",
|
|
724
|
+
(drop_id, drop_id)).fetchall()
|
|
725
|
+
rewired = 0
|
|
726
|
+
dropped = 0
|
|
727
|
+
for row in rows:
|
|
728
|
+
edge = _row_edge(row)
|
|
729
|
+
src = keep_id if edge["src"] == drop_id else edge["src"]
|
|
730
|
+
dst = keep_id if edge["dst"] == drop_id else edge["dst"]
|
|
731
|
+
changed = src != edge["src"] or dst != edge["dst"]
|
|
732
|
+
if changed and src == dst:
|
|
733
|
+
self._conn.execute("DELETE FROM edges WHERE id=?",
|
|
734
|
+
(edge["id"],))
|
|
735
|
+
dropped += 1
|
|
736
|
+
continue
|
|
737
|
+
if not changed:
|
|
738
|
+
continue
|
|
739
|
+
rewired += 1
|
|
740
|
+
new_id = _id("r_", src, edge["rel"], dst, edge["ts"])
|
|
741
|
+
existing = self._conn.execute(
|
|
742
|
+
"SELECT id FROM edges WHERE id=?", (new_id,)).fetchone()
|
|
743
|
+
if existing is None:
|
|
744
|
+
self._conn.execute(
|
|
745
|
+
"INSERT INTO edges(" + _EDGE_COLS + ") "
|
|
746
|
+
"VALUES(?,?,?,?,?,?,?)",
|
|
747
|
+
(new_id, src, edge["rel"], dst, edge["ts"],
|
|
748
|
+
edge["confidence"], edge["source_entry"]))
|
|
749
|
+
else:
|
|
750
|
+
self._conn.execute(
|
|
751
|
+
"UPDATE edges SET confidence=MAX(confidence,?), "
|
|
752
|
+
"source_entry=CASE WHEN ?<>'' THEN ? ELSE source_entry END "
|
|
753
|
+
"WHERE id=?",
|
|
754
|
+
(edge["confidence"], edge["source_entry"],
|
|
755
|
+
edge["source_entry"], new_id))
|
|
756
|
+
dropped += 1
|
|
757
|
+
self._conn.execute("DELETE FROM edges WHERE id=?", (edge["id"],))
|
|
758
|
+
return rewired, dropped
|
|
759
|
+
|
|
760
|
+
# -- queries ------------------------------------------------------------
|
|
761
|
+
def neighbors(self, entity, hops=2, window_days=None, now=None, limit=80):
|
|
762
|
+
start = self._find_entity(entity)
|
|
763
|
+
if start is None:
|
|
764
|
+
return {"entities": [], "edges": []}
|
|
765
|
+
try:
|
|
766
|
+
depth_limit = max(1, min(4, int(hops)))
|
|
767
|
+
except Exception:
|
|
768
|
+
depth_limit = 2
|
|
769
|
+
try:
|
|
770
|
+
row_limit = max(1, min(500, int(limit)))
|
|
771
|
+
except Exception:
|
|
772
|
+
row_limit = 80
|
|
773
|
+
window = _window_start(window_days, now)
|
|
774
|
+
params = [str(start["id"]), depth_limit]
|
|
775
|
+
window_sql = "? IS NULL OR e.ts >= ?"
|
|
776
|
+
params_sql = (None, window) if window is None else (1, window)
|
|
777
|
+
sql = ("WITH RECURSIVE reach(id, depth) AS ("
|
|
778
|
+
"SELECT ?, 0 "
|
|
779
|
+
"UNION "
|
|
780
|
+
"SELECT CASE WHEN e.src = reach.id THEN e.dst ELSE e.src END,"
|
|
781
|
+
" reach.depth + 1 "
|
|
782
|
+
"FROM edges e JOIN reach "
|
|
783
|
+
"ON (e.src = reach.id OR e.dst = reach.id) "
|
|
784
|
+
"WHERE reach.depth < ? AND (" + window_sql + ")) "
|
|
785
|
+
"SELECT id FROM reach")
|
|
786
|
+
rows = self._conn.execute(sql, params + list(params_sql)).fetchall()
|
|
787
|
+
ids = [str(row[0]) for row in rows]
|
|
788
|
+
if not ids:
|
|
789
|
+
return {"entities": [], "edges": []}
|
|
790
|
+
qmarks = ",".join("?" for _ in ids)
|
|
791
|
+
entities = [_row_entity(row) for row in self._conn.execute(
|
|
792
|
+
"SELECT " + _ENTITY_COLS + " FROM entities WHERE id IN (" + qmarks + ") LIMIT ?",
|
|
793
|
+
ids + [row_limit]).fetchall()]
|
|
794
|
+
edge_rows = self._conn.execute(
|
|
795
|
+
"SELECT " + _EDGE_COLS + " FROM edges "
|
|
796
|
+
"WHERE src IN (" + qmarks + ") AND dst IN (" + qmarks + ") "
|
|
797
|
+
"AND (? IS NULL OR ts >= ?) "
|
|
798
|
+
"ORDER BY ts DESC, confidence DESC LIMIT ?",
|
|
799
|
+
ids + ids + [window, window, row_limit]).fetchall()
|
|
800
|
+
return {"entities": entities,
|
|
801
|
+
"edges": [_row_edge(row) for row in edge_rows]}
|
|
802
|
+
|
|
803
|
+
def search(self, text, limit=8):
|
|
804
|
+
haystack = _clean_text(text)
|
|
805
|
+
if not haystack:
|
|
806
|
+
return []
|
|
807
|
+
try:
|
|
808
|
+
cap = max(1, int(limit or 8))
|
|
809
|
+
except Exception:
|
|
810
|
+
cap = 8
|
|
811
|
+
try:
|
|
812
|
+
rows = self._conn.execute(
|
|
813
|
+
"SELECT " + _ENTITY_COLS + ", length(name) AS ln FROM entities "
|
|
814
|
+
"WHERE ? LIKE '%' || name || '%' OR EXISTS("
|
|
815
|
+
"SELECT 1 FROM json_each(entities.aliases) "
|
|
816
|
+
"WHERE ? LIKE '%' || json_each.value || '%') "
|
|
817
|
+
"ORDER BY ln DESC LIMIT ?", (haystack, haystack, cap)).fetchall()
|
|
818
|
+
except sqlite3.DatabaseError:
|
|
819
|
+
rows = []
|
|
820
|
+
if rows:
|
|
821
|
+
return [_row_entity(row[:7]) for row in rows]
|
|
822
|
+
# Fallback for SQLite builds without JSON1: bounded LIKE scan.
|
|
823
|
+
out = []
|
|
824
|
+
for row in self._conn.execute(
|
|
825
|
+
"SELECT " + _ENTITY_COLS + " FROM entities LIMIT 500").fetchall():
|
|
826
|
+
entity = _row_entity(row)
|
|
827
|
+
for name in [entity["name"]] + list(entity["aliases"]):
|
|
828
|
+
if _norm(name) and _norm(name) in _norm(haystack):
|
|
829
|
+
out.append((len(_norm(name)), entity))
|
|
830
|
+
break
|
|
831
|
+
out.sort(key=lambda item: item[0], reverse=True)
|
|
832
|
+
return [entity for _length, entity in out[:cap]]
|
|
833
|
+
|
|
834
|
+
def stats(self):
|
|
835
|
+
entities = int(self._conn.execute(
|
|
836
|
+
"SELECT COUNT(*) FROM entities").fetchone()[0] or 0)
|
|
837
|
+
edges = int(self._conn.execute(
|
|
838
|
+
"SELECT COUNT(*) FROM edges").fetchone()[0] or 0)
|
|
839
|
+
types = {str(kind): int(count) for kind, count in self._conn.execute(
|
|
840
|
+
"SELECT type, COUNT(*) FROM entities GROUP BY type").fetchall()}
|
|
841
|
+
try:
|
|
842
|
+
aliases = int(self._conn.execute(
|
|
843
|
+
"SELECT COALESCE(SUM(json_array_length(aliases)),0) FROM entities"
|
|
844
|
+
).fetchone()[0] or 0)
|
|
845
|
+
except sqlite3.DatabaseError:
|
|
846
|
+
aliases = 0
|
|
847
|
+
latest = _num(self._conn.execute(
|
|
848
|
+
"SELECT COALESCE(MAX(ts),0) FROM edges").fetchone()[0], 0.0)
|
|
849
|
+
return {"schema": SCHEMA_VERSION, "entities": entities, "edges": edges,
|
|
850
|
+
"types": types, "aliases": aliases, "latest_ts": latest}
|
|
851
|
+
|
|
852
|
+
# -- JSON export / import ----------------------------------------------
|
|
853
|
+
def export_json(self, path=None):
|
|
854
|
+
target = Path(path) if path else self.path.with_name("graph.json")
|
|
855
|
+
graph = {"schema": SCHEMA_VERSION,
|
|
856
|
+
"entities": [_row_entity(row) for row in self._conn.execute(
|
|
857
|
+
"SELECT " + _ENTITY_COLS + " FROM entities ORDER BY name")],
|
|
858
|
+
"edges": [_row_edge(row) for row in self._conn.execute(
|
|
859
|
+
"SELECT " + _EDGE_COLS + " FROM edges ORDER BY ts")]}
|
|
860
|
+
save_graph(target, graph)
|
|
861
|
+
return target
|
|
862
|
+
|
|
863
|
+
def import_json(self, path):
|
|
864
|
+
data = load_graph(path)
|
|
865
|
+
imported = {"entities": 0, "edges": 0}
|
|
866
|
+
if not data["entities"] and not data["edges"]:
|
|
867
|
+
return imported
|
|
868
|
+
with self._conn:
|
|
869
|
+
for entity in data["entities"]:
|
|
870
|
+
self._conn.execute(
|
|
871
|
+
"INSERT OR IGNORE INTO entities(" + _ENTITY_COLS + ") "
|
|
872
|
+
"VALUES(?,?,?,?,?,?,?)",
|
|
873
|
+
(entity["id"], entity["name"], entity["type"],
|
|
874
|
+
json.dumps(entity["aliases"], ensure_ascii=False),
|
|
875
|
+
entity["first_seen"], entity["last_seen"],
|
|
876
|
+
json.dumps(entity["sources"], ensure_ascii=False)))
|
|
877
|
+
imported["entities"] += 1
|
|
878
|
+
for edge in data["edges"]:
|
|
879
|
+
self._conn.execute(
|
|
880
|
+
"INSERT OR IGNORE INTO edges(" + _EDGE_COLS + ") "
|
|
881
|
+
"VALUES(?,?,?,?,?,?,?)",
|
|
882
|
+
(edge["id"], edge["src"], edge["rel"], edge["dst"],
|
|
883
|
+
edge["ts"], edge["confidence"], edge["source_entry"]))
|
|
884
|
+
imported["edges"] += 1
|
|
885
|
+
return imported
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def open_store(path, recover=True, similarity_threshold=None):
|
|
889
|
+
return GraphStore(path, recover=recover,
|
|
890
|
+
similarity_threshold=similarity_threshold)
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def checkpoint(path):
|
|
894
|
+
"""Fold WAL into the SQLite file; safe no-op when the file is absent/broken."""
|
|
895
|
+
target = Path(path)
|
|
896
|
+
if not target.is_file():
|
|
897
|
+
return False
|
|
898
|
+
try:
|
|
899
|
+
conn = sqlite3.connect(str(target), timeout=5.0)
|
|
900
|
+
try:
|
|
901
|
+
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
902
|
+
finally:
|
|
903
|
+
conn.close()
|
|
904
|
+
return True
|
|
905
|
+
except Exception:
|
|
906
|
+
return False
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
def export_json(store, path=None):
|
|
910
|
+
if isinstance(store, GraphStore):
|
|
911
|
+
return store.export_json(path)
|
|
912
|
+
target = Path(path)
|
|
913
|
+
save_graph(target, store)
|
|
914
|
+
return target
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def import_json(store, path):
|
|
918
|
+
if isinstance(store, GraphStore):
|
|
919
|
+
return store.import_json(path)
|
|
920
|
+
data = load_graph(path)
|
|
921
|
+
graph = normalize_graph(store)
|
|
922
|
+
entity_sources = {entity["id"]: entity for entity in graph["entities"]}
|
|
923
|
+
merged = {row["id"]: row for row in graph["entities"]}
|
|
924
|
+
for entity in data["entities"]:
|
|
925
|
+
merged.setdefault(entity["id"], entity)
|
|
926
|
+
edges = {row["id"]: row for row in graph["edges"]}
|
|
927
|
+
for edge in data["edges"]:
|
|
928
|
+
edges.setdefault(edge["id"], edge)
|
|
929
|
+
target = {"schema": SCHEMA_VERSION, "entities": list(merged.values()),
|
|
930
|
+
"edges": list(edges.values())}
|
|
931
|
+
return target
|