ltcai 10.10.0 → 11.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/README.md +57 -42
  2. package/docs/CHANGELOG.md +68 -0
  3. package/docs/COMMUNITY_AND_PLUGINS.md +1 -1
  4. package/docs/DEVELOPMENT.md +1 -1
  5. package/docs/ONBOARDING.md +1 -1
  6. package/docs/OPERATIONS.md +1 -1
  7. package/docs/PERFORMANCE.md +135 -11
  8. package/docs/TRUST_MODEL.md +1 -1
  9. package/docs/WHY_LATTICE.md +1 -1
  10. package/docs/kg-schema.md +1 -1
  11. package/docs/v11.1.0_PRODUCT_INTELLIGENCE_PLAN.md +313 -0
  12. package/lattice_brain/__init__.py +1 -1
  13. package/lattice_brain/embeddings.py +12 -37
  14. package/lattice_brain/graph/_kg_contract.py +11 -0
  15. package/lattice_brain/graph/curator.py +1 -1
  16. package/lattice_brain/graph/discovery_index.py +30 -32
  17. package/lattice_brain/graph/fusion.py +184 -3
  18. package/lattice_brain/graph/image_vectors.py +230 -0
  19. package/lattice_brain/graph/ingest.py +11 -5
  20. package/lattice_brain/graph/proactive.py +138 -1
  21. package/lattice_brain/graph/projection.py +10 -2
  22. package/lattice_brain/graph/provenance.py +27 -2
  23. package/lattice_brain/graph/retrieval.py +249 -8
  24. package/lattice_brain/graph/retrieval_docgen.py +23 -23
  25. package/lattice_brain/graph/retrieval_policy.py +6 -0
  26. package/lattice_brain/graph/retrieval_reads.py +188 -1
  27. package/lattice_brain/graph/retrieval_vector.py +474 -110
  28. package/lattice_brain/graph/schema.py +134 -2
  29. package/lattice_brain/graph/vector_index/__init__.py +85 -0
  30. package/lattice_brain/graph/vector_index/base.py +170 -0
  31. package/lattice_brain/graph/vector_index/brute_force.py +114 -0
  32. package/lattice_brain/graph/vector_index/hnsw.py +293 -0
  33. package/lattice_brain/graph/vector_index/jobs.py +287 -0
  34. package/lattice_brain/graph/vector_index/quantized.py +151 -0
  35. package/lattice_brain/graph/vector_index/selector.py +131 -0
  36. package/lattice_brain/ingestion.py +259 -7
  37. package/lattice_brain/multimodal.py +738 -0
  38. package/lattice_brain/portability.py +654 -2
  39. package/lattice_brain/runtime/agent_runtime.py +1 -1
  40. package/lattice_brain/runtime/contracts.py +1 -1
  41. package/lattice_brain/runtime/multi_agent.py +1 -1
  42. package/lattice_brain/self_model.py +675 -0
  43. package/lattice_brain/synthesis.py +801 -0
  44. package/latticeai/__init__.py +1 -1
  45. package/latticeai/api/brain_intelligence.py +83 -1
  46. package/latticeai/api/chat_stream.py +4 -1
  47. package/latticeai/api/local_files.py +62 -0
  48. package/latticeai/api/memory.py +128 -1
  49. package/latticeai/api/models.py +1 -1
  50. package/latticeai/api/portability.py +130 -1
  51. package/latticeai/api/security_dashboard.py +48 -13
  52. package/latticeai/api/voice_capture.py +4 -1
  53. package/latticeai/api/workspace.py +11 -5
  54. package/latticeai/core/agent.py +5 -1
  55. package/latticeai/core/agent_prompts.py +66 -0
  56. package/latticeai/core/context_builder.py +92 -8
  57. package/latticeai/core/embedding_providers.py +548 -1
  58. package/latticeai/core/legacy_compatibility.py +1 -1
  59. package/latticeai/core/marketplace.py +1 -1
  60. package/latticeai/core/messages.py +51 -0
  61. package/latticeai/core/model_compat.py +2 -2
  62. package/latticeai/core/tool_registry.py +0 -7
  63. package/latticeai/core/workspace_os.py +43 -0
  64. package/latticeai/core/workspace_os_constants.py +1 -1
  65. package/latticeai/core/workspace_os_utils.py +4 -50
  66. package/latticeai/core/workspace_reorganization.py +335 -0
  67. package/latticeai/core/workspace_review_items.py +12 -1
  68. package/latticeai/integrations/telegram_bot.py +28 -10
  69. package/latticeai/models/router.py +1 -1
  70. package/latticeai/runtime/access_runtime.py +1 -1
  71. package/latticeai/runtime/build_phases.py +5 -2
  72. package/latticeai/runtime/network_boundary_wiring.py +9 -5
  73. package/latticeai/runtime/permission_mode_wiring.py +9 -6
  74. package/latticeai/runtime/persistence_runtime.py +41 -4
  75. package/latticeai/runtime/router_registration.py +3 -0
  76. package/latticeai/runtime/runtime_context.py +1 -0
  77. package/latticeai/services/architecture_readiness.py +1 -1
  78. package/latticeai/services/brain_intelligence.py +253 -0
  79. package/latticeai/services/change_proposals.py +50 -10
  80. package/latticeai/services/memory_service.py +35 -1
  81. package/latticeai/services/model_catalog.py +4 -3
  82. package/latticeai/services/model_engines.py +28 -14
  83. package/latticeai/services/multimodal_ports.py +87 -0
  84. package/latticeai/services/obsidian_bridge.py +618 -0
  85. package/latticeai/services/product_readiness.py +1 -1
  86. package/latticeai/services/self_model_service.py +171 -0
  87. package/latticeai/services/voice_capture.py +27 -1
  88. package/latticeai/tools/filesystem.py +4 -1
  89. package/package.json +1 -1
  90. package/scripts/bench_vector_index.py +295 -0
  91. package/scripts/check_current_release_docs.mjs +1 -1
  92. package/scripts/release_screen_claims.json +26 -0
  93. package/src-tauri/Cargo.lock +1 -1
  94. package/src-tauri/Cargo.toml +1 -1
  95. package/src-tauri/tauri.conf.json +1 -1
  96. package/static/app/asset-manifest.json +37 -37
  97. package/static/app/assets/{Act-CC8SpNRm.js → Act-D0HWqtn0.js} +1 -1
  98. package/static/app/assets/{AdminConsole-DGIY_Tb_.js → AdminConsole-D-QDW-A4.js} +1 -1
  99. package/static/app/assets/{Brain-BDSC2Mol.js → Brain-CzCsI1mi.js} +1 -1
  100. package/static/app/assets/BrainHome-Btns-_TA.js +2 -0
  101. package/static/app/assets/BrainSignals-2dHQNkns.js +1 -0
  102. package/static/app/assets/{Capture-ClcKLsmI.js → Capture-CT8v1StE.js} +1 -1
  103. package/static/app/assets/{CommandPalette-DpwbyF1S.js → CommandPalette-DoLXC2KH.js} +1 -1
  104. package/static/app/assets/{Library-D8VNtffM.js → Library-DDoxFE5c.js} +1 -1
  105. package/static/app/assets/{LivingBrain-CeWwc0NN.js → LivingBrain-BXMWIK_2.js} +1 -1
  106. package/static/app/assets/{ProductFlow-CS5CCM4t.js → ProductFlow-DOYf7JIs.js} +1 -1
  107. package/static/app/assets/{ReviewCard-Bap4yPfI.js → ReviewCard-COQsqidK.js} +1 -1
  108. package/static/app/assets/{System-odPxU1YL.js → System-BRllvYXd.js} +1 -1
  109. package/static/app/assets/arrow-left-DnyMzss-.js +1 -0
  110. package/static/app/assets/{bot-CmvhqTeF.js → bot-4BvN07ux.js} +1 -1
  111. package/static/app/assets/brain-uMb_5hnO.js +1 -0
  112. package/static/app/assets/{button-DyppYhFH.js → button-CDjtnAoU.js} +1 -1
  113. package/static/app/assets/{circle-pause-Bm5z7EBI.js → circle-pause-D_RMn7tp.js} +1 -1
  114. package/static/app/assets/{circle-play-GjKdH0Fg.js → circle-play-B5OpB8ae.js} +1 -1
  115. package/static/app/assets/{cpu-CuVMv-F5.js → cpu-BIlWInHf.js} +1 -1
  116. package/static/app/assets/{download-C9_946tf.js → download-BtjXfL3z.js} +1 -1
  117. package/static/app/assets/{folder-open-B3QKdxib.js → folder-open-DefMpxI2.js} +1 -1
  118. package/static/app/assets/{hard-drive-BwW9Hx4r.js → hard-drive-BQ8NZVkw.js} +1 -1
  119. package/static/app/assets/{index-BrUQ-bF8.js → index-0AvoEBzJ.js} +3 -3
  120. package/static/app/assets/index-vtEfYvQY.css +2 -0
  121. package/static/app/assets/{input-Db2Hw2VH.js → input-B_5ZJ9oy.js} +1 -1
  122. package/static/app/assets/{permissionCopy-BpbIHRio.js → permissionCopy-BqZ5tsgL.js} +1 -1
  123. package/static/app/assets/{primitives-4B_Ynf7r.js → primitives-CVwew78r.js} +1 -1
  124. package/static/app/assets/search-DkhnOKZt.js +1 -0
  125. package/static/app/assets/{share-2-BgvRTDPJ.js → share-2-D5zg_0fY.js} +1 -1
  126. package/static/app/assets/{shield-alert-DWFLyykG.js → shield-alert-B5pZzkUb.js} +1 -1
  127. package/static/app/assets/{textarea-Bk4vEqzT.js → textarea-nEVIweKY.js} +1 -1
  128. package/static/app/assets/{useFocusTrap-CGRXEg_p.js → useFocusTrap-Cm99AHlz.js} +1 -1
  129. package/static/app/assets/{useQuery-CdPzp3_U.js → useQuery-Dm__N6bL.js} +1 -1
  130. package/static/app/assets/{utils-orU8igel.js → utils-DcDMoZIe.js} +1 -1
  131. package/static/app/assets/{workspace-Brj2Ylb2.js → workspace-LtRRSKTf.js} +1 -1
  132. package/static/app/index.html +4 -4
  133. package/static/sw.js +1 -1
  134. package/static/app/assets/BrainHome-BYgYp3ON.js +0 -2
  135. package/static/app/assets/BrainSignals-DYwVFmcO.js +0 -1
  136. package/static/app/assets/arrow-left-J47Yde1U.js +0 -1
  137. package/static/app/assets/brain-DLT9uuGq.js +0 -1
  138. package/static/app/assets/index-CYb18LtP.css +0 -2
  139. package/static/app/assets/search-YH-7-x5Y.js +0 -1
@@ -28,7 +28,7 @@ from __future__ import annotations
28
28
  import json
29
29
  import os
30
30
  import re
31
- from typing import Any, Dict, Mapping, Optional
31
+ from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple
32
32
 
33
33
  from ..quiet import quiet
34
34
 
@@ -36,6 +36,26 @@ QUERY_CLASSES = ("fact", "code", "person", "recency")
36
36
 
37
37
  FUSION_WEIGHTS_ENV = "LATTICEAI_FUSION_WEIGHTS"
38
38
 
39
+ # ── fusion strategy (v11.1.0) ────────────────────────────────────────────────
40
+ # Linear alpha fusion needs the two channels' scores to be comparable. They
41
+ # are not: the lexical channel scores 1/rank while the vector channel scores a
42
+ # max-normalized cosine, so a query where every cosine lands near 0.9 flattens
43
+ # the vector channel to noise. Reciprocal Rank Fusion ignores score magnitudes
44
+ # entirely and fuses *positions*, which is exactly the property that survives
45
+ # incomparable scales.
46
+ #
47
+ # It is opt-in, per query class, and off everywhere by default: alpha fusion
48
+ # is what every existing ranking assertion in this repo describes, and a
49
+ # ranking change dressed up as a bug fix is how retrieval quality regressions
50
+ # get shipped. Turn it on with LATTICEAI_FUSION_STRATEGY=rrf (all classes) or
51
+ # a JSON object like {"code": "rrf"} (per class).
52
+ FUSION_STRATEGY_ENV = "LATTICEAI_FUSION_STRATEGY"
53
+ FUSION_STRATEGIES = ("alpha", "rrf")
54
+ DEFAULT_FUSION_STRATEGY: Dict[str, str] = dict.fromkeys(QUERY_CLASSES, "alpha")
55
+ #: The smoothing constant from the original RRF paper (Cormack et al., 2009).
56
+ #: Larger k flattens the curve, so rank 1 wins by less.
57
+ DEFAULT_RRF_K = 60
58
+
39
59
  # Per-class fusion weights. "fact" mirrors the pre-fusion-gate defaults
40
60
  # (SearchService DEFAULT_HYBRID_WEIGHTS + graph-layer alpha=0.6) so the
41
61
  # fallback class introduces zero behavioral drift.
@@ -145,16 +165,163 @@ def fusion_weight_table(
145
165
  return table
146
166
 
147
167
 
168
+ def _env_strategy_overrides() -> Dict[str, str]:
169
+ """Parse ``LATTICEAI_FUSION_STRATEGY`` → per-class strategy overrides.
170
+
171
+ Two accepted forms: a bare strategy name (``rrf``) applies to every class,
172
+ a JSON object (``{"code": "rrf"}``) applies per class. Anything else —
173
+ a typo, malformed JSON, an unknown strategy — is ignored rather than
174
+ raising, so a bad env var degrades to the default fusion instead of
175
+ taking every search down.
176
+ """
177
+ raw = os.getenv(FUSION_STRATEGY_ENV, "").strip()
178
+ if not raw:
179
+ return {}
180
+ if raw.lower() in FUSION_STRATEGIES:
181
+ return dict.fromkeys(QUERY_CLASSES, raw.lower())
182
+ try:
183
+ payload = json.loads(raw)
184
+ except (ValueError, TypeError):
185
+ return {}
186
+ if not isinstance(payload, dict):
187
+ return {}
188
+ return {
189
+ str(cls): str(value).lower()
190
+ for cls, value in payload.items()
191
+ if cls in DEFAULT_FUSION_STRATEGY and str(value).lower() in FUSION_STRATEGIES
192
+ }
193
+
194
+
195
+ def fusion_strategy_table(
196
+ overrides: Optional[Mapping[str, str]] = None,
197
+ ) -> Dict[str, str]:
198
+ """Full per-class strategy table: defaults ← env override ← caller."""
199
+ table = dict(DEFAULT_FUSION_STRATEGY)
200
+ for source in (_env_strategy_overrides(), overrides or {}):
201
+ for cls, value in source.items():
202
+ if cls in table and str(value).lower() in FUSION_STRATEGIES:
203
+ table[cls] = str(value).lower()
204
+ return table
205
+
206
+
207
+ def rrf_score(rank: int, *, k: int = DEFAULT_RRF_K) -> float:
208
+ """One channel's contribution for an item it ranked at ``rank`` (1-based)."""
209
+ return 1.0 / (int(k) + max(1, int(rank)))
210
+
211
+
212
+ def rrf_fuse(
213
+ rankings: Mapping[str, Sequence[str]],
214
+ *,
215
+ k: int = DEFAULT_RRF_K,
216
+ weights: Optional[Mapping[str, float]] = None,
217
+ ) -> Dict[str, float]:
218
+ """Reciprocal Rank Fusion over per-channel id rankings.
219
+
220
+ ``rankings`` maps a channel name to its ids in rank order (best first).
221
+ An id absent from a channel simply contributes nothing from it — RRF has
222
+ no notion of a "missing score" to guess at, which is the other reason it
223
+ survives channels that fail independently.
224
+
225
+ ``weights`` optionally scales a channel's contribution (a channel missing
226
+ from the mapping weighs 1.0). Deterministic and pure.
227
+ """
228
+ fused: Dict[str, float] = {}
229
+ for channel, ordered in rankings.items():
230
+ weight = 1.0 if weights is None else float(weights.get(channel, 1.0))
231
+ for rank, item_id in enumerate(ordered, start=1):
232
+ fused[item_id] = fused.get(item_id, 0.0) + weight * rrf_score(rank, k=k)
233
+ return fused
234
+
235
+
236
+ # ── graph traversal candidate expansion (v11.1.0) ────────────────────────────
237
+ # Both retrieval channels can only return what they matched. A note that
238
+ # never says "postgres" but is one CONTAINS edge away from the one that does
239
+ # is unreachable by either — and it is often the answer. Expansion walks one
240
+ # hop out from the top hits and offers those neighbours as candidates.
241
+ #
242
+ # It is capped, off by default, and reports its own counts, because the
243
+ # failure mode is dilution: an unbounded expansion turns a precise answer into
244
+ # a tour of the graph.
245
+ GRAPH_EXPANSION_ENV = "LATTICEAI_GRAPH_EXPANSION"
246
+ DEFAULT_EXPANSION_SEEDS = 3
247
+ DEFAULT_EXPANSION_CAP = 5
248
+ #: Expanded candidates inherit a damped share of their seed's score: they are
249
+ #: related to a hit, they are not themselves a hit.
250
+ EXPANSION_DECAY = 0.5
251
+
252
+
253
+ def graph_expansion_enabled() -> bool:
254
+ """True when ``LATTICEAI_GRAPH_EXPANSION`` opts in (default: off)."""
255
+ raw = os.getenv(GRAPH_EXPANSION_ENV, "").strip().lower()
256
+ return raw in {"1", "true", "yes", "on"}
257
+
258
+
259
+ def expand_with_neighbors(
260
+ seeds: Sequence[Tuple[str, float]],
261
+ neighbors_fn: Callable[[str], Any],
262
+ *,
263
+ exclude: Optional[Sequence[str]] = None,
264
+ cap: int = DEFAULT_EXPANSION_CAP,
265
+ ) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
266
+ """One-hop neighbours of ``seeds`` as extra candidates + an honest report.
267
+
268
+ ``seeds`` is ``[(node_id, score)]`` best first; ``neighbors_fn`` is the
269
+ store's ``neighbors``. Returns ``(candidates, report)`` where each
270
+ candidate carries its originating seed and a decayed score, and the report
271
+ states how many seeds were walked, how many candidates were kept, and
272
+ whether the cap bit. A neighbour lookup that fails is skipped and counted,
273
+ never raised: expansion is an optimization, not a dependency.
274
+ """
275
+ known = {str(item) for item in (exclude or ())}
276
+ cap = max(0, int(cap))
277
+ candidates: List[Dict[str, Any]] = []
278
+ failures = 0
279
+ for seed_id, seed_score in seeds:
280
+ if len(candidates) >= cap:
281
+ break
282
+ try:
283
+ payload = neighbors_fn(seed_id) or {}
284
+ except Exception: # noqa: BLE001 — a bad seed must not fail the search
285
+ failures += 1
286
+ continue
287
+ for node in payload.get("neighbors") or []:
288
+ node_id = str(node.get("id") or "")
289
+ if not node_id or node_id in known:
290
+ continue
291
+ known.add(node_id)
292
+ candidates.append(
293
+ {
294
+ "node": node,
295
+ "seed": str(seed_id),
296
+ "score": round(float(seed_score) * EXPANSION_DECAY, 6),
297
+ }
298
+ )
299
+ if len(candidates) >= cap:
300
+ break
301
+ report = {
302
+ "enabled": True,
303
+ "seeds": len(seeds),
304
+ "added": len(candidates),
305
+ "cap": cap,
306
+ "truncated": len(candidates) >= cap,
307
+ "failed_seeds": failures,
308
+ }
309
+ return candidates, report
310
+
311
+
148
312
  def fusion_profile(
149
313
  query: Any,
150
314
  *,
151
315
  overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
316
+ strategies: Optional[Mapping[str, str]] = None,
152
317
  ) -> Dict[str, Any]:
153
318
  """Classify ``query`` and return its resolved fusion weights.
154
319
 
155
320
  Returns ``{"query_class": str, "weights": {keyword, vector, graph},
156
- "alpha": float}``. ``weights`` feeds the three-channel service fusion;
157
- ``alpha`` feeds the two-channel graph-layer fusion.
321
+ "alpha": float, "strategy": "alpha"|"rrf"}``. ``weights`` feeds the
322
+ three-channel service fusion; ``alpha`` feeds the two-channel graph-layer
323
+ fusion; ``strategy`` selects *how* those channels combine and is
324
+ ``"alpha"`` for every class unless configured otherwise.
158
325
  """
159
326
  query_class = classify_query(query)
160
327
  table = fusion_weight_table(overrides)
@@ -170,14 +337,28 @@ def fusion_profile(
170
337
  "graph": resolved["graph"],
171
338
  },
172
339
  "alpha": resolved["alpha"],
340
+ "strategy": fusion_strategy_table(strategies)[query_class],
173
341
  }
174
342
 
175
343
 
176
344
  __all__ = [
345
+ "DEFAULT_EXPANSION_CAP",
346
+ "DEFAULT_EXPANSION_SEEDS",
347
+ "DEFAULT_FUSION_STRATEGY",
177
348
  "DEFAULT_FUSION_WEIGHTS",
349
+ "DEFAULT_RRF_K",
350
+ "EXPANSION_DECAY",
351
+ "FUSION_STRATEGIES",
352
+ "FUSION_STRATEGY_ENV",
178
353
  "FUSION_WEIGHTS_ENV",
354
+ "GRAPH_EXPANSION_ENV",
179
355
  "QUERY_CLASSES",
180
356
  "classify_query",
357
+ "expand_with_neighbors",
181
358
  "fusion_profile",
359
+ "fusion_strategy_table",
182
360
  "fusion_weight_table",
361
+ "graph_expansion_enabled",
362
+ "rrf_fuse",
363
+ "rrf_score",
183
364
  ]
@@ -0,0 +1,230 @@
1
+ """The image vector space — stored apart, joined by late fusion (v11.1.0).
2
+
3
+ A CLIP image vector and a BGE text vector are both "1024 floats", and that is
4
+ the entire extent of what they have in common. Filing them in one index means
5
+ every cosine between them is a number with no meaning, and the ranking that
6
+ comes out looks exactly like a working search.
7
+
8
+ So image vectors get their own table (``image_embeddings``) keyed by the vision
9
+ model that produced them, and their own search. Text queries never touch this
10
+ index: they reach pictures through OCR text and captions, which are text and
11
+ live in the text index like everything else. This index answers one question —
12
+ *which stored images look like this vector* — and its scores enter
13
+ ``hybrid_search`` by late fusion, as a separate channel with its own weight,
14
+ rather than by pretending to be comparable up front.
15
+
16
+ The table is created on demand, is derivable from the images themselves, and is
17
+ never the source of truth: dropping it costs a re-embed, not a memory.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import sqlite3
23
+ import struct
24
+ from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
25
+
26
+ from .vector_index import build_index, resolve_vector_index
27
+
28
+ IMAGE_VECTOR_TABLE = "image_embeddings"
29
+ #: Default weight the image channel carries when fused with text scores.
30
+ DEFAULT_IMAGE_FUSION_WEIGHT = 0.5
31
+
32
+
33
+ def ensure_image_vector_table(conn: sqlite3.Connection) -> None:
34
+ """Create the image-vector table if this graph has never held one."""
35
+ conn.execute(
36
+ f"""
37
+ CREATE TABLE IF NOT EXISTS {IMAGE_VECTOR_TABLE} (
38
+ node_id TEXT PRIMARY KEY,
39
+ model_id TEXT NOT NULL,
40
+ dim INTEGER NOT NULL,
41
+ space TEXT NOT NULL DEFAULT 'image',
42
+ vector BLOB NOT NULL,
43
+ updated_at TEXT
44
+ )
45
+ """
46
+ )
47
+
48
+
49
+ def encode_vector(vector: Sequence[float]) -> bytes:
50
+ values = [float(value) for value in vector]
51
+ return struct.pack(f"<{len(values)}f", *values)
52
+
53
+
54
+ def decode_vector(payload: bytes) -> List[float]:
55
+ count = len(payload) // 4
56
+ return list(struct.unpack(f"<{count}f", payload[: count * 4]))
57
+
58
+
59
+ def record_image_vector(
60
+ store: Any,
61
+ *,
62
+ node_id: str,
63
+ vector: Sequence[float],
64
+ model_id: str,
65
+ space: str = "image",
66
+ updated_at: Optional[str] = None,
67
+ ) -> bool:
68
+ """Persist one image vector. Returns False when there is nothing to store.
69
+
70
+ Never raises: an image whose vector could not be filed is still a memory,
71
+ and :func:`image_index_status` reports it as backlog.
72
+ """
73
+ values = [float(value) for value in (vector or [])]
74
+ if not node_id or not values or not model_id:
75
+ return False
76
+ try:
77
+ with store._connect() as conn:
78
+ ensure_image_vector_table(conn)
79
+ conn.execute(
80
+ f"""
81
+ INSERT INTO {IMAGE_VECTOR_TABLE}
82
+ (node_id, model_id, dim, space, vector, updated_at)
83
+ VALUES (?, ?, ?, ?, ?, ?)
84
+ ON CONFLICT(node_id) DO UPDATE SET
85
+ model_id=excluded.model_id,
86
+ dim=excluded.dim,
87
+ space=excluded.space,
88
+ vector=excluded.vector,
89
+ updated_at=excluded.updated_at
90
+ """,
91
+ (
92
+ str(node_id),
93
+ str(model_id),
94
+ len(values),
95
+ str(space or "image"),
96
+ encode_vector(values),
97
+ updated_at,
98
+ ),
99
+ )
100
+ return True
101
+ except Exception: # noqa: BLE001 — a vector is derivable; the image is not
102
+ return False
103
+
104
+
105
+ def _rows(
106
+ store: Any, model_id: Optional[str], dim: Optional[int]
107
+ ) -> List[Tuple[str, List[float]]]:
108
+ with store._connect() as conn:
109
+ ensure_image_vector_table(conn)
110
+ if model_id:
111
+ cursor = conn.execute(
112
+ f"SELECT node_id, vector, dim FROM {IMAGE_VECTOR_TABLE} WHERE model_id=? ORDER BY node_id ASC",
113
+ (str(model_id),),
114
+ )
115
+ else:
116
+ cursor = conn.execute(
117
+ f"SELECT node_id, vector, dim FROM {IMAGE_VECTOR_TABLE} ORDER BY node_id ASC"
118
+ )
119
+ rows = cursor.fetchall()
120
+ out: List[Tuple[str, List[float]]] = []
121
+ for row in rows:
122
+ if dim is not None and int(row["dim"]) != dim:
123
+ # A different width means a different model. Comparing across it is
124
+ # the exact silent wrongness this whole module exists to prevent.
125
+ continue
126
+ out.append((str(row["node_id"]), decode_vector(row["vector"])))
127
+ return out
128
+
129
+
130
+ def image_similarity_search(
131
+ store: Any,
132
+ vector: Sequence[float],
133
+ *,
134
+ top_k: int = 10,
135
+ model_id: Optional[str] = None,
136
+ min_score: float = 0.0,
137
+ ) -> Dict[str, Any]:
138
+ """Rank stored images against a query *image* vector.
139
+
140
+ Scoring is delegated to the pluggable backend layer (Track 1), so this
141
+ index picks up ``LATTICEAI_VECTOR_INDEX`` and reports the same
142
+ ``approx``/``exhaustive`` honesty bits as the text index.
143
+ """
144
+ query = [float(value) for value in (vector or [])]
145
+ result: Dict[str, Any] = {
146
+ "matches": [],
147
+ "count": 0,
148
+ "candidates": 0,
149
+ "model_id": model_id,
150
+ "detail": None,
151
+ }
152
+ if not query:
153
+ result["detail"] = "an image query needs an image vector"
154
+ return result
155
+ try:
156
+ rows = _rows(store, model_id, len(query))
157
+ except Exception as exc: # noqa: BLE001 — degrade, never fail the search
158
+ result["detail"] = f"image vector index unavailable: {exc}"
159
+ return result
160
+ result["candidates"] = len(rows)
161
+ selection = resolve_vector_index()
162
+ index = build_index(selection, dim=len(query))
163
+ index.rebuild((node_id, values, {}) for node_id, values in rows)
164
+ scored = index.search(query, max(1, int(top_k)), {"min_score": min_score})
165
+ result["matches"] = [
166
+ {"node_id": node_id, "score": round(float(score), 6)} for node_id, score in scored
167
+ ]
168
+ result["count"] = len(result["matches"])
169
+ result["index"] = selection.as_dict()
170
+ return result
171
+
172
+
173
+ def image_index_status(store: Any) -> Dict[str, Any]:
174
+ """How many images carry a vector, and under which model."""
175
+ status: Dict[str, Any] = {"vectors": 0, "models": {}, "detail": None}
176
+ try:
177
+ with store._connect() as conn:
178
+ ensure_image_vector_table(conn)
179
+ rows = conn.execute(
180
+ f"SELECT model_id, COUNT(*) AS total FROM {IMAGE_VECTOR_TABLE} GROUP BY model_id"
181
+ ).fetchall()
182
+ except Exception as exc: # noqa: BLE001 — status must never raise
183
+ status["detail"] = str(exc)
184
+ return status
185
+ models = {str(row["model_id"]): int(row["total"]) for row in rows}
186
+ status["models"] = models
187
+ status["vectors"] = sum(models.values())
188
+ return status
189
+
190
+
191
+ def fuse_image_scores(
192
+ matches: Iterable[Dict[str, Any]],
193
+ image_scores: Dict[str, float],
194
+ *,
195
+ weight: float = DEFAULT_IMAGE_FUSION_WEIGHT,
196
+ ) -> int:
197
+ """Late-fuse image-space scores into an already-fused match list.
198
+
199
+ Mutates each match in place — ``scores.image`` records the raw image-space
200
+ similarity and ``score`` blends it in — and returns how many matches the
201
+ image channel actually touched. Late fusion is the point: the two spaces
202
+ are ranked separately and combined at the end, so neither is ever asked to
203
+ interpret the other's numbers.
204
+ """
205
+ weight = max(0.0, min(1.0, float(weight)))
206
+ touched = 0
207
+ for match in matches:
208
+ node_id = str(match.get("node_id") or "")
209
+ raw = image_scores.get(node_id)
210
+ if raw is None:
211
+ continue
212
+ touched += 1
213
+ scores = match.setdefault("scores", {})
214
+ scores["image"] = round(float(raw), 6)
215
+ blended = (1.0 - weight) * float(match.get("score") or 0.0) + weight * float(raw)
216
+ match["score"] = round(blended, 6)
217
+ return touched
218
+
219
+
220
+ __all__ = [
221
+ "DEFAULT_IMAGE_FUSION_WEIGHT",
222
+ "IMAGE_VECTOR_TABLE",
223
+ "decode_vector",
224
+ "encode_vector",
225
+ "ensure_image_vector_table",
226
+ "fuse_image_scores",
227
+ "image_index_status",
228
+ "image_similarity_search",
229
+ "record_image_vector",
230
+ ]
@@ -636,14 +636,20 @@ class KnowledgeGraphIngestMixin(_Core):
636
636
  modified_at: Optional[str] = None,
637
637
  conversation_id: Optional[str] = None,
638
638
  metadata: Optional[Dict[str, Any]] = None,
639
+ node_type: str = "Document",
639
640
  ) -> Dict[str, Any]:
640
641
  """Unified text/web ingestion: one shape for URL, browser tab, note, text.
641
642
 
642
- Creates a content ``Document`` node (idempotent by content hash), a
643
- ``Source`` node linked via ``indexed_from``, RAG chunks, and extracted
643
+ Creates a content node (idempotent by content hash), a ``Source`` node
644
+ linked via ``indexed_from``, RAG chunks, and extracted
644
645
  Concept/Task/Decision nodes — mirroring ingest_document for non-file
645
646
  sources. Returns the full set of ids the caller needs to record
646
647
  provenance, including ``duplicate`` (was the content already indexed).
648
+
649
+ ``node_type`` names the content node. It defaults to ``Document``, so
650
+ every historical caller is unchanged; a door that knows its material is
651
+ something else (``Audio`` for a recording) says so here rather than
652
+ writing a second ingest path to say it.
647
653
  """
648
654
  source_type = str(source_type or "text")
649
655
  text = str(text or "")
@@ -674,11 +680,11 @@ class KnowledgeGraphIngestMixin(_Core):
674
680
 
675
681
  with self._connect() as conn:
676
682
  duplicate = self._node_exists(conn, content_id)
677
- # ── 콘텐츠 노드 (점: 명사 — 문서) ────────────────────────────────
683
+ # ── 콘텐츠 노드 (점: 명사 — 기본은 문서) ─────────────────────────
678
684
  self._upsert_node(
679
685
  conn,
680
686
  content_id,
681
- "Document",
687
+ node_type,
682
688
  title,
683
689
  summary=(text or title)[:500],
684
690
  metadata=node_meta,
@@ -813,7 +819,7 @@ class KnowledgeGraphIngestMixin(_Core):
813
819
 
814
820
  return {
815
821
  "node_id": content_id,
816
- "type": "Document",
822
+ "type": node_type,
817
823
  "source_node_id": source_node_id,
818
824
  "content_hash": content_hash,
819
825
  "chunk_ids": chunk_ids,
@@ -48,6 +48,21 @@ _DEFAULT_CONTRADICTION_NODES = 300
48
48
  _DEFAULT_NEAR_THRESHOLD = 0.75
49
49
  _DEFAULT_MAX_PAIRS = 200
50
50
  _DEFAULT_STALE_DAYS = 90 # mirrors MemoryQualityManager.apply_retention
51
+ _DEFAULT_HALF_LIFE_DAYS = 30.0
52
+ # Node types that record *what happened* rather than *what is known*. Only
53
+ # these are offered for consolidation: folding a Decision or a Document into a
54
+ # summary would lose the thing the user actually keeps a Brain for.
55
+ _EPISODIC_TYPES = frozenset(
56
+ {
57
+ "chat",
58
+ "conversation",
59
+ "message",
60
+ "airesponse",
61
+ "ai_response",
62
+ "event",
63
+ "chunk",
64
+ }
65
+ )
51
66
 
52
67
 
53
68
  def _parse_ts(value: Any) -> Optional[datetime]:
@@ -78,6 +93,28 @@ def _node_text(node: Dict[str, Any]) -> str:
78
93
  return f"{title} {summary}".strip()
79
94
 
80
95
 
96
+ def _is_episodic(node: Dict[str, Any]) -> bool:
97
+ return str(node.get("type") or "").strip().lower() in _EPISODIC_TYPES
98
+
99
+
100
+ def _access_count(node: Dict[str, Any], stored: Optional[Dict[str, Any]]) -> float:
101
+ """Access count for a node: ingested metadata first, then the store counter.
102
+
103
+ A surface that already tracks reads (``metadata.access_count``) is more
104
+ accurate than our own read-path counter, so it wins; ``0`` from metadata is
105
+ a real answer and is not treated as "missing".
106
+ """
107
+ metadata = node.get("metadata")
108
+ if isinstance(metadata, dict):
109
+ for key in ("access_count", "accesses", "access"):
110
+ value = metadata.get(key)
111
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
112
+ return float(value)
113
+ if stored is None:
114
+ return 0.0
115
+ return float(stored.get("accesses") or 0.0)
116
+
117
+
81
118
  def _slim(node: Dict[str, Any]) -> Dict[str, Any]:
82
119
  return {
83
120
  "id": node.get("id"),
@@ -129,6 +166,17 @@ class ProactiveBrain:
129
166
  edges.append(normalized)
130
167
  return {"nodes": nodes, "edges": edges}
131
168
 
169
+ def sample(
170
+ self, *, workspace_id: Optional[str] = None, limit: Optional[int] = None
171
+ ) -> Dict[str, List[Dict[str, Any]]]:
172
+ """One normalized graph sample (``source``/``target`` edge keys).
173
+
174
+ Public seam for :mod:`lattice_brain.synthesis`, which needs the *same*
175
+ sample for several passes; taking it once keeps a synthesis run to one
176
+ graph read and keeps every pass looking at identical data.
177
+ """
178
+ return self._sample(workspace_id=workspace_id, limit=limit)
179
+
132
180
  # ── duplicates ───────────────────────────────────────────────────────
133
181
 
134
182
  def find_duplicates(
@@ -246,6 +294,16 @@ class ProactiveBrain:
246
294
  sample["nodes"], sample["edges"], max_nodes=max_nodes
247
295
  )
248
296
 
297
+ def contradictions_in(
298
+ self,
299
+ nodes: List[Dict[str, Any]],
300
+ edges: List[Dict[str, Any]],
301
+ *,
302
+ max_nodes: int = _DEFAULT_CONTRADICTION_NODES,
303
+ ) -> Dict[str, Any]:
304
+ """Contradiction signals over an already-taken :meth:`sample`."""
305
+ return self._detect_contradictions_in(nodes, edges, max_nodes=max_nodes)
306
+
249
307
  def _detect_contradictions_in(
250
308
  self,
251
309
  nodes: List[Dict[str, Any]],
@@ -403,6 +461,85 @@ class ProactiveBrain:
403
461
  "generated_at": datetime.now(timezone.utc).isoformat(),
404
462
  }
405
463
 
464
+ # ── importance & decay (v11.1.0) ─────────────────────────────────────
465
+
466
+ def importance_report(
467
+ self,
468
+ *,
469
+ workspace_id: Optional[str] = None,
470
+ limit: Optional[int] = None,
471
+ half_life_days: float = _DEFAULT_HALF_LIFE_DAYS,
472
+ max_candidates: int = 20,
473
+ sample: Optional[Dict[str, List[Dict[str, Any]]]] = None,
474
+ ) -> Dict[str, Any]:
475
+ """Score every sampled node by use, then name the weakest episodic ones.
476
+
477
+ The score is deliberately boring and reproducible — no model, no
478
+ randomness::
479
+
480
+ score = (1 + accesses + degree) * 0.5 ** (age_days / half_life)
481
+
482
+ *accesses* prefers a real counter: ``metadata.access_count`` when the
483
+ ingesting surface recorded one, otherwise the store's own read-path
484
+ counter (``access_stats``), otherwise zero. *Episodic* types (chats,
485
+ messages, events, chunks) are the only consolidation candidates —
486
+ a decayed Document or Decision is stale knowledge to review, not
487
+ noise to fold away.
488
+ """
489
+ data = sample if sample is not None else self._sample(
490
+ workspace_id=workspace_id, limit=limit
491
+ )
492
+ nodes, edges = data["nodes"], data["edges"]
493
+ degree: Dict[str, int] = {}
494
+ for edge in edges:
495
+ for key in ("source", "target"):
496
+ node_id = str(edge.get(key) or "")
497
+ if node_id:
498
+ degree[node_id] = degree.get(node_id, 0) + 1
499
+
500
+ stats_fn = getattr(self._store, "access_stats", None)
501
+ stored: Dict[str, Any] = {}
502
+ if callable(stats_fn):
503
+ try:
504
+ stored = dict(stats_fn([n.get("id") for n in nodes]) or {})
505
+ except Exception: # noqa: BLE001 — the report degrades, never fails
506
+ logger.exception("access stats read failed")
507
+
508
+ now = datetime.now(timezone.utc)
509
+ half_life = max(0.5, float(half_life_days))
510
+ scored: List[Dict[str, Any]] = []
511
+ for node in nodes:
512
+ node_id = str(node.get("id") or "")
513
+ accesses = _access_count(node, stored.get(node_id))
514
+ ts = _parse_ts(node.get("updated_at"))
515
+ age_days = 0.0 if ts is None else max(
516
+ 0.0, (now - ts).total_seconds() / 86400.0
517
+ )
518
+ decay = 0.5 ** (age_days / half_life)
519
+ scored.append(
520
+ {
521
+ **_slim(node),
522
+ "accesses": accesses,
523
+ "degree": degree.get(node_id, 0),
524
+ "age_days": round(age_days, 2),
525
+ "score": round((1.0 + accesses + degree.get(node_id, 0)) * decay, 4),
526
+ "episodic": _is_episodic(node),
527
+ }
528
+ )
529
+ scored.sort(key=lambda item: (item["score"], str(item.get("id") or "")))
530
+ candidates = [item for item in scored if item["episodic"]][
531
+ : max(1, int(max_candidates))
532
+ ]
533
+ return {
534
+ "nodes_scanned": len(nodes),
535
+ "half_life_days": half_life,
536
+ "access_source": "store" if stored else "metadata",
537
+ "candidates": candidates,
538
+ "candidate_count": len(candidates),
539
+ "strongest": list(reversed(scored[-5:])),
540
+ "generated_at": datetime.now(timezone.utc).isoformat(),
541
+ }
542
+
406
543
  # ── consolidation ────────────────────────────────────────────────────
407
544
 
408
545
  def consolidate_duplicates(
@@ -470,7 +607,7 @@ class ProactiveBrain:
470
607
  for group in groups:
471
608
  try:
472
609
  if merge_fn is None: # guarded by apply_supported above
473
- raise RuntimeError("store has no merge_nodes")
610
+ raise RuntimeError("store has no merge_nodes") # pragma: no cover — unreachable: callable(merge_fn) above proves it is not None
474
611
  result = merge_fn(group["keep"], group["remove"])
475
612
  applied.append({"keep": group["keep"], "result": result})
476
613
  except Exception as exc: # keep going; report per-group failure